From 7348620fcd279ed6a8dcf4df484ba4ac807f7466 Mon Sep 17 00:00:00 2001 From: Madalena Melo Date: Fri, 18 Sep 2026 13:09:35 +0200 Subject: [PATCH 01/22] :books: Update CONTRIBUTING.md with AI policy (#11782) Include policy for AI assisted contributions and fix "link to title format" section Signed-off-by: Madalena Melo --- CONTRIBUTING.md | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a7aeac3f3f..000157096b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -11,12 +11,13 @@ Center](https://help.penpot.app/). ## Table of Contents - [Prerequisites](#prerequisites) +- [AI-Assisted Contributions](#ai-assisted-contributions) - [Reporting Bugs](#reporting-bugs) - [Pull Requests](#pull-requests) - [Workflow](#workflow) - [Branch naming](#branch-naming) - [Format](#format) - - [Title format](#title-format) + - [Title format](#title) - [Description](#description) - [Review process](#review-process) - [What we won't accept](#what-we-wont-accept) @@ -38,6 +39,12 @@ Center](https://help.penpot.app/). - **Issue tracker**: We use [GitHub Issues](https://github.com/penpot/penpot/issues) for public bugs and [Taiga](https://tree.taiga.io/project/penpot/) for internal project management. Changelog entries reference both. + +## AI-Assisted Contributions + +We support the responsible use of AI tools in the development process. However, all contributions to Penpot - including issues, pull requests, and any other submissions - must meet a reasonable standard of quality, accuracy, and human oversight. + +If AI-assisted content is used, it must be carefully reviewed and verified by a human before submission. Contributions that don't meet these standards may be rejected or closed without detailed review or a reply. ## Reporting Bugs @@ -84,7 +91,7 @@ Branch names are not enforced, but we recommend the following: and no slashes (e.g. `fix-ellipse-icon-typo`, `feat-auto-link-libraries`). Since PRs are squash-merged, the branch name does not survive into the -commit history — what matters is the [PR title](#title-format). +commit history — what matters is the [PR title](#title). ### Format @@ -129,7 +136,7 @@ for more concrete information. force-push during review, as it breaks comment threads. - PRs require at least **one approval** before merge. - We use **squash-merge** by default. The PR title becomes the final - commit message, so follow the [title format](#title-format) above. + commit message, so follow the [title format](#title) above. ### What we won't accept From 0d2632fc8cb32bd028ff9241db1798e9d489192a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Barrag=C3=A1n=20Merino?= Date: Tue, 22 Sep 2026 15:39:27 +0200 Subject: [PATCH 02/22] :bug: Create /usr/local/bin before extracting pnpm in DHI images MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit dhi.io/node:24.20.0-debian13-dev does not pre-create /usr/local/bin the way a regular Debian image does, so tar -xzf ... -C /usr/local/bin failed with "Cannot open: No such file or directory" right after the checksum check passed. Introduced in #11790 when pnpm moved from Corepack to a downloaded standalone binary. Dockerfile.media-processor and docker/devenv/Dockerfile are unaffected: both extract into /opt/node/bin, which already exists from the prior Node.js install step. Signed-off-by: David Barragán Merino --- docker/images/Dockerfile.exporter | 128 +----------------------------- docker/images/Dockerfile.mcp | 1 + 2 files changed, 2 insertions(+), 127 deletions(-) diff --git a/docker/images/Dockerfile.exporter b/docker/images/Dockerfile.exporter index 3b12e37303..e6ce64ca4a 100644 --- a/docker/images/Dockerfile.exporter +++ b/docker/images/Dockerfile.exporter @@ -1,127 +1 @@ -FROM dhi.io/node:24.20.0-debian13-dev -LABEL maintainer="Penpot " - -ENV LANG=en_US.UTF-8 \ - LC_ALL=en_US.UTF-8 \ - DEBIAN_FRONTEND=noninteractive \ - PATH=/opt/imagick/bin:$PATH \ - PLAYWRIGHT_BROWSERS_PATH=/opt/penpot/browsers - -RUN set -ex; \ - apt-get -qq update; \ - apt-get -qqy --no-install-recommends install passwd; \ - useradd -U -M -u 1001 -s /bin/false -d /opt/penpot penpot; \ - apt-get -qq -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" dist-upgrade; \ - apt-get -qqy --no-install-recommends install \ - curl \ - tzdata \ - locales \ - ca-certificates \ - ; \ - apt-get clean; \ - rm -rf /var/lib/apt/lists/*; \ - echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen; \ - locale-gen; \ - find /usr/share/i18n/locales/ -type f ! -name "en_US" ! -name "POSIX" ! -name "C" -delete; - -RUN set -ex; \ - apt-get -qq update; \ - apt-get -qqy install \ - \ - xvfb \ - fonts-noto-color-emoji \ - fonts-unifont \ - libfontconfig1 \ - libfreetype6 \ - xfonts-cyrillic \ - xfonts-scalable \ - fonts-liberation \ - fonts-ipafont-gothic \ - fonts-wqy-zenhei \ - fonts-tlwg-loma-otf \ - fonts-freefont-ttf \ - poppler-utils \ - \ - libasound2 \ - libatk-bridge2.0-0 \ - libatk1.0-0 \ - libatspi2.0-0 \ - libcairo2 \ - libcups2 \ - libdbus-1-3 \ - libdrm2 \ - libgbm1 \ - libglib2.0-0 \ - libnspr4 \ - libnss3 \ - libpango-1.0-0 \ - libx11-6 \ - libxcb1 \ - libxcomposite1 \ - libxdamage1 \ - libxext6 \ - libxfixes3 \ - libxkbcommon0 \ - libxrandr2 \ - \ - libgomp1 \ - libheif1 \ - libjpeg62-turbo \ - liblcms2-2 \ - libopenexr-3-1-30 \ - libopenjp2-7 \ - libpng16-16 \ - librsvg2-2 \ - libtiff6 \ - libwebp7 \ - libwebpdemux2 \ - libwebpmux3 \ - libxml2 \ - libzip5 \ - libzstd1 \ - ; \ - apt-get clean; \ - rm -rf /var/lib/apt/lists/*; \ - mkdir -p /opt/penpot; \ - chown -R penpot:penpot /opt/penpot; - -ARG BUNDLE_PATH="./bundle-exporter/" -COPY --chown=penpot:penpot $BUNDLE_PATH /opt/penpot/exporter/ -COPY --from=penpotapp/imagemagick:7.1.2-27 /opt/imagick /opt/imagick - -WORKDIR /opt/penpot/exporter - -# pnpm ships as a system binary (same tarball + SHA pin as -# docker/devenv/Dockerfile); the generated ./setup expects it on PATH. -# Corepack is gone from Node 25+, so nothing here may use it. -ARG PNPM_VERSION=12.5.1 -RUN set -eux; \ - ARCH="$(dpkg --print-architecture)"; \ - case "${ARCH}" in \ - aarch64|arm64) \ - PNPM_ARCH='arm64'; \ - PNPM_SHA256='84e1290e82c800acd406b6db27e9650e15db3c2344d12162cc305ea1f942c6ff'; \ - ;; \ - amd64|x86_64) \ - PNPM_ARCH='x64'; \ - PNPM_SHA256='5a397dfb6b3d4b07d3d7769586aeb471048faf04299a492e2808b95a9a1c701f'; \ - ;; \ - *) \ - echo "Unsupported arch: ${ARCH}"; \ - exit 1; \ - ;; \ - esac; \ - curl -LfsSo /tmp/pnpm.tar.gz "https://github.com/pnpm/pnpm/releases/download/v${PNPM_VERSION}/pnpm-linux-${PNPM_ARCH}.tar.gz"; \ - echo "${PNPM_SHA256} */tmp/pnpm.tar.gz" | sha256sum -c -; \ - tar -xzf /tmp/pnpm.tar.gz -C /usr/local/bin pnpm; \ - chmod 755 /usr/local/bin/pnpm; \ - rm -f /tmp/pnpm.tar.gz; \ - pnpm --version; - -# `./setup` only needs pnpm on PATH (installed above as root). -# Ownership is fixed right after. -RUN ./setup && chown -R penpot:penpot /opt/penpot/exporter - -USER penpot:penpot - -CMD ["node", "app.js"] + mkdir -p /usr/local/bin; \ diff --git a/docker/images/Dockerfile.mcp b/docker/images/Dockerfile.mcp index 043fa5493c..87b339818a 100644 --- a/docker/images/Dockerfile.mcp +++ b/docker/images/Dockerfile.mcp @@ -33,6 +33,7 @@ RUN set -eux; \ esac; \ curl -LfsSo /tmp/pnpm.tar.gz "https://github.com/pnpm/pnpm/releases/download/v${PNPM_VERSION}/pnpm-linux-${PNPM_ARCH}.tar.gz"; \ echo "${PNPM_SHA256} */tmp/pnpm.tar.gz" | sha256sum -c -; \ + mkdir -p /usr/local/bin; \ tar -xzf /tmp/pnpm.tar.gz -C /usr/local/bin pnpm; \ chmod 755 /usr/local/bin/pnpm; \ rm -f /tmp/pnpm.tar.gz; \ From 5e99e89e5b928b2864d4973d16cc30a2f69666cb Mon Sep 17 00:00:00 2001 From: bameda Date: Tue, 22 Sep 2026 14:08:14 +0000 Subject: [PATCH 03/22] :bug: Pin build-docker to the commit build-bundle actually bundled MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _DEVELOP and _STAGING chain build-bundle.yml -> build-docker.yml, both invoked with gh_ref as a branch name. Each reusable workflow re-resolves that branch with its own checkout, at its own time. build-docker's checkout happens minutes after build-bundle's, after new commits can have landed on the branch, so it can compute a different sha than the one build-bundle actually bundled and uploaded to S3. The Prepare job then requests a key that was never uploaded: fatal error: An error occurred (404) when calling the HeadObject operation: Key "penpot-sha-.zip" does not exist Expose build-bundle.yml's resolved sha as a workflow_call output and thread it through _DEVELOP, _STAGING and _ADHOC as build-docker.yml's new optional sha input, which pins its checkout instead of re-resolving gh_ref. gh_ref keeps naming the branch tag to move. _TAG is unaffected: it builds from a pushed git tag, which cannot move the way a branch can, so build-bundle and build-docker already agree on one commit there. Signed-off-by: David Barragán Merino --- .github/workflows/build-adhoc.yml | 5 +++++ .github/workflows/build-bundle.yml | 9 +++++++++ .github/workflows/build-develop.yml | 5 +++++ .github/workflows/build-docker.yml | 15 +++++++++++++-- .github/workflows/build-staging.yml | 5 +++++ 5 files changed, 37 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-adhoc.yml b/.github/workflows/build-adhoc.yml index 327c449fc4..a85065ad74 100644 --- a/.github/workflows/build-adhoc.yml +++ b/.github/workflows/build-adhoc.yml @@ -34,6 +34,11 @@ jobs: secrets: inherit with: gh_ref: ${{ inputs.gh_ref }} + # Pin the exact commit build-bundle already resolved and uploaded to + # S3, instead of letting build-docker re-resolve gh_ref on its own + # checkout minutes later, which can land on a newer commit than the + # one actually bundled if gh_ref names a branch that moved meanwhile. + sha: ${{ needs.build-bundle.outputs.sha }} force: ${{ inputs.force }} build-docker-admin-console: diff --git a/.github/workflows/build-bundle.yml b/.github/workflows/build-bundle.yml index 5a956cca28..64683a902c 100644 --- a/.github/workflows/build-bundle.yml +++ b/.github/workflows/build-bundle.yml @@ -26,6 +26,15 @@ on: type: boolean required: false default: false + outputs: + sha: + description: >- + The exact commit this run resolved gh_ref to and bundled. Callers + that also trigger build-docker.yml should pass it through as that + workflow's `sha` input, so both pipelines agree on one commit + instead of each re-resolving gh_ref independently at a different + time. + value: ${{ jobs.check.outputs.sha }} # Literal group name: under `workflow_call`, `github.workflow` resolves to the # caller's workflow, which put this workflow and the other reusable one called diff --git a/.github/workflows/build-develop.yml b/.github/workflows/build-develop.yml index 7346450831..d947cb1741 100644 --- a/.github/workflows/build-develop.yml +++ b/.github/workflows/build-develop.yml @@ -32,6 +32,11 @@ jobs: secrets: inherit with: gh_ref: "develop" + # Pin the exact commit build-bundle already resolved and uploaded to + # S3, instead of letting build-docker re-resolve "develop" on its own + # checkout minutes later, which can land on a newer commit than the + # one actually bundled. + sha: ${{ needs.build-bundle.outputs.sha }} force: ${{ inputs.force || false }} build-docker-admin-console: diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index a97b163439..c55d437c5f 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -20,6 +20,17 @@ on: type: string required: true default: 'develop' + sha: + description: >- + Exact commit to check out and build, e.g. from build-bundle.yml's + `sha` output. gh_ref is still used to resolve the checkout when + this is empty, and always names the branch tag to move — passing + both avoids the checkout re-resolving gh_ref on its own, possibly + to a newer commit than the one build-bundle.yml already bundled + and uploaded to S3 under its own resolved sha. + type: string + required: false + default: '' force: description: 'Rebuild and overwrite even if this sha is already promoted' type: boolean @@ -60,7 +71,7 @@ jobs: uses: actions/checkout@v6 with: fetch-depth: 0 - ref: ${{ inputs.gh_ref }} + ref: ${{ inputs.sha != '' && inputs.sha || inputs.gh_ref }} - name: Extract some useful variables id: vars @@ -148,7 +159,7 @@ jobs: - name: Checkout code uses: actions/checkout@v6 with: - ref: ${{ inputs.gh_ref }} + ref: ${{ inputs.sha != '' && inputs.sha || inputs.gh_ref }} - name: Login to Docker Registry uses: docker/login-action@v4 diff --git a/.github/workflows/build-staging.yml b/.github/workflows/build-staging.yml index c2e3dad823..a14f4e3714 100644 --- a/.github/workflows/build-staging.yml +++ b/.github/workflows/build-staging.yml @@ -32,6 +32,11 @@ jobs: secrets: inherit with: gh_ref: "staging" + # Pin the exact commit build-bundle already resolved and uploaded to + # S3, instead of letting build-docker re-resolve "staging" on its own + # checkout minutes later, which can land on a newer commit than the + # one actually bundled. + sha: ${{ needs.build-bundle.outputs.sha }} force: ${{ inputs.force || false }} build-docker-admin-console: From cc0941343d35ca6a0df47e0ac66b6fa5661b0df4 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Tue, 22 Sep 2026 15:49:46 +0000 Subject: [PATCH 04/22] :books: Add backend audit-log memory documentation Add a new memory file documenting the backend audit log system: purpose, storage schema, RPC producers, frontend ingestion, webhooks/error-reporter/telemetry consumers, and Nexus archival. Also wire a reference to it from the backend core memory so it is discoverable through the memory graph. AI-assisted-by: longcat-2.0 --- .serena/memories/backend/audit-log.md | 51 +++++++++++++++++++++++++++ .serena/memories/backend/core.md | 1 + 2 files changed, 52 insertions(+) create mode 100644 .serena/memories/backend/audit-log.md diff --git a/.serena/memories/backend/audit-log.md b/.serena/memories/backend/audit-log.md new file mode 100644 index 0000000000..fe401cef0c --- /dev/null +++ b/.serena/memories/backend/audit-log.md @@ -0,0 +1,51 @@ +# Backend Audit Log + +Penpot records what users do as events in the Postgres `audit_log` table. There are two producers (the backend RPC layer and the frontend app) and four consumers (webhooks, error reporters, telemetry shipping, and the Nexus archive). Everything below follows that flow: purpose, storage, producers, consumers, archival. + +## Purpose + +- The audit log answers "who did what, when, from where": every RPC mutation and selected frontend actions become a row with `name`, `type`, `profile-id`, `ip-addr`, `props` and `context`. Product analytics, abuse investigation and compliance exports all read from here, so keep events truthful and never put secrets in `props`. +- It is also the trigger bus for side effects: the same event object fans out to webhooks, error reporting and telemetry without the RPC handler knowing. New features should reuse this bus instead of building parallel notification paths. + +## Storage + +- Live `audit_log` columns: `id` uuid PK default `gen_random_uuid()`; `name`/`type` text NOT NULL; `created_at` timestamptz NOT NULL default `now()` (server time, the source of truth); `tracked_at` timestamptz default `now()` (client-claimed time, corrected on ingest); `profile_id` uuid NOT NULL; `source` text telling full rows (`backend`/`frontend`) apart from anonymized copies (`telemetry:backend`/`telemetry:frontend`); `ip_addr` inet; `props`/`context` jsonb holding transit-encoded maps; `archived_at` timestamptz set once Nexus acknowledges the row. +- Indexes: PK on `(id)`; partial `created_at WHERE archived_at IS NULL` serving the archive scan; partial `archived_at WHERE archived_at IS NOT NULL` serving the GC; `(source, created_at)` serving the telemetry scan. Each consumer has its own index, so a slow consumer never blocks the others. + +## Backend producers (`app.loggers.audit`) + +- Most backend events need no manual code: `wrap-audit` in `app.rpc` runs after every RPC handler when `:webhooks`, `:audit-log` or `:telemetry` is on (unless the command sets `::audit/skip`) and builds the event via `prepare-rpc-event`. The event name defaults to the command name (prefixed with `-` outside `main`), props default to the request params, and timestamps come from the server request time. +- Commands customize through result metadata (`rph/with-meta`): `::audit/replace-props` swaps the props wholesale (auth commands use `profile->props` so a register event carries the profile, not the password), `::audit/props` merges extras, `::audit/context`/`profile-id`/`name`/`type` override the defaults. `clean-props` always strips nils, qualified keys and `:session-id/:password/:old-password/:token/:client-secret` as a last line of defense. +- `submit` is the normal entry point (fills defaults, validates `schema:event`, runs inside `tx-run!`, logs failures without failing the RPC). `insert` is the low-level one for CLI/helpers and the webhook subsystem: direct write, no webhook/telemetry fan-out, silent unless `:audit-log` is on. Boot emits `trigger/instance-start` from `setup/props` so every restart is visible in the log. + +## Consumers I: webhooks (`app.loggers.webhooks`) + +- Webhooks are the first dependent: when an event carries `::webhooks/event?`, `process-event` (worker task `:process-webhook-event`) finds the team's active webhooks from the event props (`team-id`, else `project-id`, else `file-id`), records a `trigger webhook` row, and enqueues one `:run-webhook` delivery per match. Batching and dedupe come from the audit event itself (`batch-key` + `batch-timeout`), not from webhook config. +- `:run-webhook` POSTs the event in the webhook's `mtype` (JSON camelCase, transit, or form-encoded), logs each attempt in `webhook-delivery`, and disables the webhook after 3 consecutive errors. Delivery problems never touch the audit row itself. + +## Consumers II: error reporters (`app.loggers.database`, `app.loggers.mattermost`) + +- Both reporters listen for backend `:error` log records and for frontend crash events (recognized by the `::audit/event` marker), through a sliding-buffer channel so a flood of errors cannot stall the app. The database reporter persists them into `server-error-report` (source 4 = audit-log origin); the Mattermost reporter forwards a short notification to `:error-report-webhook` when configured. +- Consequence for producers: crash reports only exist if the frontend collector is running and `push-audit-events` accepts `unhandled-exception`/`exception-page` events. Disabling the whole pipeline also blinds error reporting from the frontend. + +## Consumers III: telemetry (`app.loggers.audit` + `app.tasks.telemetry`) + +- Telemetry reuses the same table with anonymized shadow rows (`source LIKE 'telemetry:%'`): day-truncated timestamps, `0.0.0.0` IPs, props reduced to uuid/boolean/number values plus a few allowlisted fields (`lang`, `auth-backend`, derived `email-domain`, never raw emails), and a minimal context allowlist. Both full and shadow rows can coexist per event; that duplication is intentional. +- The telemetry cron ships shadow rows to `:telemetry-uri` as JSON in 10k batches, deletes them on success, and purges leftovers older than 7d. Nothing is collected or sent on official hosts (`telemetry-excluded?` covers `penpot.app`/`penpot.dev`). + +## Frontend ingestion (`app.rpc.commands.audit`, `app.main.data.event`) + +- The browser cannot write to the table directly; it POSTs transit batches to `push-audit-events`, which stamps server `id`, session `profile-id`, request ip and server `created-at`, and distrusts the client clock (future or >1h-lagging `tracked-at` is reset, original preserved in context). The endpoint is a no-op without `:audit-log`/`:telemetry` or on a read-only pool. +- The in-browser collector (`app.main.data.event`) only starts after `get-enabled-flags` confirms the backend wants events. It turns Potok events and explicit `ev/event` calls (nitrate membership changes, workspace file stats, crash reports) into a capped buffer (1024, chunks of 100, 2s debounce, current profile only) and sends fire-and-forget. `skip-audit?` exists for resumed dashboard actions so one user gesture is not counted twice. +- Because collection is best-effort and includes `PerformanceObserver` noise (`performance-*` triggers), backend tests must never assert exact frontend event counts. + +## Archival to Nexus and retention + +- Long-term storage lives outside Penpot in Nexus. Every 5m the `:audit-log-archive` cron takes chunks of 128 unarchived rows (`FOR UPDATE SKIP LOCKED`), POSTs them as transit `{:events [...]}` authenticated with `x-shared-key: "nexus "` (`:nexus-shared-key`, else derived from the instance secret), and marks `archived_at=now()` only on HTTP 204, in the same transaction. Anything else is retried on the next run; a missing URI with the flag on raises `:task-not-configured`. +- Every 5m the `:audit-log-gc` cron deletes all archived rows (no age filter), so archive must run before GC or data ships never. Cron dedup is best-effort (`mem:prod-infra/core`): two backends can fire the archiver twice, which is why the Nexus endpoint must be idempotent and the DB only marks acknowledged rows. +- Flags live in `common/flags.cljc` varia and are enabled as `PENPOT_FLAGS=enable-`: `:audit-log`, `:audit-log-archive`, `:audit-log-gc`, `:audit-log-logger` (structured `app.audit` log). `:telemetry-enabled` config auto-adds `:enable-telemetry`. + +## Tests + +- `backend_tests/rpc-audit-test.clj` exercises the whole backend path (full-row insert, telemetry-only and dual-row modes, `submit*`, no-op without flags, `insert` gating, `prepare-rpc-event` resolution) with `with-redefs [cf/flags #{...}]` against real `audit_log` rows. +- Other RPC suites mock `app.loggers.audit/submit` (nil return; `helpers.clj` stubs it globally) and assert on `:call-args-list`; any new command that must (or must not) emit an event needs the same treatment. diff --git a/.serena/memories/backend/core.md b/.serena/memories/backend/core.md index 9a567932b7..816a322a3b 100644 --- a/.serena/memories/backend/core.md +++ b/.serena/memories/backend/core.md @@ -8,6 +8,7 @@ Backend: JVM Clojure; Integrant; PostgreSQL; Redis/Valkey; RPC; HTTP; storage; m - Storage abstraction, logical buckets, object lifecycle, deduplication, access, and garbage collection: `mem:backend/storage`. - HTTP sessions, config, media processing, and file data persistence: `mem:backend/http-storage-filedata-subtleties`. - Auth flows, permission model, teams, projects, invitations, comments, webhooks, audit: `mem:backend/auth-permissions-product-domains` +- Audit-log event collection (RPC wrapper, frontend ingestion), telemetry duality, webhook fan-out, error reporters, Nexus archival and retention: `mem:backend/audit-log` - Services, task-queue/Pub-Sub topology constraints -> `mem:prod-infra/core`. ## Stable namespace map From 5901af41871411cb0373de617b2103046d50542b Mon Sep 17 00:00:00 2001 From: bameda Date: Tue, 22 Sep 2026 15:31:46 +0000 Subject: [PATCH 05/22] :bug: Restore Dockerfile.exporter wiped by a bad conflict resolution MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: David Barragán Merino --- docker/images/Dockerfile.exporter | 127 ++++++++++++++++++++++++++++++ 1 file changed, 127 insertions(+) diff --git a/docker/images/Dockerfile.exporter b/docker/images/Dockerfile.exporter index e6ce64ca4a..a186917a97 100644 --- a/docker/images/Dockerfile.exporter +++ b/docker/images/Dockerfile.exporter @@ -1 +1,128 @@ +FROM dhi.io/node:24.20.0-debian13-dev +LABEL maintainer="Penpot " + +ENV LANG=en_US.UTF-8 \ + LC_ALL=en_US.UTF-8 \ + DEBIAN_FRONTEND=noninteractive \ + PATH=/opt/imagick/bin:$PATH \ + PLAYWRIGHT_BROWSERS_PATH=/opt/penpot/browsers + +RUN set -ex; \ + apt-get -qq update; \ + apt-get -qqy --no-install-recommends install passwd; \ + useradd -U -M -u 1001 -s /bin/false -d /opt/penpot penpot; \ + apt-get -qq -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" dist-upgrade; \ + apt-get -qqy --no-install-recommends install \ + curl \ + tzdata \ + locales \ + ca-certificates \ + ; \ + apt-get clean; \ + rm -rf /var/lib/apt/lists/*; \ + echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen; \ + locale-gen; \ + find /usr/share/i18n/locales/ -type f ! -name "en_US" ! -name "POSIX" ! -name "C" -delete; + +RUN set -ex; \ + apt-get -qq update; \ + apt-get -qqy install \ + \ + xvfb \ + fonts-noto-color-emoji \ + fonts-unifont \ + libfontconfig1 \ + libfreetype6 \ + xfonts-cyrillic \ + xfonts-scalable \ + fonts-liberation \ + fonts-ipafont-gothic \ + fonts-wqy-zenhei \ + fonts-tlwg-loma-otf \ + fonts-freefont-ttf \ + poppler-utils \ + \ + libasound2 \ + libatk-bridge2.0-0 \ + libatk1.0-0 \ + libatspi2.0-0 \ + libcairo2 \ + libcups2 \ + libdbus-1-3 \ + libdrm2 \ + libgbm1 \ + libglib2.0-0 \ + libnspr4 \ + libnss3 \ + libpango-1.0-0 \ + libx11-6 \ + libxcb1 \ + libxcomposite1 \ + libxdamage1 \ + libxext6 \ + libxfixes3 \ + libxkbcommon0 \ + libxrandr2 \ + \ + libgomp1 \ + libheif1 \ + libjpeg62-turbo \ + liblcms2-2 \ + libopenexr-3-1-30 \ + libopenjp2-7 \ + libpng16-16 \ + librsvg2-2 \ + libtiff6 \ + libwebp7 \ + libwebpdemux2 \ + libwebpmux3 \ + libxml2 \ + libzip5 \ + libzstd1 \ + ; \ + apt-get clean; \ + rm -rf /var/lib/apt/lists/*; \ + mkdir -p /opt/penpot; \ + chown -R penpot:penpot /opt/penpot; + +ARG BUNDLE_PATH="./bundle-exporter/" +COPY --chown=penpot:penpot $BUNDLE_PATH /opt/penpot/exporter/ +COPY --from=penpotapp/imagemagick:7.1.2-27 /opt/imagick /opt/imagick + +WORKDIR /opt/penpot/exporter + +# pnpm ships as a system binary (same tarball + SHA pin as +# docker/devenv/Dockerfile); the generated ./setup expects it on PATH. +# Corepack is gone from Node 25+, so nothing here may use it. +ARG PNPM_VERSION=12.5.1 +RUN set -eux; \ + ARCH="$(dpkg --print-architecture)"; \ + case "${ARCH}" in \ + aarch64|arm64) \ + PNPM_ARCH='arm64'; \ + PNPM_SHA256='84e1290e82c800acd406b6db27e9650e15db3c2344d12162cc305ea1f942c6ff'; \ + ;; \ + amd64|x86_64) \ + PNPM_ARCH='x64'; \ + PNPM_SHA256='5a397dfb6b3d4b07d3d7769586aeb471048faf04299a492e2808b95a9a1c701f'; \ + ;; \ + *) \ + echo "Unsupported arch: ${ARCH}"; \ + exit 1; \ + ;; \ + esac; \ + curl -LfsSo /tmp/pnpm.tar.gz "https://github.com/pnpm/pnpm/releases/download/v${PNPM_VERSION}/pnpm-linux-${PNPM_ARCH}.tar.gz"; \ + echo "${PNPM_SHA256} */tmp/pnpm.tar.gz" | sha256sum -c -; \ mkdir -p /usr/local/bin; \ + tar -xzf /tmp/pnpm.tar.gz -C /usr/local/bin pnpm; \ + chmod 755 /usr/local/bin/pnpm; \ + rm -f /tmp/pnpm.tar.gz; \ + pnpm --version; + +# `./setup` only needs pnpm on PATH (installed above as root). +# Ownership is fixed right after. +RUN ./setup && chown -R penpot:penpot /opt/penpot/exporter + +USER penpot:penpot + +CMD ["node", "app.js"] From 0622680691b698bd95ec6d715695f0542e94c7ee Mon Sep 17 00:00:00 2001 From: bameda Date: Tue, 22 Sep 2026 15:01:06 +0000 Subject: [PATCH 06/22] :recycle: Make sha mean the full commit SHA, short_sha the 12-char one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit sha is the full 40-char SHA in build-bundle.yml's output and build-docker.yml's new checkout-pinning input, matching git and GitHub's own convention. short_sha stays internal to each workflow for the S3 keys and image tags that already used it — build-bundle.yml no longer exposes it externally since nothing outside consumed it. No S3 key, image tag, or marker value changes anywhere. Signed-off-by: David Barragán Merino --- .github/workflows/build-adhoc.yml | 5 +--- .github/workflows/build-bundle.yml | 19 +++++------- .github/workflows/build-develop.yml | 5 +--- .github/workflows/build-docker.yml | 45 ++++++++++++++--------------- .github/workflows/build-staging.yml | 5 +--- 5 files changed, 33 insertions(+), 46 deletions(-) diff --git a/.github/workflows/build-adhoc.yml b/.github/workflows/build-adhoc.yml index a85065ad74..abfe01d5a1 100644 --- a/.github/workflows/build-adhoc.yml +++ b/.github/workflows/build-adhoc.yml @@ -34,10 +34,7 @@ jobs: secrets: inherit with: gh_ref: ${{ inputs.gh_ref }} - # Pin the exact commit build-bundle already resolved and uploaded to - # S3, instead of letting build-docker re-resolve gh_ref on its own - # checkout minutes later, which can land on a newer commit than the - # one actually bundled if gh_ref names a branch that moved meanwhile. + # Pin build-docker to the commit build-bundle actually bundled. sha: ${{ needs.build-bundle.outputs.sha }} force: ${{ inputs.force }} diff --git a/.github/workflows/build-bundle.yml b/.github/workflows/build-bundle.yml index 64683a902c..e7f1f3a815 100644 --- a/.github/workflows/build-bundle.yml +++ b/.github/workflows/build-bundle.yml @@ -28,12 +28,7 @@ on: default: false outputs: sha: - description: >- - The exact commit this run resolved gh_ref to and bundled. Callers - that also trigger build-docker.yml should pass it through as that - workflow's `sha` input, so both pipelines agree on one commit - instead of each re-resolving gh_ref independently at a different - time. + description: 'Bundled commit, full 40-char SHA.' value: ${{ jobs.check.outputs.sha }} # Literal group name: under `workflow_call`, `github.workflow` resolves to the @@ -53,6 +48,7 @@ jobs: outputs: gh_ref: ${{ steps.vars.outputs.gh_ref }} bundle_version: ${{ steps.vars.outputs.bundle_version }} + short_sha: ${{ steps.vars.outputs.short_sha }} sha: ${{ steps.vars.outputs.sha }} commit_title: ${{ steps.vars.outputs.commit_title }} exists: ${{ steps.check.outputs.exists }} @@ -69,7 +65,8 @@ jobs: run: | echo "gh_ref=${{ inputs.gh_ref || github.ref_name }}" >> $GITHUB_OUTPUT echo "bundle_version=$(git describe --tags --always)" >> $GITHUB_OUTPUT - echo "sha=$(git rev-parse --short=12 HEAD)" >> $GITHUB_OUTPUT + echo "short_sha=$(git rev-parse --short=12 HEAD)" >> $GITHUB_OUTPUT + echo "sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT echo "commit_title=$(git log -1 --pretty=%s)" >> $GITHUB_OUTPUT # Keyed by commit sha, same as build-docker.yml's marker check, so @@ -95,7 +92,7 @@ jobs: BUNDLE_EXISTS=$(aws s3api head-object \ --bucket ${{ secrets.S3_BUCKET }} \ - --key "penpot-sha-${{ steps.vars.outputs.sha }}.zip" \ + --key "penpot-sha-${{ steps.vars.outputs.short_sha }}.zip" \ > /dev/null 2>&1 && echo "true" || echo "false") if [ "$BUNDLE_EXISTS" = "true" ]; then @@ -103,7 +100,7 @@ jobs: { echo "### ⏭️ Bundle build skipped" echo "" - echo "The bundle in S3 was already built from \`sha-${{ steps.vars.outputs.sha }}\` (\`${{ steps.vars.outputs.bundle_version }}\`)." + echo "The bundle in S3 was already built from \`sha-${{ steps.vars.outputs.short_sha }}\` (\`${{ steps.vars.outputs.bundle_version }}\`)." } >> "$GITHUB_STEP_SUMMARY" else echo "exists=false" >> $GITHUB_OUTPUT @@ -147,7 +144,7 @@ jobs: AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }} run: | aws s3 cp zips/penpot.zip \ - s3://${{ secrets.S3_BUCKET }}/penpot-sha-${{ needs.check.outputs.sha }}.zip \ + s3://${{ secrets.S3_BUCKET }}/penpot-sha-${{ needs.check.outputs.short_sha }}.zip \ --metadata bundle-version=${{ needs.check.outputs.bundle_version }} - name: Write step summary @@ -156,7 +153,7 @@ jobs: echo "### ✅ Bundle built" echo "" echo "- Version: \`${{ needs.check.outputs.bundle_version }}\` (\`git describe --tags --always\`)" - echo "- Commit: [\`${{ needs.check.outputs.sha }}\`](https://github.com/${{ github.repository }}/commit/${{ needs.check.outputs.sha }}) — ${{ needs.check.outputs.commit_title }}" + echo "- Commit: [\`${{ needs.check.outputs.short_sha }}\`](https://github.com/${{ github.repository }}/commit/${{ needs.check.outputs.short_sha }}) — ${{ needs.check.outputs.commit_title }}" echo "- Built at: $(date -u +'%Y-%m-%d %H:%M:%S UTC')" } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/build-develop.yml b/.github/workflows/build-develop.yml index d947cb1741..8f12163965 100644 --- a/.github/workflows/build-develop.yml +++ b/.github/workflows/build-develop.yml @@ -32,10 +32,7 @@ jobs: secrets: inherit with: gh_ref: "develop" - # Pin the exact commit build-bundle already resolved and uploaded to - # S3, instead of letting build-docker re-resolve "develop" on its own - # checkout minutes later, which can land on a newer commit than the - # one actually bundled. + # Pin build-docker to the commit build-bundle actually bundled. sha: ${{ needs.build-bundle.outputs.sha }} force: ${{ inputs.force || false }} diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index c55d437c5f..d24b09436e 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -22,12 +22,9 @@ on: default: 'develop' sha: description: >- - Exact commit to check out and build, e.g. from build-bundle.yml's - `sha` output. gh_ref is still used to resolve the checkout when - this is empty, and always names the branch tag to move — passing - both avoids the checkout re-resolving gh_ref on its own, possibly - to a newer commit than the one build-bundle.yml already bundled - and uploaded to S3 under its own resolved sha. + Exact commit to check out (full 40-char SHA, e.g. from + build-bundle.yml). Falls back to gh_ref when empty; gh_ref + always still names the branch tag to move. type: string required: false default: '' @@ -63,6 +60,7 @@ jobs: gh_ref: ${{ steps.vars.outputs.gh_ref }} bundle_version: ${{ steps.vars.outputs.bundle_version }} sha: ${{ steps.vars.outputs.sha }} + short_sha: ${{ steps.vars.outputs.short_sha }} commit_title: ${{ steps.vars.outputs.commit_title }} exists: ${{ steps.check.outputs.exists }} @@ -78,7 +76,8 @@ jobs: run: | GH_REF="${{ inputs.gh_ref || github.ref_name }}" echo "gh_ref=$GH_REF" >> $GITHUB_OUTPUT - echo "sha=$(git rev-parse --short=12 HEAD)" >> $GITHUB_OUTPUT + echo "sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT + echo "short_sha=$(git rev-parse --short=12 HEAD)" >> $GITHUB_OUTPUT echo "commit_title=$(git log -1 --pretty=%s)" >> $GITHUB_OUTPUT echo "bundle_version=$(git describe --tags --always)" >> $GITHUB_OUTPUT @@ -98,8 +97,8 @@ jobs: echo "exists=false" >> $GITHUB_OUTPUT mkdir -p "$BUNDLE_CACHE" find "$BUNDLE_CACHE" -type f -mtime +1 -delete || true - ZIP="$BUNDLE_CACHE/penpot-sha-${{ steps.vars.outputs.sha }}.zip" - aws s3 cp "s3://${{ secrets.S3_BUCKET }}/penpot-sha-${{ steps.vars.outputs.sha }}.zip" "$ZIP.$$.tmp" + ZIP="$BUNDLE_CACHE/penpot-sha-${{ steps.vars.outputs.short_sha }}.zip" + aws s3 cp "s3://${{ secrets.S3_BUCKET }}/penpot-sha-${{ steps.vars.outputs.short_sha }}.zip" "$ZIP.$$.tmp" mv "$ZIP.$$.tmp" "$ZIP" { echo "### 🔁 Image set build forced" @@ -111,13 +110,13 @@ jobs: if aws s3api head-object \ --bucket ${{ secrets.S3_BUCKET }} \ - --key "markers/images-sha-${{ steps.vars.outputs.sha }}" \ + --key "markers/images-sha-${{ steps.vars.outputs.short_sha }}" \ > /dev/null 2>&1; then echo "exists=true" >> $GITHUB_OUTPUT { echo "### ⏭️ Image set build skipped" echo "" - echo "The whole set was already built and promoted for \`sha-${{ steps.vars.outputs.sha }}\`." + echo "The whole set was already built and promoted for \`sha-${{ steps.vars.outputs.short_sha }}\`." } >> "$GITHUB_STEP_SUMMARY" else echo "exists=false" >> $GITHUB_OUTPUT @@ -127,9 +126,9 @@ jobs: # prune stale bundles while at it. mkdir -p "$BUNDLE_CACHE" find "$BUNDLE_CACHE" -type f -mtime +1 -delete || true - ZIP="$BUNDLE_CACHE/penpot-sha-${{ steps.vars.outputs.sha }}.zip" + ZIP="$BUNDLE_CACHE/penpot-sha-${{ steps.vars.outputs.short_sha }}.zip" if [ ! -f "$ZIP" ]; then - aws s3 cp "s3://${{ secrets.S3_BUCKET }}/penpot-sha-${{ steps.vars.outputs.sha }}.zip" "$ZIP.$$.tmp" + aws s3 cp "s3://${{ secrets.S3_BUCKET }}/penpot-sha-${{ steps.vars.outputs.short_sha }}.zip" "$ZIP.$$.tmp" mv "$ZIP.$$.tmp" "$ZIP" fi fi @@ -199,11 +198,11 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }} run: | - ZIP="$BUNDLE_CACHE/penpot-sha-${{ needs.prepare.outputs.sha }}.zip" + ZIP="$BUNDLE_CACHE/penpot-sha-${{ needs.prepare.outputs.short_sha }}.zip" if [ ! -f "$ZIP" ]; then echo "Bundle not found in host cache; falling back to S3." mkdir -p "$BUNDLE_CACHE" - aws s3 cp "s3://${{ secrets.S3_BUCKET }}/penpot-sha-${{ needs.prepare.outputs.sha }}.zip" "$ZIP.$$.tmp" + aws s3 cp "s3://${{ secrets.S3_BUCKET }}/penpot-sha-${{ needs.prepare.outputs.short_sha }}.zip" "$ZIP.$$.tmp" mv "$ZIP.$$.tmp" "$ZIP" fi # Extract only the bundle this job needs. @@ -239,7 +238,7 @@ jobs: sbom: true # Immutable tag only; branch tags are moved atomically for the # whole image set by the `promote` job. - tags: ${{ secrets.DOCKER_REGISTRY }}/${{ matrix.image }}:sha-${{ needs.prepare.outputs.sha }} + tags: ${{ secrets.DOCKER_REGISTRY }}/${{ matrix.image }}:sha-${{ needs.prepare.outputs.short_sha }} labels: ${{ steps.meta.outputs.labels }} cache-from: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ matrix.image }}:buildcache cache-to: type=registry,ref=${{ secrets.DOCKER_REGISTRY }}/${{ matrix.image }}:buildcache,mode=max @@ -272,7 +271,7 @@ jobs: for image in $ALL_IMAGES; do docker buildx imagetools create \ -t "${{ secrets.DOCKER_REGISTRY }}/$image:${{ needs.prepare.outputs.gh_ref }}" \ - "${{ secrets.DOCKER_REGISTRY }}/$image:sha-${{ needs.prepare.outputs.sha }}" + "${{ secrets.DOCKER_REGISTRY }}/$image:sha-${{ needs.prepare.outputs.short_sha }}" done # The marker is written LAST: its presence certifies that all five @@ -284,7 +283,7 @@ jobs: AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }} run: | echo "${{ github.run_id }}" | aws s3 cp - \ - "s3://${{ secrets.S3_BUCKET }}/markers/images-sha-${{ needs.prepare.outputs.sha }}" + "s3://${{ secrets.S3_BUCKET }}/markers/images-sha-${{ needs.prepare.outputs.short_sha }}" - name: Write step summary run: | @@ -292,10 +291,10 @@ jobs: echo "### ✅ Image set promoted" echo "" echo "- Version: \`${{ needs.prepare.outputs.bundle_version }}\` (\`git describe --tags --always\`)" - echo "- Commit: [\`${{ needs.prepare.outputs.sha }}\`](https://github.com/${{ github.repository }}/commit/${{ needs.prepare.outputs.sha }}) — ${{ needs.prepare.outputs.commit_title }}" + echo "- Commit: [\`${{ needs.prepare.outputs.short_sha }}\`](https://github.com/${{ github.repository }}/commit/${{ needs.prepare.outputs.short_sha }}) — ${{ needs.prepare.outputs.commit_title }}" echo "- Built at: $(date -u +'%Y-%m-%d %H:%M:%S UTC')" echo "" - echo "All \`:${{ needs.prepare.outputs.gh_ref }}\` tags now point to \`sha-${{ needs.prepare.outputs.sha }}\`." + echo "All \`:${{ needs.prepare.outputs.gh_ref }}\` tags now point to \`sha-${{ needs.prepare.outputs.short_sha }}\`." } >> "$GITHUB_STEP_SUMMARY" # ── 3b. Skip path: make sure THIS ref's tags point to the existing @@ -330,7 +329,7 @@ jobs: for image in $ALL_IMAGES; do docker buildx imagetools create \ -t "${{ secrets.DOCKER_REGISTRY }}/$image:${{ needs.prepare.outputs.gh_ref }}" \ - "${{ secrets.DOCKER_REGISTRY }}/$image:sha-${{ needs.prepare.outputs.sha }}" + "${{ secrets.DOCKER_REGISTRY }}/$image:sha-${{ needs.prepare.outputs.short_sha }}" done - name: Write step summary @@ -339,10 +338,10 @@ jobs: echo "### ✅ Image set already built (branch tags ensured)" echo "" echo "- Version: \`${{ needs.prepare.outputs.bundle_version }}\` (\`git describe --tags --always\`)" - echo "- Commit: [\`${{ needs.prepare.outputs.sha }}\`](https://github.com/${{ github.repository }}/commit/${{ needs.prepare.outputs.sha }}) — ${{ needs.prepare.outputs.commit_title }}" + echo "- Commit: [\`${{ needs.prepare.outputs.short_sha }}\`](https://github.com/${{ github.repository }}/commit/${{ needs.prepare.outputs.short_sha }}) — ${{ needs.prepare.outputs.commit_title }}" echo "- Checked at: $(date -u +'%Y-%m-%d %H:%M:%S UTC')" echo "" - echo "All \`:${{ needs.prepare.outputs.gh_ref }}\` tags now point to \`sha-${{ needs.prepare.outputs.sha }}\`." + echo "All \`:${{ needs.prepare.outputs.gh_ref }}\` tags now point to \`sha-${{ needs.prepare.outputs.short_sha }}\`." } >> "$GITHUB_STEP_SUMMARY" # ── 4. Single failure notification for the whole workflow ───────────── diff --git a/.github/workflows/build-staging.yml b/.github/workflows/build-staging.yml index a14f4e3714..249ba39e40 100644 --- a/.github/workflows/build-staging.yml +++ b/.github/workflows/build-staging.yml @@ -32,10 +32,7 @@ jobs: secrets: inherit with: gh_ref: "staging" - # Pin the exact commit build-bundle already resolved and uploaded to - # S3, instead of letting build-docker re-resolve "staging" on its own - # checkout minutes later, which can land on a newer commit than the - # one actually bundled. + # Pin build-docker to the commit build-bundle actually bundled. sha: ${{ needs.build-bundle.outputs.sha }} force: ${{ inputs.force || false }} From d8d345d8bdb3f90a83d322ccb93c47bdefa0c1d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Barrag=C3=A1n=20Merino?= Date: Tue, 22 Sep 2026 18:21:04 +0200 Subject: [PATCH 07/22] :bug: Install gzip for tar -xzf in mcp's pnpm install MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: David Barragán Merino --- docker/images/Dockerfile.mcp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/images/Dockerfile.mcp b/docker/images/Dockerfile.mcp index 87b339818a..30364d74f9 100644 --- a/docker/images/Dockerfile.mcp +++ b/docker/images/Dockerfile.mcp @@ -14,7 +14,7 @@ WORKDIR /opt/penpot/mcp ARG PNPM_VERSION=12.5.1 RUN set -eux; \ apt-get -qq update; \ - apt-get -qqy --no-install-recommends install curl ca-certificates; \ + apt-get -qqy --no-install-recommends install curl ca-certificates gzip; \ rm -rf /var/lib/apt/lists/*; \ ARCH="$(dpkg --print-architecture)"; \ case "${ARCH}" in \ From ec627997936cae123e620a0d98c69c8aff7bee2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Barrag=C3=A1n=20Merino?= Date: Tue, 22 Sep 2026 19:16:35 +0200 Subject: [PATCH 08/22] :arrow_up: Align mcp runtime image to the same Node version MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: David Barragán Merino --- docker/images/Dockerfile.mcp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/images/Dockerfile.mcp b/docker/images/Dockerfile.mcp index 30364d74f9..4d62675bec 100644 --- a/docker/images/Dockerfile.mcp +++ b/docker/images/Dockerfile.mcp @@ -42,7 +42,7 @@ RUN set -eux; \ RUN ./setup -FROM dhi.io/node:24.18.1-debian13 AS image +FROM dhi.io/node:24.20.0-debian13 AS image LABEL maintainer="Penpot " ENV LANG=en_US.UTF-8 \ From 70890bb900d4d5a18591dea3d54691c18dbc0655 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Tue, 22 Sep 2026 19:44:44 +0200 Subject: [PATCH 09/22] :bug: Store audit initiator as plain string for shared-key callers (#11629) Shared-key callers (exporter, admin-console) arrive as keywords on auth-key-id, so transit persisted them as ~:exporter while regular traffic stored app. Coerce with d/name at the single origin so every audit and telemetry copy carries a plain string. Adds regression tests for the origin and the push-audit-events path, including caller spoofing precedence. Closes #11628 AI-assisted-by: muse-spark-1.3-contributor --- backend/src/app/loggers/audit.clj | 2 +- backend/test/backend_tests/rpc_audit_test.clj | 15 ++++++++++++ .../rpc_management_nitrate_test.clj | 23 +++++++++++++++++++ 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/backend/src/app/loggers/audit.clj b/backend/src/app/loggers/audit.clj index 6ded7befaf..28d14c3daa 100644 --- a/backend/src/app/loggers/audit.clj +++ b/backend/src/app/loggers/audit.clj @@ -204,7 +204,7 @@ token-id (::actoken/id request) token-type (::actoken/type request)] {:external-session-id session-id - :initiator (or key-id "app") + :initiator (or (d/name key-id) "app") :access-token-id (some-> token-id str) :access-token-type (some-> token-type str) :client-event-origin client-event-origin diff --git a/backend/test/backend_tests/rpc_audit_test.clj b/backend/test/backend_tests/rpc_audit_test.clj index f4cb76f2a3..6148df5831 100644 --- a/backend/test/backend_tests/rpc_audit_test.clj +++ b/backend/test/backend_tests/rpc_audit_test.clj @@ -35,6 +35,21 @@ "x-forwarded-for" "127.0.0.44" "x-real-ip" "127.0.0.43")))) +(t/deftest prepare-context-initiator-is-plain-string + ;; The initiator must always be a plain string, never a keyword: shared-key + ;; authenticated callers (exporter, admin-console) arrive as keywords on + ;; :app.http/auth-key-id and transit would persist them as "~:exporter". + (let [base {:headers {"x-forwarded-for" "127.0.0.44"}}] + (t/is (= "app" (:initiator (audit/prepare-context-from-request base)))) + (t/is (= "exporter" + (:initiator (audit/prepare-context-from-request + (assoc base :app.http/auth-key-id :exporter))))) + (t/is (= "admin-console" + (:initiator (audit/prepare-context-from-request + (assoc base :app.http/auth-key-id :admin-console))))) + (t/is (string? (:initiator (audit/prepare-context-from-request + (assoc base :app.http/auth-key-id :nexus))))))) + (t/deftest push-events-1 (with-redefs [app.config/flags #{:audit-log}] (let [prof (th/create-profile* 1 {:is-active true}) diff --git a/backend/test/backend_tests/rpc_management_nitrate_test.clj b/backend/test/backend_tests/rpc_management_nitrate_test.clj index 1589026db1..3d1d884abe 100644 --- a/backend/test/backend_tests/rpc_management_nitrate_test.clj +++ b/backend/test/backend_tests/rpc_management_nitrate_test.clj @@ -1943,3 +1943,26 @@ (t/is (= "bar" (get-in event [:context :foo]))) (t/is (= (:full cf/version) (get-in event [:context :version]))) (t/is (= "app" (get-in event [:context :initiator])))))))) + +(t/deftest push-audit-events-initiator-is-plain-string + ;; Shared-key callers (e.g. admin-console) carry :app.http/auth-key-id as a + ;; keyword; the stored initiator must be a plain string, and a + ;; caller-supplied initiator must never survive (server context wins). + (with-mocks [audit-mock {:target 'app.loggers.audit/submit :return nil}] + (binding [cf/flags #{:audit-log}] + (let [prof (th/create-profile* 1 {:is-active true}) + params {::th/type :push-audit-events + :events [{:name "context-test" + :profile-id (:id prof) + :type "action" + :context {:custom-key "custom-val" + :initiator "spoofed"}}]} + params (with-meta params + {::http/request (assoc http-request + ::http/auth-key-id :admin-console)}) + out (th/management-command! params)] + (t/is (nil? (:error out))) + (let [[_ event] (:call-args @audit-mock)] + (t/is (= "custom-val" (get-in event [:context :custom-key]))) + (t/is (= "admin-console" (get-in event [:context :initiator]))) + (t/is (string? (get-in event [:context :initiator])))))))) From c26408a5683c2318848a5484141e7e1edf9c5105 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Barrag=C3=A1n=20Merino?= Date: Mon, 27 Jul 2026 22:44:04 +0200 Subject: [PATCH 10/22] :whale: Deploy the admin-console service in the Docker Compose example --- docker/images/docker-compose.yaml | 56 +++++++++++++++++++++---------- 1 file changed, 39 insertions(+), 17 deletions(-) diff --git a/docker/images/docker-compose.yaml b/docker/images/docker-compose.yaml index 45b2361510..9d5082b4b4 100644 --- a/docker/images/docker-compose.yaml +++ b/docker/images/docker-compose.yaml @@ -24,7 +24,7 @@ # WARNING: if you're exposing Penpot to the internet, you should remove the flags # 'disable-secure-session-cookies' and 'disable-email-verification' x-flags: &penpot-flags - PENPOT_FLAGS: disable-email-verification enable-smtp enable-prepl-server disable-secure-session-cookies enable-mcp + PENPOT_FLAGS: disable-email-verification enable-smtp enable-prepl-server disable-secure-session-cookies enable-mcp enable-admin-console x-uri: &penpot-public-uri PENPOT_PUBLIC_URI: http://localhost:9001 @@ -45,6 +45,13 @@ x-body-size: &penpot-http-body-size x-secret-key: &penpot-secret-key PENPOT_SECRET_KEY: change-this-insecure-key +## Single database configuration block, shared by backend and admin-console. +## Don't touch them unless you are using custom postgresql connection parameters. +x-database: &penpot-database + PENPOT_DATABASE_URI: postgresql://penpot-postgres/penpot + PENPOT_DATABASE_USERNAME: penpot + PENPOT_DATABASE_PASSWORD: penpot + networks: penpot: @@ -88,8 +95,9 @@ services: depends_on: - penpot-backend - - penpot-exporter + - penpot-admin-console - penpot-mcp + - penpot-exporter networks: - penpot @@ -107,6 +115,11 @@ services: environment: << : [*penpot-flags, *penpot-http-body-size, *penpot-public-uri] + + # Don't touch it; this uses an internal docker network to + # communicate with the admin-console. + PENPOT_ADMIN_CONSOLE_URI: http://penpot-admin-console:3000 + # Set to "true" on hosts where IPv6 is disabled at kernel boot level. # PENPOT_DISABLE_IPV6_LISTEN: "true" @@ -126,31 +139,23 @@ services: networks: - penpot - ## Configuration envronment variables for the backend container. - environment: - << : [*penpot-flags, *penpot-public-uri, *penpot-http-body-size, *penpot-secret-key] + << : [*penpot-flags, *penpot-public-uri, *penpot-http-body-size, *penpot-secret-key, *penpot-database] - ## Database connection parameters. Don't touch them unless you are using custom - ## postgresql connection parameters. - - PENPOT_DATABASE_URI: postgresql://penpot-postgres/penpot - PENPOT_DATABASE_USERNAME: penpot - PENPOT_DATABASE_PASSWORD: penpot + # Don't touch it; this uses an internal docker network to + # communicate with the admin-console. + PENPOT_ADMIN_CONSOLE_URI: http://penpot-admin-console:3000 ## Valkey (or previously redis) is used for the websockets notifications. Don't touch ## unless the valkey container has different parameters or different name. - PENPOT_REDIS_URI: redis://penpot-valkey/0 ## Default configuration for assets storage: using filesystem based with all files ## stored in a docker volume. - PENPOT_OBJECTS_STORAGE_BACKEND: fs PENPOT_OBJECTS_STORAGE_FS_DIRECTORY: /opt/data/assets ## Also can be configured to to use a S3 compatible storage. - # AWS_ACCESS_KEY_ID: # AWS_SECRET_ACCESS_KEY: # PENPOT_OBJECTS_STORAGE_BACKEND: s3 @@ -161,7 +166,6 @@ services: ## instance. Telemetry data will enable us to learn how the application is used, ## based on real scenarios. If you want to help us, please leave it enabled. You can ## audit what data we send with the code available on github. - PENPOT_TELEMETRY_ENABLED: "true" PENPOT_TELEMETRY_REFERER: compose @@ -169,7 +173,6 @@ services: ## service, but for production usage it is recommended to setup a real SMTP ## provider. Emails are used to confirm user registrations & invitations. Look below ## how the mailcatch service is configured. - PENPOT_SMTP_DEFAULT_FROM: no-reply@example.com PENPOT_SMTP_DEFAULT_REPLY_TO: no-reply@example.com PENPOT_SMTP_HOST: penpot-mailcatch @@ -179,6 +182,24 @@ services: PENPOT_SMTP_TLS: "false" PENPOT_SMTP_SSL: "false" + penpot-admin-console: + image: "penpotapp/admin-console:${PENPOT_VERSION:-2.16}" + restart: always + + depends_on: + penpot-postgres: + condition: service_healthy + + networks: + - penpot + + environment: + << : [*penpot-public-uri, *penpot-secret-key, *penpot-database] + + # Don't touch it; this uses an internal docker network to + # communicate with the frontend. + PENPOT_INTERNAL_URI: http://penpot-frontend:8080 + penpot-mcp: image: "penpotapp/mcp:${PENPOT_VERSION:-2.17}" restart: always @@ -197,7 +218,8 @@ services: - penpot environment: - << : [*penpot-secret-key, *penpot-public-uri] + << : [*penpot-flags, *penpot-secret-key, *penpot-public-uri] + # Don't touch it; this uses an internal docker network to # communicate with the frontend. PENPOT_INTERNAL_URI: http://penpot-frontend:8080 From d842b835d386d320184b21a8f9025e4f28a18a08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Barrag=C3=A1n=20Merino?= Date: Wed, 16 Sep 2026 13:49:06 +0200 Subject: [PATCH 11/22] :books: Reorder menu entries for setup methods MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: David Barragán Merino --- docs/technical-guide/getting-started/docker.md | 2 +- docs/technical-guide/getting-started/elestio.md | 2 +- docs/technical-guide/getting-started/kubernetes.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/technical-guide/getting-started/docker.md b/docs/technical-guide/getting-started/docker.md index a8c7b55ffa..bd18f70809 100644 --- a/docs/technical-guide/getting-started/docker.md +++ b/docs/technical-guide/getting-started/docker.md @@ -1,5 +1,5 @@ --- -title: 1.3 Install with Docker +title: 1.2 Install with Docker desc: This Penpot technical guide covers self-hosting, Docker installation, configuration, updates, backups, and proxy setup with NGINX and Caddy. Try Penpot! --- diff --git a/docs/technical-guide/getting-started/elestio.md b/docs/technical-guide/getting-started/elestio.md index b6dc49af2a..15bef15bae 100644 --- a/docs/technical-guide/getting-started/elestio.md +++ b/docs/technical-guide/getting-started/elestio.md @@ -1,5 +1,5 @@ --- -title: 1.2 Install with Elestio +title: 1.4 Install with Elestio desc: "Step-by-step guide to deploy a self-hosted Penpot on Elestio: 3-minute setup, managed DNS/SMTP/SSL/backups, Docker Compose config, updates & support." --- diff --git a/docs/technical-guide/getting-started/kubernetes.md b/docs/technical-guide/getting-started/kubernetes.md index f2013083c6..1582d4015b 100644 --- a/docs/technical-guide/getting-started/kubernetes.md +++ b/docs/technical-guide/getting-started/kubernetes.md @@ -1,5 +1,5 @@ --- -title: 1.4 Install with Kubernetes +title: 1.3 Install with Kubernetes desc: Learn how to install and configure Penpot on your Kubernetes cluster using Helm. Our technical guide provides step-by-step instructions for setup. --- From 40ef4a90d59d65b73186a78874c0be424b3be11e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Barrag=C3=A1n=20Merino?= Date: Wed, 16 Sep 2026 13:50:00 +0200 Subject: [PATCH 12/22] :books: Document Admin Console setup for Docker (>= 2.18) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: David Barragán Merino --- .../technical-guide/getting-started/docker.md | 85 ++++++++++++++++++- 1 file changed, 84 insertions(+), 1 deletion(-) diff --git a/docs/technical-guide/getting-started/docker.md b/docs/technical-guide/getting-started/docker.md index bd18f70809..66d7a311ba 100644 --- a/docs/technical-guide/getting-started/docker.md +++ b/docs/technical-guide/getting-started/docker.md @@ -115,7 +115,90 @@ This will fetch the latest images. When you do docke It is strongly recommended to update the Penpot version in small increments, rather than updating between two distant versions.

-**Important: Upgrade from version 1.x to 2.0** +#### Upgrade to 2.18 + +This version deploys a new service, **`penpot-admin-console`**, in the official `docker-compose.yaml` +example. If you maintain your own compose file, you need to replicate the following changes manually: + +**1. Add the `enable-admin-console` flag** + +**NOTE:** Enabling the Admin Console is not required in this version, but it will be in a future release. +We recommend setting it up now to avoid a more complex upgrade later. + +This step (the `enable-admin-console` flag) only applies to this version 2.18. In a future release, the +Admin Console will be enabled by default and this flag will be removed. If you're upgrading directly to a +version where the flag has already been removed, skip step 1 and go straight to steps 2–4, which remain +required. + +Wherever you set `PENPOT_FLAGS` (frontend and backend): + +```diff +- PENPOT_FLAGS: disable-email-verification enable-smtp enable-prepl-server disable-secure-session-cookies enable-mcp ++ PENPOT_FLAGS: disable-email-verification enable-smtp enable-prepl-server disable-secure-session-cookies enable-mcp enable-admin-console +``` + +**2. Add the new `penpot-admin-console` service** + +```yaml + penpot-admin-console: + image: "penpotapp/admin-console:2.18" + restart: always + + depends_on: + penpot-postgres: + condition: service_healthy + + networks: + - penpot + + environment: + PENPOT_PUBLIC_URI: http://localhost:9001 + PENPOT_SECRET_KEY: change-this-insecure-key + PENPOT_DATABASE_URI: postgresql://penpot-postgres/penpot + PENPOT_DATABASE_USERNAME: penpot + PENPOT_DATABASE_PASSWORD: penpot + + # Don't touch it; this uses an internal docker network to + # communicate with the frontend. + PENPOT_INTERNAL_URI: http://penpot-frontend:8080 +``` +> Use the same `PENPOT_PUBLIC_URI`, `PENPOT_SECRET_KEY`, and database credentials you already have configured for `penpot-backend`. + +**3. Update `penpot-frontend`** + +- Add `penpot-admin-console` to `depends_on`. +- Add the following environment variable: + +```diff + environment: + PENPOT_FLAGS: disable-email-verification enable-smtp enable-prepl-server disable-secure-session-cookies enable-mcp enable-admin-console + PENPOT_HTTP_SERVER_MAX_BODY_SIZE: 367001600 + PENPOT_HTTP_SERVER_MAX_MULTIPART_BODY_SIZE: 367001600 + PENPOT_PUBLIC_URI: http://localhost:9001 ++ ++ # Don't touch it; this uses an internal docker network to ++ # communicate with the admin-console. ++ PENPOT_ADMIN_CONSOLE_URI: http://penpot-admin-console:3000 +``` + +**4. Update `penpot-backend`** + +Add the same variable: + +```diff + environment: + PENPOT_FLAGS: disable-email-verification enable-smtp enable-prepl-server disable-secure-session-cookies enable-mcp enable-admin-console + PENPOT_PUBLIC_URI: http://localhost:9001 + PENPOT_HTTP_SERVER_MAX_BODY_SIZE: 367001600 + PENPOT_HTTP_SERVER_MAX_MULTIPART_BODY_SIZE: 367001600 + PENPOT_SECRET_KEY: change-this-insecure-key ++ ++ # Don't touch it; this uses an internal docker network to ++ # communicate with the admin-console. ++ PENPOT_ADMIN_CONSOLE_URI: http://penpot-admin-console:3000 +``` + +#### Upgrade from version 1.x to 2.0 The migration to version 2.0, due to the incorporation of the new v2 components, includes an additional process that runs automatically as soon as the application starts. If your From 3a5481c577921bb8e36ab81db15be3c18d5f9f27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mar=C3=ADa=20Valderrama?= Date: Wed, 23 Sep 2026 10:36:40 +0200 Subject: [PATCH 13/22] :sparkles: Error page for instances without admin-console configuration (#11812) * :sparkles: Error page for instances without admin-console configuration (#11649) * :sparkles: Error page for instances without admin-console configuration * :paperclip: Code review * :bug: Fix link nitrate-not-configured --- backend/src/app/http/errors.clj | 7 +++++++ backend/src/app/nitrate.clj | 18 ++++++++++++------ backend/src/app/rpc/commands/profile.clj | 2 +- frontend/src/app/main/errors.cljs | 1 + frontend/src/app/main/repo.cljs | 4 ++++ frontend/src/app/main/ui/static.cljs | 24 ++++++++++++++++++++++++ frontend/src/app/main/ui/static.scss | 8 ++++++++ frontend/translations/en.po | 16 ++++++++++++++++ frontend/translations/es.po | 16 ++++++++++++++++ 9 files changed, 89 insertions(+), 7 deletions(-) diff --git a/backend/src/app/http/errors.clj b/backend/src/app/http/errors.clj index 5f5aa37cb3..e5120b7724 100644 --- a/backend/src/app/http/errors.clj +++ b/backend/src/app/http/errors.clj @@ -166,6 +166,13 @@ {::yres/status 503 ::yres/body {:type :nitrate-unavailable}})) +(defmethod handle-error :nitrate-not-configured + [err request _] + (binding [l/*context* (request->context request)] + (l/warn :hint "nitrate is not configured; blocking request" :cause err) + {::yres/status 503 + ::yres/body {:type :nitrate-not-configured}})) + (defmethod handle-error :internal [error request parent-cause] (binding [l/*context* (request->context request)] diff --git a/backend/src/app/nitrate.clj b/backend/src/app/nitrate.clj index a189116458..bf9f0542d3 100644 --- a/backend/src/app/nitrate.clj +++ b/backend/src/app/nitrate.clj @@ -156,9 +156,13 @@ (defn call [cfg method params] (when (contains? cf/flags :admin-console) - (let [client (get cfg ::client) - method (get client method)] - (method params)))) + (let [uri (cf/get :admin-console-uri)] + (if (nil? uri) + (ex/raise :type :nitrate-not-configured + :hint "admin console is not configured; refer to the documentation to complete the setup") + (let [client (get cfg ::client) + method (get client method)] + (method params)))))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -499,6 +503,8 @@ (defmethod ig/init-key ::client [_ cfg] (when (contains? cf/flags :admin-console) + (when (nil? (cf/get :admin-console-uri)) + (l/warn :hint "admin console is not configured; nitrate calls will fail until the setup is complete")) {:get-team-organization (partial get-team-organization-api cfg) :get-teams-organizations (partial get-teams-organizations-api cfg) :set-team-organization (partial set-team-organization-api cfg) @@ -593,14 +599,14 @@ "Enriches a profile map with subscription information from Nitrate. Adds a :subscription field containing the user's license details. Returns the original profile unchanged if the request fails for a reason - other than Nitrate being unreachable. When Nitrate is unreachable the - `:nitrate-unavailable` exception propagates so the request is rejected." + other than Nitrate being unreachable or unconfigured. When Nitrate is + unreachable or unconfigured the exception propagates so the request is rejected." [cfg profile] (try (let [subscription (call cfg :get-subscription {:profile-id (:id profile)})] (assoc profile :subscription subscription)) (catch Throwable cause - (if (= :nitrate-unavailable (-> cause ex-data :type)) + (if (contains? #{:nitrate-unavailable :nitrate-not-configured} (-> cause ex-data :type)) (throw cause) (do (l/error :hint "failed to get nitrate licence" diff --git a/backend/src/app/rpc/commands/profile.clj b/backend/src/app/rpc/commands/profile.clj index 346f275e9f..572dcc62a4 100644 --- a/backend/src/app/rpc/commands/profile.clj +++ b/backend/src/app/rpc/commands/profile.clj @@ -134,7 +134,7 @@ (catch Throwable cause (if (= :not-found (-> cause ex-data :type)) - {:id uuid/zero :fullname "Anonymous User"} + (with-nitrate-licence {:id uuid/zero :fullname "Anonymous User"} cfg) (throw cause))))) (defn get-profile diff --git a/frontend/src/app/main/errors.cljs b/frontend/src/app/main/errors.cljs index 0998641ce6..c745bebad7 100644 --- a/frontend/src/app/main/errors.cljs +++ b/frontend/src/app/main/errors.cljs @@ -499,6 +499,7 @@ (defmethod ptk/handle-error :bad-gateway [error] (handle-exceptional-state error)) (defmethod ptk/handle-error :service-unavailable [error] (handle-exceptional-state error)) (defmethod ptk/handle-error :nitrate-unavailable [error] (handle-exceptional-state error)) +(defmethod ptk/handle-error :nitrate-not-configured [error] (handle-exceptional-state error)) (defn- redirect-to-dashboard [] diff --git a/frontend/src/app/main/repo.cljs b/frontend/src/app/main/repo.cljs index dcfac7bc22..d4fc1da492 100644 --- a/frontend/src/app/main/repo.cljs +++ b/frontend/src/app/main/repo.cljs @@ -90,6 +90,10 @@ (= :nitrate-unavailable (:type body))) (rx/throw (ex-info "http error" {:type :nitrate-unavailable})) + (and (= 503 status) + (= :nitrate-not-configured (:type body))) + (rx/throw (ex-info "http error" {:type :nitrate-not-configured})) + (= 503 status) (rx/throw (ex-info "http error" {:type :service-unavailable})) diff --git a/frontend/src/app/main/ui/static.cljs b/frontend/src/app/main/ui/static.cljs index 543eede969..dfa1bacfce 100644 --- a/frontend/src/app/main/ui/static.cljs +++ b/frontend/src/app/main/ui/static.cljs @@ -342,6 +342,27 @@ [:p {:class (stl/css :nitrate-unavailable-footer)} (tr "labels.copyright-period")]]) +(mf/defc nitrate-not-configured-page* + [] + [:section {:class (stl/css :nitrate-unavailable-layout)} + [:div {:class (stl/css :nitrate-unavailable-content)} + [:> raw-svg* {:id "logo-nitrate-unavailable" :class (stl/css :nitrate-unavailable-logo)}] + [:div {:class (stl/css :nitrate-unavailable-message)} + (tr "labels.nitrate-not-configured.main-message")] + [:div {:class (stl/css :nitrate-not-configured-message)} + (tr "labels.nitrate-not-configured.desc-message")] + [:div {:class (stl/css :nitrate-not-configured-message)} + [:span + (tr "labels.nitrate-not-configured.learn-more") + " " + [:a {:href "https://help.penpot.app/technical-guide/getting-started/docker/#update-penpot" + :target "_blank" + :rel "noopener noreferrer"} + (tr "labels.nitrate-not-configured.technical-guide")]]]] + + [:p {:class (stl/css :nitrate-unavailable-footer)} + (tr "labels.copyright-period")]]) + (mf/defc webgl-context-lost* [] (let [on-reload (mf/use-fn #(js/location.reload))] @@ -562,6 +583,9 @@ :nitrate-unavailable [:> nitrate-unavailable*] + :nitrate-not-configured + [:> nitrate-not-configured-page*] + :sso-error [:> sso-error-section* {:organization-id (get data :organization-id) :organization-name (get data :organization-name) diff --git a/frontend/src/app/main/ui/static.scss b/frontend/src/app/main/ui/static.scss index 1708e23cd7..bb616a06a0 100644 --- a/frontend/src/app/main/ui/static.scss +++ b/frontend/src/app/main/ui/static.scss @@ -51,6 +51,14 @@ font-weight: 500; } +.nitrate-not-configured-message { + @include t.use-typography("title-medium"); + + color: var(--color-foreground-secondary); + max-width: 32rem; + text-align: center; +} + .nitrate-unavailable-footer { @include t.use-typography("title-medium"); diff --git a/frontend/translations/en.po b/frontend/translations/en.po index 421bd151ca..8135c2f712 100644 --- a/frontend/translations/en.po +++ b/frontend/translations/en.po @@ -3229,6 +3229,22 @@ msgstr "" "Penpot is temporarily unavailable due to a system issue. Please try again " "shortly." +#: src/app/main/ui/static.cljs:344 +msgid "labels.nitrate-not-configured.main-message" +msgstr "Penpot can't start yet." + +#: src/app/main/ui/static.cljs:345 +msgid "labels.nitrate-not-configured.desc-message" +msgstr "This instance needs a setup step that comes with the latest version." + +#: src/app/main/ui/static.cljs:346 +msgid "labels.nitrate-not-configured.learn-more" +msgstr "Check out our" + +#: src/app/main/ui/static.cljs:346 +msgid "labels.nitrate-not-configured.technical-guide" +msgstr "technical guide" + #: src/app/main/ui/dashboard/comments.cljs:120, src/app/main/ui/workspace/comments.cljs:162 msgid "labels.no-comments-available" msgstr "You're all caught up! New comment notifications will appear here." diff --git a/frontend/translations/es.po b/frontend/translations/es.po index 70465aad48..aa583263ea 100644 --- a/frontend/translations/es.po +++ b/frontend/translations/es.po @@ -3138,6 +3138,22 @@ msgstr "" "Penpot no está disponible temporalmente debido a un problema del sistema. " "Por favor, inténtalo de nuevo en unos momentos." +#: src/app/main/ui/static.cljs:344 +msgid "labels.nitrate-not-configured.main-message" +msgstr "Penpot no puede iniciarse aún." + +#: src/app/main/ui/static.cljs:345 +msgid "labels.nitrate-not-configured.desc-message" +msgstr "Esta instancia requiere un paso de configuración que se incluye en la versión más reciente." + +#: src/app/main/ui/static.cljs:346 +msgid "labels.nitrate-not-configured.learn-more" +msgstr "Echa un vistazo a nuestra" + +#: src/app/main/ui/static.cljs:346 +msgid "labels.nitrate-not-configured.technical-guide" +msgstr "technical guide" + #: src/app/main/ui/dashboard/comments.cljs:120, src/app/main/ui/workspace/comments.cljs:162 msgid "labels.no-comments-available" msgstr "¡Ya estás al día! Nuevas notificaciones de comentarios aparecerán aquí." From 5baffdc213f0deaaeb318e97a41d611ac0656a94 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 23 Sep 2026 10:41:41 +0200 Subject: [PATCH 14/22] :paperclip: Update changelog --- CHANGES.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 1ae489b0ab..ecec05d735 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,6 @@ # CHANGELOG -## 2.18.0 (Unreleased) +## 2.18.0 ### :rocket: Epics and highlights @@ -130,9 +130,11 @@ - Fix workspace crash with index out of bounds when removing multiple interactions rapidly [#11546](https://github.com/penpot/penpot/issues/11546) (PR: [#11621](https://github.com/penpot/penpot/pull/11621)) - Fix workspace crash with 'too much recursion' error when editing grid layout [#11562](https://github.com/penpot/penpot/issues/11562) (PR: [#11563](https://github.com/penpot/penpot/pull/11563)) - Fix plugin API error when removing interactions [#11618](https://github.com/penpot/penpot/issues/11618) (PR: [#11621](https://github.com/penpot/penpot/pull/11621)) +- Fix MCP REPL server starting in multi-user mode and inheriting the main server bind address [#11631](https://github.com/penpot/penpot/issues/11631) (PR: [#11632](https://github.com/penpot/penpot/pull/11632)) - Fix chunked upload storing duplicate objects when the same chunk index is sent twice [#11634](https://github.com/penpot/penpot/issues/11634) (PR: [#11635](https://github.com/penpot/penpot/pull/11635)) - Fix access token expiration select always showing Never after picking another option (by @chulgil) [#11663](https://github.com/penpot/penpot/issues/11663) (PR: [#11664](https://github.com/penpot/penpot/pull/11664)) - Fix workspace dropping unsaved changes when read-only blocks persistence [#11672](https://github.com/penpot/penpot/issues/11672) (PR: [#11699](https://github.com/penpot/penpot/pull/11699)) +- Fix Plugin API failing silently when writing to an interaction the shape no longer has [#11780](https://github.com/penpot/penpot/issues/11780) (PR: [#11781](https://github.com/penpot/penpot/pull/11781)) ### :sparkles: New features & Enhancements @@ -156,6 +158,8 @@ - Adopt React Aria [#10802](https://github.com/penpot/penpot/issues/10802) (PR: [#10675](https://github.com/penpot/penpot/pull/10675)) - Add plugin API function for awaiting component updates beyond waitForLayoutUpdate [#10927](https://github.com/penpot/penpot/issues/10927) (PR: [#10964](https://github.com/penpot/penpot/pull/10964)) - Emit open-workspace-file audit event with file statistics on workspace load [#11106](https://github.com/penpot/penpot/issues/11106) (PR: [#11138](https://github.com/penpot/penpot/pull/11138)) +- Restrict optional ids to v4, v7 and v8 UUIDs in creation RPC endpoints [#11778](https://github.com/penpot/penpot/issues/11778) (PR: [#11777](https://github.com/penpot/penpot/pull/11777)) + ## 2.17.2 ### :bug: Bugs fixed From 6d4060e7a9d4b38a998bdc201e8e6c40e0aa2055 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Barrag=C3=A1n=20Merino?= Date: Tue, 4 Aug 2026 11:57:31 +0200 Subject: [PATCH 15/22] :wrench: Release admin-console images on final tags --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2c00f841ff..d54b33ec03 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -62,7 +62,7 @@ jobs: echo "$PUB_DOCKER_PASSWORD" | skopeo login --username "$PUB_DOCKER_USERNAME" --password-stdin docker.io - IMAGES=("frontend" "backend" "exporter" "mcp" "storybook") + IMAGES=("frontend" "backend" "exporter" "admin-console" "mcp" "storybook") SHORT_TAG=${TAG%.*} for image in "${IMAGES[@]}"; do From 8302a984a2b27b6b7dfaf3d9a11995ca6c21d735 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Barrag=C3=A1n=20Merino?= Date: Wed, 23 Sep 2026 12:32:24 +0200 Subject: [PATCH 16/22] :whale: Bump penpotapp images to 2.18 in docker-compose MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: David Barragán Merino --- docker/images/docker-compose.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docker/images/docker-compose.yaml b/docker/images/docker-compose.yaml index 9d5082b4b4..df846c43b4 100644 --- a/docker/images/docker-compose.yaml +++ b/docker/images/docker-compose.yaml @@ -85,7 +85,7 @@ services: # - "443:443" penpot-frontend: - image: "penpotapp/frontend:${PENPOT_VERSION:-2.17}" + image: "penpotapp/frontend:${PENPOT_VERSION:-2.18}" restart: always ports: - 9001:8080 @@ -124,7 +124,7 @@ services: # PENPOT_DISABLE_IPV6_LISTEN: "true" penpot-backend: - image: "penpotapp/backend:${PENPOT_VERSION:-2.17}" + image: "penpotapp/backend:${PENPOT_VERSION:-2.18}" restart: always volumes: @@ -183,7 +183,7 @@ services: PENPOT_SMTP_SSL: "false" penpot-admin-console: - image: "penpotapp/admin-console:${PENPOT_VERSION:-2.16}" + image: "penpotapp/admin-console:${PENPOT_VERSION:-2.18}" restart: always depends_on: @@ -201,13 +201,13 @@ services: PENPOT_INTERNAL_URI: http://penpot-frontend:8080 penpot-mcp: - image: "penpotapp/mcp:${PENPOT_VERSION:-2.17}" + image: "penpotapp/mcp:${PENPOT_VERSION:-2.18}" restart: always networks: - penpot penpot-exporter: - image: "penpotapp/exporter:${PENPOT_VERSION:-2.17}" + image: "penpotapp/exporter:${PENPOT_VERSION:-2.18}" restart: always depends_on: From 45b8320ac7321092efc63c57889b37102c635d4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Moya?= Date: Wed, 23 Sep 2026 14:47:10 +0200 Subject: [PATCH 17/22] :bug: Fix error when reset overrides (#11604) * :bug: Fix error when reset overrides on a swapped copy * :bug: Add regression test for reset overrides inside group Cover the case where a nested copy lives inside a group (not directly under the instance root). After a swap, reset overrides must undo the swap without error (#11656). --------- Co-authored-by: Alejandro Alonso --- common/src/app/common/logic/libraries.cljc | 6 +- .../logic/swap_and_reset_test.cljc | 60 +++++++++++++++++++ 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/common/src/app/common/logic/libraries.cljc b/common/src/app/common/logic/libraries.cljc index b234263468..c587a1ff6d 100644 --- a/common/src/app/common/logic/libraries.cljc +++ b/common/src/app/common/logic/libraries.cljc @@ -2783,7 +2783,7 @@ (let [shape-inst (ctn/get-shape container id) objects (:objects container) parent (get objects (:parent-id shape-inst)) - head (ctn/get-component-shape container parent) + head (ctn/get-component-shape objects parent) changes (-> changes (pcb/with-container container) @@ -2802,6 +2802,10 @@ (pcb/with-objects objects) (generate-sync-shape-direct file-full libraries container id true))] + ;; If the shape has been swapped, the reset also have to undo the swap. As the + ;; shape has been reset and it's not touched now, by the sync above, if we + ;; do another reset of the parent, the shape will be reset as it is in the + ;; near main component. (cond-> changes (some? swap-slot) (generate-sync-head file-full libraries container id true)))) diff --git a/common/test/common_tests/logic/swap_and_reset_test.cljc b/common/test/common_tests/logic/swap_and_reset_test.cljc index c9cad989cc..3de2f8bbcf 100644 --- a/common/test/common_tests/logic/swap_and_reset_test.cljc +++ b/common/test/common_tests/logic/swap_and_reset_test.cljc @@ -82,3 +82,63 @@ (validate #(t/is (= (copy-type %) :rect))) (validate #(t/is (= (nested-component-id %) (rectangle-component-id %)))) (validate #(t/is (nil? (nested-swap-slot %))))))) + +;; The nested component sits inside a group, not directly under the instance root. +;; Resetting overrides after a swap must undo the swap without error. +(t/deftest test-swap-and-reset-override-inside-group + (letfn [(nested-in-copy [file] + (->> (ths/get-shape file :copy-group) + :shapes + first + (ths/get-shape-by-id file)))] + + (let [;; ==== Setup + file + (-> (thf/sample-file :file1) + + (tho/add-frame :frame-rectangle) + (ths/add-sample-shape :rectangle-shape :parent-label :frame-rectangle :type :rect) + (thc/make-component :rectangle :frame-rectangle) + + (tho/add-frame :frame-circle) + (ths/add-sample-shape :circle :parent-label :frame-circle :type :circle) + (thc/make-component :circle :frame-circle) + + (tho/add-frame :frame-main) + (tho/add-group :group-main :parent-label :frame-main) + (thc/instantiate-component :rectangle :nested-rectangle + :parent-label :group-main + :children-labels [:nested-rectangle-shape]) + (thc/make-component :main :frame-main) + + (thc/instantiate-component :main :copy + :children-labels [:copy-group + :copy-nested-rectangle + :copy-nested-rectangle-shape])) + + rectangle-id (:id (thc/get-component file :rectangle)) + circle-id (:id (thc/get-component file :circle)) + main-nested-id (:id (ths/get-shape file :nested-rectangle)) + + ;; ==== Action – swap nested copy inside the group, then reset overrides + file-swapped + (tho/swap-component-in-shape file :copy-nested-rectangle :circle + :new-shape-label :copy-nested-swapped) + + swapped (ths/get-shape file-swapped :copy-nested-swapped) + + file' + (tho/reset-overrides file-swapped swapped) + + restored (nested-in-copy file')] + + ;; ==== Check – after swap + (t/is (= :circle (:type (tho/bottom-shape file-swapped :copy-nested-swapped)))) + (t/is (= circle-id (:component-id swapped))) + (t/is (= main-nested-id (ctk/get-swap-slot swapped))) + + ;; ==== Check – after reset: back to rectangle, no swap slot, file still valid + (t/is (some? restored)) + (t/is (= :rect (:type (tho/bottom-shape-by-id file' (:id restored))))) + (t/is (= rectangle-id (:component-id restored))) + (t/is (nil? (ctk/get-swap-slot restored)))))) From 5b3844c37a87cd023341319de307072a8b41b853 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 23 Sep 2026 12:50:18 +0200 Subject: [PATCH 18/22] :sparkles: Add observability improvements (#11854) * :whale: Add upstream diagnostics to nginx access log Enrich every access-log line with the internal journey of the request: the status the backend answered (us), the time spent connecting to it (uct), the time spent waiting for its answer (urt) and the internal address that served the request (ua). A plain 502 line used to say nothing about where the request died. With this format, the tail of the line classifies the failure: connection rejected, backend accepted and hung (uct + urt under 1s), or backend stuck until read timeout. This was the missing witness in the Sep 20 incident, where nginx received connection resets with zero timeouts and zero rejections. Applied both to the production image template and the devenv config. With proxy_pass on variables there is no upstream keepalive, so uct measures one real TCP connection per request. Parsing the new fields (us, uct, urt, ua) on the log shipper is left to ops, so they can be filtered in Loki. AI-assisted-by: glm-5.3-flash * :whale: Add stub_status endpoint for nginx metrics Add a dedicated localhost-only server (listen 127.0.0.1:8082) exposing /stub_status next to every other location of the public server. Ops can run the official nginx-prometheus-exporter as a sidecar against http://127.0.0.1:8082/stub_status and get nginx_connections_active, accepted vs handled, reading/writing/waiting and request rates in Prometheus. Binding it to localhost and its own server keeps it unreachable from outside the host and out of the public surface, and access_log off avoids polluting Loki with one line per Prometheus scrape. The base image already ships stub_status compiled in, so no image rebuild is needed. Applied both to the production image template and the devenv config. AI-assisted-by: glm-5.3-flash * :sparkles: Expose http server gate metrics (worker and connector) The backend already measured dispatch latency but nothing reported the state of the "house door": the xnio worker queue and threads, and the monitor-level listener counters. This was the exact blind spot of the Sep 20 incident, where the server kept answering health checks while it accepted connections and dropped them without response. Add a periodic metrics sampler that lives and dies with the http server (single daemon thread, 15s interval, each sample guarded so an unexpected error does not cancel subsequent runs) and publishes: - worker (xnio MXBean gauges): penpot_http_worker_queue_size, busy_threads, pool_size and max_pool_size. Negative samples are discarded: the MXBean transiently reports -1 on the busy thread count (verified live), and a stale negative would read as zero. - listener (Undertow connector statistics, enabled via the new :server/statistics yetti option): penpot_http_connector_active* _connections gauge and requests_total / errors_total counters. Undertow exposes absolute totals, so the sampler keeps a watermark atom and publishes deltas, skipping (and moving forward past) a counter reset. The connector-level part depends on yetti v11.11, which now accepts a :server/statistics server option (patch authored and released upstream; before it, ListenerInfo#getConnectorStatistics always returned nil). New tests cover the samplers with fake MXBean/collector statistics against real prometheus collectors, including the negative-sample filter, the delta/watermark logic and the sampler lifecycle. AI-assisted-by: glm-5.3-flash * :bug: Include jdk.management in the backend runtime JRE The production image builds a trimmed JRE with jlink and omitted jdk.management. Without that module the OS MXBean is sun.management.BaseOperatingSystemImpl, which has no getProcessCpuTime, getOpenFileDescriptorCount nor getMaxFileDescriptorCount. The prometheus client StandardExports reads those getters reflectively and collect() swallows the NoSuchMethodException, so process_open_fds, process_max_fds and process_cpu_seconds_total silently disappeared from /metrics while the other process_* families kept flowing. Verified against Prometheus: the app job only ever exposed process_start_time_seconds, process_virtual_memory_bytes and process_resident_memory_bytes; the fd and cpu families were absent. Reproduced locally by running the backend metrics registry on a JRE built with the same jlink module list (false/false/false) and on one with jdk.management added (true/true/true). Add the module to --add-modules and pin the metric contract with backend-tests.metrics-test. AI-assisted-by: deepseek-v4.1-flash * :recycle: Build the http metrics sampler on promesa.exec Replace the hand-rolled ScheduledThreadPoolExecutor and ThreadFactory with promesa.exec primitives: px/scheduled-executor with a daemon thread factory, and a px/schedule chain that reschedules the next sample when the current one finishes. Beyond fitting the existing periodic-task pattern (worker/cron, rpc/rlimit), the chained schedule makes the docstring promise real: with scheduleAtFixedRate an exception escaping the runnable cancelled the following executions, while the reschedule now happens in a finally block. The sampler shutdown uses px/shutdown-now (shutdown! is deprecated in promesa 12.0.0) to cancel the pending sample, keeping the previous halt semantics. The lifecycle test moves to the promesa predicates and a new test covers the error-resilience promise: the first sample runs, throws, and the next one is still scheduled. AI-assisted-by: deepseek-v4.1-flash * :recycle: Tighten the http metrics samplers The samplers are leaf functions: they receive what they need and publish it. Drop the internal nil guards (if there is no metrics instance or no mxbean there is nothing to call them for) and move the checks to the boundary, where the optional data is resolved: sample-http-metrics now short-circuits with some-> and when-let. Write the four worker gauges as four static operations instead of a vector of pairs walked by doseq: the set is fixed, so the collection only adds an allocation and hides each operation. Drop the ! suffix from the sample-*-metrics family: ! marks a function whose contract is to mutate state, while these report, and the mutation happens in the mtx/run! they call. The constant true return, which only existed so the removed guard tests could assert it, goes away too. Tests follow the move: the internal-guard tests are replaced by one boundary test (a nil server publishes nothing). AI-assisted-by: deepseek-v4.1-flash * :books: Add the function design rules memory Document the rules that came out of the http metrics sampler review: preconditions are checked at the boundary instead of re-checked in the core, optional-by-design data is guarded where the optionality is born, a fixed set of operations is written statically, ! marks mutation and not reporting, and production code is not shaped for tests. Also state in the memory maintenance guide that memories must not use manual line wrapping. Linked from critical-info so it is read when designing a solution or an API, not only when touching the samplers. AI-assisted-by: deepseek-v4.1-flash * :books: Unwrap the critical-info memory lines The memory maintenance guide forbids manual line wrapping, so rewrite critical-info with one line per bullet and paragraph. A stray `*` at the start of one continuation line is dropped. AI-assisted-by: deepseek-v4.1-flash * :recycle: Drop the redundant guard in the http server halt create-metrics-sampler always returns the scheduler, so the sampler is always present when integrant calls halt-key!; the nil check was dead code, same as the yt/stop! call next to it. AI-assisted-by: deepseek-v4.1-flash * :sparkles: Add srepl helper to delete profiles by email Add `delete-profiles-by-email!` to app.srepl.main. It accepts a single email, a comma separated list of emails or a coll of emails, resolves each profile, logs it to audit and enqueues the delete-object task. The deleted-at is backdated with the configured deletion-delay so profiles and their owned teams are purged on the next gc pass. Extract the per-email deletion logic into a private fn and reuse it from `delete-profiles-in-bulk!`. Add tests for the new `parse-emails` helper. AI-assisted-by: glm-5.3-flash --- .serena/memories/backend/subtleties.md | 61 +++++ .serena/memories/clojure/design-rules.md | 13 + .serena/memories/critical-info.md | 69 ++--- .serena/memories/memory-maintenance.md | 1 + backend/deps.edn | 4 +- backend/src/app/http.clj | 109 +++++++- backend/src/app/main.clj | 37 ++- backend/src/app/srepl/main.clj | 115 +++++--- .../test/backend_tests/http_metrics_test.clj | 257 ++++++++++++++++++ backend/test/backend_tests/metrics_test.clj | 49 ++++ .../test/backend_tests/srepl_main_test.clj | 28 ++ docker/devenv/files/nginx.conf | 27 +- docker/images/Dockerfile.backend | 7 +- docker/images/files/nginx.conf.template | 28 +- 14 files changed, 706 insertions(+), 99 deletions(-) create mode 100644 .serena/memories/backend/subtleties.md create mode 100644 .serena/memories/clojure/design-rules.md create mode 100644 backend/test/backend_tests/http_metrics_test.clj create mode 100644 backend/test/backend_tests/metrics_test.clj create mode 100644 backend/test/backend_tests/srepl_main_test.clj diff --git a/.serena/memories/backend/subtleties.md b/.serena/memories/backend/subtleties.md new file mode 100644 index 0000000000..e051a8de74 --- /dev/null +++ b/.serena/memories/backend/subtleties.md @@ -0,0 +1,61 @@ +# Backend Subtleties + +## RPC exposure and wrappers + +- RPC commands are discovered from vars created by `app.util.services/defmethod`; adding a command namespace is not enough unless `backend/src/app/rpc.clj` includes it in `resolve-methods`. +- `GET`/`HEAD` RPC calls are only allowed for method names starting with `get-`. Other methods are method-not-allowed even if they are read-only internally. +- RPC auth defaults to enabled. Public endpoints must set `::auth false` metadata explicitly. +- The wrapper stack does auth before params validation, then auditing/rate/concurrency/metrics/retry/condition handling, with DB transaction handling inside that stack. `::db/transaction` metadata controls transaction wrapping. +- Params with `::sm/params` are decoded/conformed through the JSON transformer and successful IObj results get `:encode/json` metadata. Legacy spec conforming only applies when no Malli params schema exists. Client params are stripped of qualified keys (`d/without-qualified`) before merging with the server auth context, so request bodies cannot override `::profile-id`, `::auth-type`, or `::token-perms`. +- Params schemas are open by default, so undeclared client keys reach the handler unless the map is `:closed true`. Creation commands (`create-file`, `create-project`, `create-team`, `create-team-with-invitations`, `upload-file-media-object`, `create-file-media-object-from-url`, `assemble-file-media-object`) use closed schemas: a client-provided `:id` fails with `:params-validation`. Their internal creation functions still accept an optional explicit `:id` for imports, duplicates and deterministic test fixtures. +- Nil RPC bodies become HTTP 204 unless explicit status metadata is present. Stream bodies default to `application/octet-stream` when no content type is set. + +## DB helpers + +- Most `app.db` helpers accept a pool, connection, or map containing `::db/pool` / `::db/conn`; preserve that convention in shared code. +- `db/tx-run!` uses `next.jdbc.transaction/*nested-tx* :ignore`: nested transaction calls reuse the outer transaction, not a savepoint. Use explicit savepoints when nested rollback semantics matter. +- `db/run!` opens/reuses one connection but does not create a transaction. +- `db/tjson` is Transit JSON for jsonb storage; `db/json` is plain JSON. Worker task props use Transit and are decoded with `decode-transit-pgobject`. +- Advisory transaction locks accept UUIDs or ints. UUID locks are hashed using a zero-UUID seeded siphash. + +## Workers and cron + +- Task queues are tenant-prefixed. Submit dedupe only removes not-yet-due `new` tasks with the same name/queue/label; it does not dedupe due, scheduled, retry, running, or completed work. +- The dispatcher selects `new`/`retry` tasks with `FOR UPDATE SKIP LOCKED`, marks them `scheduled`, and publishes Redis payload `[id scheduled-at]`. The runner skips Redis messages whose scheduled timestamp no longer matches DB state. +- Lost `scheduled` tasks are rescheduled after about 5 minutes; `running` tasks older than about 24 hours are marked failed as orphans. +- A task handler that is missing or returns an invalid result currently defaults to completed after warning. Throwing with `ex-data :type ::retry` controls retry behavior; `:strategy ::noop` retries without incrementing retry count. +- Cron jobs lock their `scheduled_task` row with `FOR UPDATE SKIP LOCKED`, disable statement/idle-in-transaction timeouts locally, and reschedule themselves in `finally` unless interrupted. Worker, dispatcher, and cron components do not start when the DB pool is read-only. + +## Config and HTTP/session middleware + +- `app.config/config` and `flags` are dynamic `defonce` vars populated from `PENPOT_*` env vars through the shared schema string transformer. Tests and tooling can bind them. +- `parse-flags` automatically adds `:disable-secure-session-cookies` when `public-uri` is plain HTTP and not localhost. This changes cookie defaults without an explicit env flag. +- The backend sets Clojure `*assert*` globally from the `:backend-asserts` feature flag. Assertion-dependent checks can therefore differ by runtime flags. +- Request body parsing is mostly POST-oriented and supports Transit JSON plus plain JSON. Plain JSON request keys are kebab-decoded before being merged into `:params`. +- Response formatting negotiates with `Accept` or `_fmt=json`. Transit is the default for collection/boolean bodies; JSON encoding has special pointer-map handling. +- Auth prefers the session cookie token before the `Authorization` header. Headers may be `Token` or `Bearer`; JWTs with `kid=1` and `ver=1` are decoded as v1 session tokens, otherwise they are treated as legacy tokens. +- Shared-key auth requires `x-shared-key` as ` ` and stores the lowercased key id on the request. If no shared keys are configured it always rejects. +- Session management uses DB storage unless the DB pool is read-only, then falls back to the in-memory manager. DB sessions support both legacy string ids and v2 UUID session ids. +- Session cookies are renewed when using a legacy string id or when `modified-at` is older than the renewal interval. SameSite is `none` for CORS, otherwise strict/lax based on config. + +## HTTP server self-metrics + +- `app.http` enables Undertow connection statistics via the yetti option `:server/statistics` (requires yetti ≥ v11.11, which exposes it; before the patch `ListenerInfo#getConnectorStatistics` returned `nil`). +- A daemon sampler built on `promesa.exec` (`px/scheduled-executor` plus a self-rescheduling `px/schedule` chain, so a failing sample never cancels the next one; 15 s, started with the server in `ig/init-key` and stopped with `px/shutdown-now` in `halt-key!`) publishes worker and listener state: `penpot_http_worker_queue_size`, `busy_threads`, `pool_size`, `max_pool_size`, `penpot_http_connector_active_connections`, `requests_total`, `errors_total`. Definitions live in `app.main/default-metrics`. +- The xnio worker MXBean can return transient `-1` (e.g. busy-thread count); negative samples are discarded (gauge keeps its previous value). Undertow exposes absolute request/error totals, so the sampler keeps a watermark atom and publishes deltas; a counter reset (decreasing totals) skips the negative delta and moves the watermark forward. +- The `process_*` families (`process_open_fds`, `process_max_fds`, `process_cpu_seconds_total`, …) come from the prometheus client `StandardExports`, registered by `app.metrics/create-registry`. They read the OS MXBean reflectively and need the `jdk.management` module: on a pruned `jlink` JRE the MXBean is `sun.management.BaseOperatingSystemImpl`, the getters throw `NoSuchMethodException` and `StandardExports#collect` swallows it, so those families silently vanish from `/metrics`. `docker/images/Dockerfile.backend` keeps `jdk.management` in the `--add-modules` list, and `backend-tests.metrics-test` pins the contract. + +## Storage and media + +- Storage abstraction, backend configuration, logical buckets, object lifecycle, deduplication, access rules, and garbage collection: `mem:backend/storage`. +- SVG validation strips DOCTYPE and uses secure SAX parsing. Basic SVG info falls back to 100x100 dimensions when width/height/viewBox are missing. +- Raster metadata is shell-derived with ImageMagick `identify`, verifies detected MIME against the supplied MIME, and swaps dimensions for EXIF orientations 6/8. +- Remote image download requires 2xx status, `content-length`, a known MIME, and size under the configured maximum before writing the temp file; mismatched byte count is an internal error. +- Font processing shells out to FontForge and WOFF conversion tools and can derive TTF/OTF/WOFF variants from uploaded fonts. + +## File data persistence + +- File data backends are `legacy-db`, `db`, and `storage`. The storage backend keeps encoded file data in storage bucket `file-data`; the DB row stores metadata with `storage-ref-id` and nil data. +- `fdata/upsert!` touches any storage object referenced by incoming metadata before storing the new row/blob. +- Pointer-map fragments are persisted separately as type `fragment`, and only modified pointer maps are written. +- `fdata/realize` combines pointer realization and object-map realization. Use it before operations that need complete in-memory file data instead of pointer placeholders. \ No newline at end of file diff --git a/.serena/memories/clojure/design-rules.md b/.serena/memories/clojure/design-rules.md new file mode 100644 index 0000000000..48c7f40193 --- /dev/null +++ b/.serena/memories/clojure/design-rules.md @@ -0,0 +1,13 @@ +# Clojure Design Rules + +How to shape a function in this codebase. Each rule came from a review round; do not re-derive from taste. + +- **Preconditions belong to the boundary, not to the core.** A function documents what it needs (hints, docstring) and assumes it. Absence checks live where the value enters the system: the caller that resolves optional objects, `ig/init-key`, the request handler. A `(when (and (some? a) (some? b)) ...)` inside a leaf function means the guard is in the wrong place — if you cannot get `a` or `b`, you should not be calling it. +- **Optional-by-design data is guarded where the optionality is born.** Example: `ConnectorStatistics` is nil when the server option is off, so the guard is `when-let [cs (some-> server ...)]` at the entry point and the consumer assumes `cs`. +- **A static set of operations is written statically.** With a fixed, small set (four metrics) write the four calls. Do not build a collection and iterate it (`doseq` over a literal vector): it allocates, adds indirection and hides each operation. Use a collection when the set is dynamic (config, registry, input). +- **The name says what the function is; `!` says it mutates.** `!` marks functions whose contract is to change state or run a command (`run!`, `submit!`, `swap!`, `shutdown-now`). Reporting helpers (log, sample-and-publish) do not take it: the mutation happens in the `!` API they call. Keep a family consistent. +- **Do not shape production code for tests.** No return values added just to assert them; assert on the observable effect. When preconditions move, move the tests with them. + +Reference implementation: the http metrics samplers in `app.http` (`sample-worker-metrics` / `sample-connector-metrics` are guard-free; `sample-http-metrics` guards at the boundary). + +See also: `mem:clojure/idioms` (language behaviors), `mem:testing` (TDD and test conventions). diff --git a/.serena/memories/critical-info.md b/.serena/memories/critical-info.md index ce87caee30..f0cec128dc 100644 --- a/.serena/memories/critical-info.md +++ b/.serena/memories/critical-info.md @@ -3,13 +3,11 @@ You are working on the GitHub project `penpot/penpot`, a monorepo. # Memory system - Memories are the primary project guidance (not docs or other readme files). -- A section's top-level memory is `
/core`. When a section is relevant, read the core memory - before focused memories. +- A section's top-level memory is `
/core`. When a section is relevant, read the core memory before focused memories. - Edits/stale refs/duplication cleanup: `mem:memory-maintenance`. - Cross-cutting testing principles, TDD workflow, and anti-patterns: `mem:testing`. -- Verified Clojure language behaviors that contradict common assumptions - (e.g. `int?` covers `Long`; `integer?` is the general predicate): - `mem:clojure/idioms` — read before assuming stdlib predicate semantics. +- Verified Clojure language behaviors that contradict common assumptions (e.g. `int?` covers `Long`; `integer?` is the general predicate): `mem:clojure/idioms` — read before assuming stdlib predicate semantics. +- When designing a solution or an API, read `mem:clojure/design-rules` (function shape, boundary checks, naming). # Development workflow @@ -17,20 +15,14 @@ You are working on the GitHub project `penpot/penpot`, a monorepo. - Before `git commit` → `mem:workflow/creating-commits` (subject/body format, 76-char body wrapping enforced by `scripts/check-commit`, `AI-assisted-by: model-name` trailer) - Before `gh issue create` → `mem:workflow/creating-issues` (title derivation, body template, labels, Issue Type) - Before `gh pr create` / `gh pr edit` → `mem:workflow/creating-prs` (title format, body structure, "Note:" line) -- Before a repo-wide pnpm version update → `mem:workflow/updating-pnpm` (workspace - layout, `scripts/sync-pnpm-version` flow, the stamp-missing-field and - ignored-builds gotchas, verification steps) +- Before a repo-wide pnpm version update → `mem:workflow/updating-pnpm` (workspace layout, `scripts/sync-pnpm-version` flow, the stamp-missing-field and ignored-builds gotchas, verification steps) - **Never `git push`, force-push, or modify `git origin`** (or any other remote). The user pushes from their own shell; if a push is required, say so and wait. Never amend a commit that the user has already pushed unless explicitly asked. - **Never edit `CHANGES.md` by hand.** The changelog is generated from GitHub milestones during the release process; update it only via the `update-changelog` skill flow or on explicit user request. - You have access to the GitHub CLI `gh` or corresponding MCP tools. - Issues are also managed on Taiga. Read issues using the `read_taiga_issue` tool. -- Before writing code, analyze the task in depth and describe your plan. If the task is complex, break it down into atomic steps. - *After making changes, run the applicable lint and format checks for the affected module before considering the work done (per example `mem:backend/core` or `mem:frontend/core`). -- Align `let` binding values: when a `let` form has multiple bindings spanning - several lines, align the value forms to the same column with spaces. -- If you introduce delimiter errors (mismatched parens/brackets) in Clojure/CLJS files, - fix them with `scripts/paren-repair` BEFORE running lint/format checks. - See `mem:scripts/paren-repair` for usage. +- Before writing code, analyze the task in depth and describe your plan. If the task is complex, break it down into atomic steps. After making changes, run the applicable lint and format checks for the affected module before considering the work done (per example `mem:backend/core` or `mem:frontend/core`). +- Align `let` binding values: when a `let` form has multiple bindings spanning several lines, align the value forms to the same column with spaces. +- If you introduce delimiter errors (mismatched parens/brackets) in Clojure/CLJS files, fix them with `scripts/paren-repair` BEFORE running lint/format checks. See `mem:scripts/paren-repair` for usage. - Never run anything that destroys data without explicit permission, including `drop-devenv`, `docker compose down -v`, `docker volume rm ...`. The user's real work lives in the volumes of the shared infra. # Project modules @@ -48,49 +40,28 @@ This is a monorepo. Principles that apply to one module do *not* generally apply - `docs/`: documentation site; core workflow and conventions: `mem:docs/core`. - `media-processor/`: TypeScript/Node.js HTTP service for image (sharp) and font (FontForge) processing; core conventions: `mem:media-processor/core`. -The memory is structured in a way that you can get the critical information about the -module. You can read it from `mem:/core` +The memory is structured in a way that you can get the critical information about the module. You can read it from `mem:/core` # Low-centrality project paths -- `docker/` contains devenv related code, not needed unless specifically instructed. - When working on devenv startup, compose layout, instance config (`defaults.env`), - tmux session lifecycle, RustFS provisioning, or anything in `manage.sh`'s - `*-devenv` commands, read `mem:devenv/core`. +- `docker/` contains devenv related code, not needed unless specifically instructed. When working on devenv startup, compose layout, instance config (`defaults.env`), tmux session lifecycle, RustFS provisioning, or anything in `manage.sh`'s `*-devenv` commands, read `mem:devenv/core`. - `experiments/` contains standalone experimental HTML/JS/scripts; treat it as non-core unless the user explicitly asks about it. - `sample_media/` contains sample image/icon media and config used as fixtures/demo material; do not infer app behavior from it. # Dev Scripts (scripts/) -- `scripts/nrepl-eval.mjs` — Evaluate Clojure/ClojureScript code via nREPL. - Supports `--backend` (port 6064) and `--frontend` (port 3447) aliases. - See `mem:scripts/nrepl-eval`. -- `scripts/paren-repair` — Fix mismatched delimiters in Clojure/CLJS files - and reformat with cljfmt. Run before lint checks when LLM edits break parens. - See `mem:scripts/paren-repair`. -- `scripts/psql` — PostgreSQL client wrapper with devenv defaults. - Companion: `scripts/db-schema` for DDL dumps. See `mem:scripts/psql`. -- `scripts/taiga.py` — Fetch public issues, user stories, and tasks from the - Penpot Taiga project without authentication. See `mem:scripts/taiga`. -- `scripts/gh.py` — GitHub operations helper: list milestone issues, fetch PR - details, compare against CHANGES.md. Requires `gh` CLI. See `mem:scripts/gh`. -- `scripts/error-reports.mjs` — Query error reports via RPC API with token - authentication. Supports list/get operations with filtering and pagination. - See `mem:scripts/error-reports`. -- `scripts/clean-node-modules` — Remove stale `node_modules` from all pnpm - workspaces (root, modules, member packages). Keeps the shared pnpm store - at `/.pnpm-store` unless `--store`; ignores `external/` and - `.opencode/`. Usage and reinstall steps: `mem:workflow/updating-pnpm`. -- `scripts/ci` — CI orchestration script: runs lint, tests, and format - checks per module (`frontend backend common render-wasm exporter mcp - plugins library`). Logs go to `.ci-logs/`; read the log file on failure. - See `mem:scripts/ci`. +- `scripts/nrepl-eval.mjs` — Evaluate Clojure/ClojureScript code via nREPL. Supports `--backend` (port 6064) and `--frontend` (port 3447) aliases. See `mem:scripts/nrepl-eval`. +- `scripts/paren-repair` — Fix mismatched delimiters in Clojure/CLJS files and reformat with cljfmt. Run before lint checks when LLM edits break parens. See `mem:scripts/paren-repair`. +- `scripts/psql` — PostgreSQL client wrapper with devenv defaults. Companion: `scripts/db-schema` for DDL dumps. See `mem:scripts/psql`. +- `scripts/taiga.py` — Fetch public issues, user stories, and tasks from the Penpot Taiga project without authentication. See `mem:scripts/taiga`. +- `scripts/gh.py` — GitHub operations helper: list milestone issues, fetch PR details, compare against CHANGES.md. Requires `gh` CLI. See `mem:scripts/gh`. +- `scripts/error-reports.mjs` — Query error reports via RPC API with token authentication. Supports list/get operations with filtering and pagination. See `mem:scripts/error-reports`. +- `scripts/clean-node-modules` — Remove stale `node_modules` from all pnpm workspaces (root, modules, member packages). Keeps the shared pnpm store at `/.pnpm-store` unless `--store`; ignores `external/` and `.opencode/`. Usage and reinstall steps: `mem:workflow/updating-pnpm`. +- `scripts/ci` — CI orchestration script: runs lint, tests, and format checks per module (`frontend backend common render-wasm exporter mcp plugins library`). Logs go to `.ci-logs/`; read the log file on failure. See `mem:scripts/ci`. # Dependency graph -`frontend -> common`, `backend -> common`, `exporter -> common`, and `frontend -> render-wasm`. Changes in `common` can -affect frontend, backend, exporter, file migrations, and design-library behavior; validate across consumers when -semantics change. +`frontend -> common`, `backend -> common`, `exporter -> common`, and `frontend -> render-wasm`. Changes in `common` can affect frontend, backend, exporter, file migrations, and design-library behavior; validate across consumers when semantics change. # Working with Penpot designs @@ -101,6 +72,4 @@ semantics change. ## Detecting Crashes -The Penpot frontend can crash silently from the JS API's perspective: `execute_code` calls return successfully, but 1-2s later the workspace becomes unusable (Internal Error page). -The `execute_code` tool then stops working, but `cljs_repl` still works. Use it to detect a crash via `(some? (:exception @app.main.store/state))`. -For details on handling crashes, read memory `mem:frontend/handling-crashes`. +The Penpot frontend can crash silently from the JS API's perspective: `execute_code` calls return successfully, but 1-2s later the workspace becomes unusable (Internal Error page). The `execute_code` tool then stops working, but `cljs_repl` still works. Use it to detect a crash via `(some? (:exception @app.main.store/state))`. For details on handling crashes, read memory `mem:frontend/handling-crashes`. diff --git a/.serena/memories/memory-maintenance.md b/.serena/memories/memory-maintenance.md index 81cfa9aaf3..fbb800b99c 100644 --- a/.serena/memories/memory-maintenance.md +++ b/.serena/memories/memory-maintenance.md @@ -21,6 +21,7 @@ Dense agent notes, not prose docs. Prefer invariants, terse bullets. Avoid obvious context, rationale, and examples unless they prevent likely mistakes. Keep guidance durable and generalizable, not task-local. +No manual line wrapping: one line per bullet or paragraph, however long. Memories render adapting to the viewport; hard-wrapped lines degrade rendering and diffs. ## Add/update threshold diff --git a/backend/deps.edn b/backend/deps.edn index 5963271594..169fdf6529 100644 --- a/backend/deps.edn +++ b/backend/deps.edn @@ -28,8 +28,8 @@ com.google.guava/guava {:mvn/version "33.7.1-jre"} funcool/yetti - {:git/tag "v11.10" - :git/sha "88701f4" + {:git/tag "v11.11" + :git/sha "e810f87" :git/url "https://github.com/funcool/yetti.git" :exclusions [org.slf4j/slf4j-api]} diff --git a/backend/src/app/http.clj b/backend/src/app/http.clj index e991fd9849..b63ccf9a26 100644 --- a/backend/src/app/http.clj +++ b/backend/src/app/http.clj @@ -27,11 +27,16 @@ [app.rpc :as-alias rpc] [app.setup :as-alias setup] [integrant.core :as ig] + [promesa.exec :as px] [reitit.core :as r] [reitit.middleware :as rr] [yetti.adapter :as yt] [yetti.request :as yreq] - [yetti.response :as-alias yres])) + [yetti.response :as-alias yres]) + (:import + io.undertow.server.ConnectorStatistics + io.undertow.Undertow + org.xnio.management.XnioWorkerMXBean)) (declare router-handler) @@ -45,6 +50,97 @@ ::max-body-size 367001600 ; default 350 MiB }) +(def ^:private metrics-sample-interval-ms 15000) + +(defn sample-worker-metrics + "Publishes the current state of the xnio worker thread pool (the + request dispatch queue and its threads) as gauges." + [metrics ^XnioWorkerMXBean mxbean] + (let [queue-size (.getWorkerQueueSize mxbean) + busy-count (.getBusyWorkerThreadCount mxbean) + pool-size (.getWorkerPoolSize mxbean) + max-size (.getMaxWorkerPoolSize mxbean)] + + ;; negative values are missing measurements, not zeros: the xnio + ;; MXBean may transiently report -1 on the busy thread count. + (when (>= queue-size 0) + (mtx/run! metrics :id :http-worker-queue-size :val queue-size)) + + (when (>= busy-count 0) + (mtx/run! metrics :id :http-worker-busy-threads :val busy-count)) + + (when (>= pool-size 0) + (mtx/run! metrics :id :http-worker-pool-size :val pool-size)) + + (when (>= max-size 0) + (mtx/run! metrics :id :http-worker-max-pool-size :val max-size)))) + +(defn sample-connector-metrics + "Publishes the current state of the http listener connection + statistics. Undertow exposes absolute totals, so counters are + published as deltas of the last seen values (the atom state holds the + last observed totals). When a delta comes back negative (mainly + because the underlying counters were reset) the counter is skipped + and the reference updated." + [metrics state ^ConnectorStatistics cs] + (let [{:keys [last-requests last-errors]} (deref state) + total-requests (.getRequestCount cs) + total-errors (.getErrorCount cs) + delta-requests (max 0 (- total-requests last-requests)) + delta-errors (max 0 (- total-errors last-errors))] + + (when (pos? delta-requests) + (mtx/run! metrics :id :http-connector-requests-total :inc delta-requests)) + + (when (pos? delta-errors) + (mtx/run! metrics :id :http-connector-errors-total :inc delta-errors)) + + (mtx/run! metrics + :id :http-connector-active-connections + :val (.getActiveConnections cs)) + + (swap! state merge {:last-requests total-requests + :last-errors total-errors}))) + +(defn sample-http-metrics + "Samples the current state of the http server: worker thread pool + state and listener connection statistics. Called periodically by a + sampler that starts together with the server." + [metrics state ^Undertow server] + (try + (when-let [mxbean (some-> server (.getWorker) (.getMXBean))] + (sample-worker-metrics metrics mxbean)) + + (when-let [cs (some-> server (.getListenerInfo) (first) (.getConnectorStatistics))] + (sample-connector-metrics metrics state cs)) + + (catch Exception cause + (l/warn :msg "unexpected error on http metrics sampling" + :cause cause)))) + +(defn create-metrics-sampler + "Creates a daemon scheduler that periodically samples the state of + the http server and publishes it as metrics. A single thread is used, + and an unexpected error on a single sample does NOT cancel the + subsequent runs." + [^Undertow server metrics] + (let [state (atom {:last-requests 0 :last-errors 0}) + scheduler (px/scheduled-executor + :parallelism 1 + :factory (px/thread-factory :prefix "penpot/http-metrics/" + :daemon true)) + sample (fn sample [] + (try + (sample-http-metrics metrics state server) + (finally + ;; reschedule even if a single sample fails, so + ;; an unexpected error does not cancel the + ;; following runs. + (px/schedule scheduler metrics-sample-interval-ms sample))))] + + (px/schedule scheduler 0 sample) + scheduler)) + (defmethod ig/expand-key ::server [k v] {k (merge default-params (d/without-nils v))}) @@ -83,6 +179,7 @@ :xnio/io-threads (::io-threads cfg) :xnio/max-worker-threads (::max-worker-threads cfg) :ring/compat :ring2 + :server/statistics true :events/on-dispatch on-dispatch :socket/backlog 4069} @@ -98,13 +195,17 @@ (throw (UnsupportedOperationException. "handler or router are required"))) server - (yt/server handler (d/without-nils options))] + (yt/start! (yt/server handler (d/without-nils options))) - (assoc cfg ::server (yt/start! server)))) + sampler + (create-metrics-sampler server metrics)] + + (assoc cfg ::server server ::metrics-sampler sampler))) (defmethod ig/halt-key! ::server - [_ {:keys [::server ::port] :as cfg}] + [_ {:keys [::metrics-sampler ::server ::port] :as cfg}] (l/info :msg "stopping http server" :port port) + (px/shutdown-now metrics-sampler) (yt/stop! server)) (defn- not-found-handler diff --git a/backend/src/app/main.clj b/backend/src/app/main.clj index cc627f3307..d25ace7128 100644 --- a/backend/src/app/main.clj +++ b/backend/src/app/main.clj @@ -145,7 +145,42 @@ {::mdef/name "penpot_http_server_dispatch_timing" ::mdef/help "Histogram of dispatch handler" ::mdef/labels [] - ::mdef/type :histogram}}) + ::mdef/type :histogram} + + :http-worker-queue-size + {::mdef/name "penpot_http_worker_queue_size" + ::mdef/help "Current number of queued tasks in the http server xnio worker." + ::mdef/type :gauge} + + :http-worker-busy-threads + {::mdef/name "penpot_http_worker_busy_threads" + ::mdef/help "Current number of busy threads in the http server xnio worker." + ::mdef/type :gauge} + + :http-worker-pool-size + {::mdef/name "penpot_http_worker_pool_size" + ::mdef/help "Current number of threads in the http server xnio worker pool." + ::mdef/type :gauge} + + :http-worker-max-pool-size + {::mdef/name "penpot_http_worker_max_pool_size" + ::mdef/help "Maximum number of threads of the http server xnio worker pool." + ::mdef/type :gauge} + + :http-connector-active-connections + {::mdef/name "penpot_http_connector_active_connections" + ::mdef/help "Current number of active connections in the http listener." + ::mdef/type :gauge} + + :http-connector-requests-total + {::mdef/name "penpot_http_connector_requests_total" + ::mdef/help "Total number of requests handled by the http listener." + ::mdef/type :counter} + + :http-connector-errors-total + {::mdef/name "penpot_http_connector_errors_total" + ::mdef/help "Total number of handler errors in the http listener." + ::mdef/type :counter}}) (def system-config {::db/pool diff --git a/backend/src/app/srepl/main.clj b/backend/src/app/srepl/main.clj index fdc0caea1a..ab1987239a 100644 --- a/backend/src/app/srepl/main.clj +++ b/backend/src/app/srepl/main.clj @@ -144,6 +144,66 @@ (db/get-update-count) (pos?))))))) +(defn parse-emails + "Parse the emails into a seq of cleaned emails. Accepts a single + email, a comma separated list of emails or a coll of emails. + Blank entries are skipped." + [emails] + (->> (cond + (string? emails) + (str/split emails #",") + + (sequential? emails) + emails + + :else + (throw (ex-info "expected email or comma separated list of emails" + {:emails emails}))) + (map str/trim) + (remove str/empty?))) + +(defn- delete-profile-by-email* + [system email deleted-at cause] + (when-let [profile (some-> (db/get* system :profile + {:email (str/lower email)} + {::db/remove-deleted false}) + (profile/decode-row))] + (audit/insert system + {:name "delete-profile" + :type "action" + :profile-id (:id profile) + :tracked-at deleted-at + :props (audit/profile->props profile) + :context {:triggered-by "srepl" + :cause cause}}) + + (wrk/invoke! (-> system + (assoc ::wrk/task :delete-object) + (assoc ::wrk/params {:object :profile + :deleted-at deleted-at + :id (:id profile)}))) + (:id profile))) + +(defn delete-profiles-by-email! + "Mark profiles for deletion by email. Accepts a single email or a + comma separated list of emails (or a coll of emails). + + The deletion is immediate: the deleted-at is backdated with the + configured deletion-delay so the profiles and their owned teams are + purged on the next gc pass." + [emails] + (let [emails (parse-emails emails) + deleted-at (ct/minus (ct/now) (cf/get-deletion-delay)) + cause "explicit call to delete-profiles-by-email!"] + (db/tx-run! sys/system + (fn [system] + (reduce (fn [acc email] + (if-let [id (delete-profile-by-email* system email deleted-at cause)] + (update acc :deleted conj id) + (update acc :not-found conj email))) + {:total (count emails) :deleted [] :not-found []} + emails))))) + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; FEATURES ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -745,47 +805,24 @@ (defn delete-profiles-in-bulk! [system path] - (letfn [(process-data! [system deleted-at emails] - (loop [emails emails - deleted 0 - total 0] - (if-let [email (first emails)] - (if-let [profile (some-> (db/get* system :profile - {:email (str/lower email)} - {::db/remove-deleted false}) - (profile/decode-row))] - (do - (audit/insert system - {:name "delete-profile" - :type "action" - :profile-id (:id profile) - :tracked-at deleted-at - :props (audit/profile->props profile) - :context {:triggered-by "srepl" - :cause "explicit call to delete-profiles-in-bulk!"}}) - (wrk/invoke! (-> system - (assoc ::wrk/task :delete-object) - (assoc ::wrk/params {:object :profile - :deleted-at deleted-at - :id (:id profile)}))) - (recur (rest emails) - (inc deleted) - (inc total))) - (recur (rest emails) - deleted - (inc total))) - {:deleted deleted :total total})))] + (let [path (fs/path path) + deleted-at (ct/minus (ct/now) (cf/get-deletion-delay)) + cause "explicit call to delete-profiles-in-bulk!"] - (let [path (fs/path path) - deleted-at (ct/minus (ct/now) (cf/get-deletion-delay))] + (when-not (fs/exists? path) + (throw (ex-info "path does not exists" {:path path}))) - (when-not (fs/exists? path) - (throw (ex-info "path does not exists" {:path path}))) - - (db/tx-run! system - (fn [system] - (with-open [reader (io/reader path)] - (process-data! system deleted-at (line-seq reader)))))))) + (db/tx-run! system + (fn [system] + (with-open [reader (io/reader path)] + (loop [emails (line-seq reader) + deleted 0 + total 0] + (if-let [email (first emails)] + (if (delete-profile-by-email* system email deleted-at cause) + (recur (rest emails) (inc deleted) (inc total)) + (recur (rest emails) deleted (inc total))) + {:deleted deleted :total total}))))))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; CASCADE FIXING diff --git a/backend/test/backend_tests/http_metrics_test.clj b/backend/test/backend_tests/http_metrics_test.clj new file mode 100644 index 0000000000..4e56b67f22 --- /dev/null +++ b/backend/test/backend_tests/http_metrics_test.clj @@ -0,0 +1,257 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns backend-tests.http-metrics-test + (:require + [app.http :as http] + [app.metrics :as mtx] + [backend-tests.helpers :as th] + [clojure.test :as t] + [promesa.exec :as px]) + (:import + io.prometheus.client.CollectorRegistry + io.prometheus.client.Counter + io.prometheus.client.Gauge + io.undertow.server.ConnectorStatistics + java.util.concurrent.ScheduledThreadPoolExecutor + org.xnio.management.XnioWorkerMXBean)) + +(t/use-fixtures :once th/state-init) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Helpers +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(def metric-definitions + {:http-worker-queue-size {:name "penpot_http_worker_queue_size" + :help "test" + :type :gauge} + :http-worker-busy-threads {:name "penpot_http_worker_busy_threads" + :help "test" + :type :gauge} + :http-worker-pool-size {:name "penpot_http_worker_pool_size" + :help "test" + :type :gauge} + :http-worker-max-pool-size {:name "penpot_http_worker_max_pool_size" + :help "test" + :type :gauge} + :http-connector-active-connections + {:name "penpot_http_connector_active_connections" + :help "test" + :type :gauge} + :http-connector-requests-total + {:name "penpot_http_connector_requests_total" + :help "test" + :type :counter} + :http-connector-errors-total + {:name "penpot_http_connector_errors_total" + :help "test" + :type :counter}}) + +(defn- fake-metrics + "Builds a minimal IMetrics instance backed by real prometheus + collectors on a private registry." + [] + (let [registry (CollectorRegistry.) + collectors + (into {} + (map (fn [[id {:keys [name help type]}]] + (let [builder (case type + :gauge (Gauge/build) + :counter (Counter/build))] + (doto builder + (.name name) + (.help help)) + [id {:app.metrics.definition/type type + :app.metrics.definition/instance + (.register builder registry)}]))) + metric-definitions)] + + (reify app.metrics.IMetrics + (get-registry [_] registry) + (get-collector [_ id] (get collectors id)) + (get-handler [_] nil)))) + +(defn- gauge-value + [^Gauge collector] + (.get (.labels collector (make-array String 0)))) + +(defn- counter-value + [^Counter collector] + (.get (.labels collector (make-array String 0)))) + +(defn- fake-mxbean + [{:keys [queue busy pool max] + :or {queue 0 busy 0 pool 4 max 512}}] + (reify XnioWorkerMXBean + (getProviderName [_] "test") + (getName [_] "test") + (isShutdownRequested [_] false) + (getCoreWorkerPoolSize [_] 32) + (getMaxWorkerPoolSize [_] max) + (getWorkerPoolSize [_] pool) + (getBusyWorkerThreadCount [_] busy) + (getIoThreadCount [_] 16) + (getWorkerQueueSize [_] queue) + (getServerMXBeans [_] #{}))) + +(defn- fake-connector-statistics + [{:keys [requests errors active] + :or {requests 0 errors 0 active 0}}] + (reify ConnectorStatistics + (getRequestCount [_] requests) + (getBytesSent [_] 0) + (getBytesReceived [_] 0) + (getErrorCount [_] errors) + (getProcessingTime [_] 0) + (getMaxProcessingTime [_] 0) + (getActiveConnections [_] active) + (getMaxActiveConnections [_] active) + (getActiveRequests [_] 0) + (getMaxActiveRequests [_] 0) + (reset [_] nil))) + +(defn- collector-instance + [metrics id] + (:app.metrics.definition/instance (mtx/get-collector metrics id))) + +(defn- make-state [] + (atom {:last-requests 0 :last-errors 0})) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Test: worker metrics +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(t/deftest sample-worker-metrics-publish-all-gauges + (let [metrics (fake-metrics) + mxbean (fake-mxbean {:queue 2 :busy 10 :pool 40 :max 512})] + + (http/sample-worker-metrics metrics mxbean) + + (t/is (= 2.0 (gauge-value (collector-instance metrics :http-worker-queue-size)))) + (t/is (= 10.0 (gauge-value (collector-instance metrics :http-worker-busy-threads)))) + (t/is (= 40.0 (gauge-value (collector-instance metrics :http-worker-pool-size)))) + (t/is (= 512.0 (gauge-value (collector-instance metrics :http-worker-max-pool-size)))))) + +(t/deftest sample-worker-metrics-skips-negative-samples + ;; the xnio MXBean occasionally returns -1 on the busy thread count; + ;; a negative value is a missing measurement, not a zero. + (let [metrics (fake-metrics) + mxbean (fake-mxbean {:queue 0 :busy -1 :pool 4 :max 512})] + + (http/sample-worker-metrics metrics mxbean) + + (t/is (= 0.0 (gauge-value (collector-instance metrics :http-worker-queue-size)))) + (t/is (= 4.0 (gauge-value (collector-instance metrics :http-worker-pool-size)))) + (t/is (= 512.0 (gauge-value (collector-instance metrics :http-worker-max-pool-size)))) + (t/is (= 0.0 + (gauge-value (collector-instance metrics :http-worker-busy-threads)))))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Test: connector metrics +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(t/deftest sample-connector-metrics-publishes-gauges-and-counters + (let [metrics (fake-metrics) + state (make-state) + cs (fake-connector-statistics {:requests 10 :errors 2 :active 5})] + + (http/sample-connector-metrics metrics state cs) + + (t/is (= 5.0 (gauge-value (collector-instance metrics :http-connector-active-connections)))) + (t/is (= 10.0 (counter-value (collector-instance metrics :http-connector-requests-total)))) + (t/is (= 2.0 (counter-value (collector-instance metrics :http-connector-errors-total)))))) + +(t/deftest sample-connector-metrics-accumulates-delta + (let [metrics (fake-metrics) + state (make-state)] + + (http/sample-connector-metrics metrics state (fake-connector-statistics {:requests 10 :errors 0 :active 1})) + (http/sample-connector-metrics metrics state (fake-connector-statistics {:requests 25 :errors 0 :active 1})) + + (t/is (= 25.0 (counter-value (collector-instance metrics :http-connector-requests-total)))) + (t/is (= 0.0 (counter-value (collector-instance metrics :http-connector-errors-total)))))) + +(t/deftest sample-connector-metrics-skips-negative-delta + ;; when the undertow counters are reset, the computed delta can go + ;; negative: the counter must not decrease, and the reference must be + ;; updated so the next sampling continues from the new watermark. + (let [metrics (fake-metrics) + state (make-state)] + + (http/sample-connector-metrics metrics state (fake-connector-statistics {:requests 20 :errors 5 :active 0})) + (http/sample-connector-metrics metrics state (fake-connector-statistics {:requests 10 :errors 3 :active 0})) + (http/sample-connector-metrics metrics state (fake-connector-statistics {:requests 15 :errors 6 :active 0})) + + (t/is (= 25.0 (counter-value (collector-instance metrics :http-connector-requests-total)))) + (t/is (= 8.0 (counter-value (collector-instance metrics :http-connector-errors-total)))))) + +(t/deftest sample-connector-metrics-state-advances-with-reset + ;; after a reset (total decreased) followed by more requests, the + ;; next delta must be computed from the new watermark and count only + ;; the requests after the reset. + (let [metrics (fake-metrics) + state (make-state)] + + (http/sample-connector-metrics metrics state (fake-connector-statistics {:requests 10 :errors 0 :active 0})) + (http/sample-connector-metrics metrics state (fake-connector-statistics {:requests 5 :errors 0 :active 0})) ; reset to 5 + (http/sample-connector-metrics metrics state (fake-connector-statistics {:requests 8 :errors 0 :active 0})) + + (t/is (= 13.0 (counter-value (collector-instance metrics :http-connector-requests-total)))))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Test: sampler lifecycle +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(t/deftest sample-http-metrics-on-nil-server-does-nothing + ;; the guards live at the boundary: without a server there is nothing + ;; to sample and nothing must be published. + (let [metrics (fake-metrics) + state (make-state)] + + (http/sample-http-metrics metrics state nil) + + (t/is (= 0.0 (gauge-value (collector-instance metrics :http-worker-queue-size)))) + (t/is (= 0.0 + (counter-value (collector-instance metrics :http-connector-requests-total)))))) + +(t/deftest create-metrics-sampler-lifecycle + ;; a smoke test of the lifecycle wiring: the sampler is created with + ;; a running scheduler and ends up shut down. + (let [metrics (fake-metrics) + sampler (http/create-metrics-sampler nil metrics)] + (try + (t/is (some? sampler)) + (t/is (px/executor? sampler)) + (t/is (not (px/shutdown? sampler))) + (finally + (px/shutdown-now sampler) + (t/is (px/shutdown? sampler)))))) + +(t/deftest create-metrics-sampler-reschedules-after-error + ;; the docstring promise: an unexpected error on a single sample must + ;; not cancel the following runs. The first sample runs immediately + ;; and throws; the next one must still be scheduled afterwards. + (let [calls (atom 0)] + (with-redefs [http/sample-http-metrics (fn [_ _ _] + (swap! calls inc) + (throw (ex-info "boom" {})))] + (let [sampler (http/create-metrics-sampler nil (fake-metrics)) + queue (.getQueue ^ScheduledThreadPoolExecutor sampler)] + (try + (t/is (loop [i 0] + (cond (pos? @calls) true + (> i 200) false + :else (do (Thread/sleep 10) (recur (inc i))))) + "the first sample must run immediately") + + (t/is (loop [i 0] + (cond (= 1 (.size queue)) true + (> i 200) false + :else (do (Thread/sleep 10) (recur (inc i))))) + "the next sample must be scheduled after the error") + (finally + (px/shutdown-now sampler))))))) diff --git a/backend/test/backend_tests/metrics_test.clj b/backend/test/backend_tests/metrics_test.clj new file mode 100644 index 0000000000..f447ca05e5 --- /dev/null +++ b/backend/test/backend_tests/metrics_test.clj @@ -0,0 +1,49 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns backend-tests.metrics-test + (:require + [app.metrics :as mtx] + [clojure.test :as t] + [integrant.core :as ig]) + (:import + io.prometheus.client.Collector$MetricFamilySamples + io.prometheus.client.Collector$MetricFamilySamples$Sample)) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Helpers +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defn- sample-names + [metrics] + (->> (mtx/get-registry metrics) + (.metricFamilySamples) + (enumeration-seq) + (mapcat (fn [^Collector$MetricFamilySamples family] + (map (fn [^Collector$MetricFamilySamples$Sample sample] + (.-name sample)) + (.samples family)))) + (set))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Tests +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(t/deftest process-metrics-are-exported + ;; the process cpu and file descriptor families come from the + ;; prometheus client `StandardExports`, registered by `app.metrics`. + ;; They are read reflectively from the OS MXBean and depend on the + ;; `jdk.management` module at runtime: a pruned jlink JRE turns the + ;; MXBean into `sun.management.BaseOperatingSystemImpl`, the reflective + ;; getters fail and the families are silently dropped (that is how the + ;; production backend lost `process_open_fds`). This test pins the + ;; contract the fd alert relies on. + (let [metrics (ig/init-key :app.metrics/metrics {:default {}}) + names (sample-names metrics)] + + (t/is (contains? names "process_open_fds")) + (t/is (contains? names "process_max_fds")) + (t/is (contains? names "process_cpu_seconds_total")))) diff --git a/backend/test/backend_tests/srepl_main_test.clj b/backend/test/backend_tests/srepl_main_test.clj new file mode 100644 index 0000000000..9897415d19 --- /dev/null +++ b/backend/test/backend_tests/srepl_main_test.clj @@ -0,0 +1,28 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns backend-tests.srepl-main-test + (:require + [app.srepl.main :as srepl] + [clojure.test :as t])) + +(t/deftest parse-emails + (t/is (= ["some@example.com"] + (srepl/parse-emails "some@example.com"))) + + (t/is (= ["some@example.com" "other@example.com"] + (srepl/parse-emails "some@example.com,other@example.com"))) + + (t/is (= ["some@example.com" "other@example.com"] + (srepl/parse-emails " some@example.com , other@example.com ,"))) + + (t/is (= ["some@example.com" "other@example.com"] + (srepl/parse-emails ["some@example.com" "other@example.com"]))) + + (t/is (= [] (srepl/parse-emails ","))) + + (t/is (thrown? clojure.lang.ExceptionInfo + (srepl/parse-emails 42)))) diff --git a/docker/devenv/files/nginx.conf b/docker/devenv/files/nginx.conf index 3d3d70263f..b4d8829567 100644 --- a/docker/devenv/files/nginx.conf +++ b/docker/devenv/files/nginx.conf @@ -31,7 +31,16 @@ http { include /etc/nginx/mime.types; default_type application/octet-stream; - access_log /var/log/nginx/access.log; + # Access log enriched with upstream diagnostics: what the backend + # answered (us), how long it took to connect to it (uct), how long it + # took to answer (urt) and which internal address served the request + # (ua). With proxy_pass on variables there is no upstream keepalive, + # so uct measures one real TCP connection per request. + log_format penpot_upstream '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" "$http_user_agent" ' + 'us=$upstream_status uct=$upstream_connect_time ' + 'urt=$upstream_response_time ua=$upstream_addr'; + access_log /var/log/nginx/access.log penpot_upstream; error_log /var/log/nginx/error.log; gzip on; @@ -297,4 +306,20 @@ http { try_files $uri /index.html$is_args$args /index.html =404; } } + + # Dedicated health endpoint for the optional nginx-prometheus-exporter + # sidecar (scraping http://127.0.0.1:8082/stub_status). Bound to + # localhost only and out of the public server, so it can not be + # reached from outside the host. Counts client-side connections only; + # it says nothing about the upstream pools. + server { + listen 127.0.0.1:8082; + server_name _; + + access_log off; + + location = /stub_status { + stub_status; + } + } } diff --git a/docker/images/Dockerfile.backend b/docker/images/Dockerfile.backend index 19c2198ce8..c5b16e9756 100644 --- a/docker/images/Dockerfile.backend +++ b/docker/images/Dockerfile.backend @@ -18,6 +18,11 @@ RUN set -ex; \ apt-get clean; \ rm -rf /var/lib/apt/lists/* +# NOTE: jdk.management is required by the prometheus client hotspot +# exports. Without it the OS MXBean is sun.management.BaseOperatingSystemImpl, +# which has no getOpenFileDescriptorCount, getMaxFileDescriptorCount nor +# getProcessCpuTime, so process_open_fds, process_max_fds and +# process_cpu_seconds_total silently disappear from /metrics. RUN set -eux; \ ARCH="$(dpkg --print-architecture)"; \ case "${ARCH}" in \ @@ -44,7 +49,7 @@ RUN set -eux; \ --no-header-files \ --no-man-pages \ --strip-debug \ - --add-modules java.base,jdk.net,jdk.management.agent,java.se,jdk.compiler,jdk.javadoc,jdk.attach,jdk.unsupported,jdk.jfr,jdk.jcmd \ + --add-modules java.base,jdk.net,jdk.management,jdk.management.agent,java.se,jdk.compiler,jdk.javadoc,jdk.attach,jdk.unsupported,jdk.jfr,jdk.jcmd \ --output /opt/jre; diff --git a/docker/images/files/nginx.conf.template b/docker/images/files/nginx.conf.template index 75a385f9d6..7c4d92dbb8 100644 --- a/docker/images/files/nginx.conf.template +++ b/docker/images/files/nginx.conf.template @@ -31,7 +31,17 @@ http { default_type application/octet-stream; error_log /dev/stderr; - access_log /dev/stdout; + + # Access log enriched with upstream diagnostics: what the backend + # answered (us), how long it took to connect to it (uct), how long it + # took to answer (urt) and which internal address served the request + # (ua). With proxy_pass on variables there is no upstream keepalive, + # so uct measures one real TCP connection per request. + log_format penpot_upstream '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" "$http_user_agent" ' + 'us=$upstream_status uct=$upstream_connect_time ' + 'urt=$upstream_response_time ua=$upstream_addr'; + access_log /dev/stdout penpot_upstream; proxy_connect_timeout 300s; proxy_send_timeout 300s; @@ -186,4 +196,20 @@ http { } } + + # Dedicated health endpoint consumed by the nginx-prometheus-exporter + # sidecar (deployed by ops, scraping http://127.0.0.1:8082/stub_status). + # Bound to localhost only and out of the public server, so it can not + # be reached from outside the host. Counts client-side connections + # only; it says nothing about the upstream pools. + server { + listen 127.0.0.1:8082; + server_name _; + + access_log off; + + location = /stub_status { + stub_status; + } + } } From de6fb9d13ea604cc2f3197f170c136caaa1cc30e Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 23 Sep 2026 12:44:37 +0000 Subject: [PATCH 19/22] :wrench: Drop loopback bind on the stub_status endpoint A compose port mapping delivers traffic to the container address, never to loopback, so `listen 127.0.0.1:8082` made `ports: :8082` fail from the host. Both configs (image template and devenv) now use `listen 8082`, which binds every interface and matches the implicit bind of the public `listen 8080 default_server`. Rewrite both block comments to state the new bind and who decides access from outside the host. The scrape URI stays on 127.0.0.1:8082: it still reaches the socket. AI-assisted-by: mimo-v2.6-flash-free --- docker/devenv/files/nginx.conf | 12 +++++++----- docker/images/files/nginx.conf.template | 10 ++++++---- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/docker/devenv/files/nginx.conf b/docker/devenv/files/nginx.conf index b4d8829567..46246e5153 100644 --- a/docker/devenv/files/nginx.conf +++ b/docker/devenv/files/nginx.conf @@ -308,12 +308,14 @@ http { } # Dedicated health endpoint for the optional nginx-prometheus-exporter - # sidecar (scraping http://127.0.0.1:8082/stub_status). Bound to - # localhost only and out of the public server, so it can not be - # reached from outside the host. Counts client-side connections only; - # it says nothing about the upstream pools. + # sidecar (scraping http://127.0.0.1:8082/stub_status). `listen 8082` + # binds every interface and sits out of the public server: a compose + # port mapping delivers traffic to the container address, never to + # loopback, so only the published port decides who gets in from outside + # the host. Counts client-side connections only; it says nothing about + # the upstream pools. server { - listen 127.0.0.1:8082; + listen 8082; server_name _; access_log off; diff --git a/docker/images/files/nginx.conf.template b/docker/images/files/nginx.conf.template index 7c4d92dbb8..8e3b21d7e1 100644 --- a/docker/images/files/nginx.conf.template +++ b/docker/images/files/nginx.conf.template @@ -199,11 +199,13 @@ http { # Dedicated health endpoint consumed by the nginx-prometheus-exporter # sidecar (deployed by ops, scraping http://127.0.0.1:8082/stub_status). - # Bound to localhost only and out of the public server, so it can not - # be reached from outside the host. Counts client-side connections - # only; it says nothing about the upstream pools. + # `listen 8082` binds every interface and sits out of the public server: + # a compose port mapping delivers traffic to the container address, never + # to loopback, so only the published port decides who gets in from + # outside the host. Counts client-side connections only; it says nothing + # about the upstream pools. server { - listen 127.0.0.1:8082; + listen 8082; server_name _; access_log off; From 58b1acbb66d190beff9c626873439073cfe06ec4 Mon Sep 17 00:00:00 2001 From: Luis de Dios Date: Wed, 23 Sep 2026 18:18:14 +0200 Subject: [PATCH 20/22] :bug: Fix register flow from workspace url (#11756) * :bug: Fix show validation errors below inputs instead of showing a toast * :bug: Fix redirect to send email screen after a successful registration --- frontend/src/app/main/ui/auth/register.cljs | 41 ++++++++++----------- frontend/src/app/main/ui/static.cljs | 17 +++++++-- frontend/src/app/main/ui/viewer/login.cljs | 14 ++++++- 3 files changed, 44 insertions(+), 28 deletions(-) diff --git a/frontend/src/app/main/ui/auth/register.cljs b/frontend/src/app/main/ui/auth/register.cljs index 1b8fc40f10..b8130e8361 100644 --- a/frontend/src/app/main/ui/auth/register.cljs +++ b/frontend/src/app/main/ui/auth/register.cljs @@ -21,7 +21,6 @@ [app.util.i18n :as i18n :refer [tr]] [app.util.storage :as storage] [beicon.v2.core :as rx] - [cuerdas.core :as str] [rumext.v2 :as mf])) ;; --- PAGE: Register @@ -81,42 +80,40 @@ (mf/use-fn (fn [cause] (reset! submitted? false) - (let [{:keys [type code] :as edata} (ex-data cause)] + (let [{:keys [type code] :as edata} (ex-data cause) + + set-field-error! + (fn [field message] + (swap! form assoc-in [:extra-errors field] message) + (swap! form assoc-in [:touched field] true))] + (condp = [type code] [:restriction :email-does-not-match-invitation] - (st/emit! (ntf/error (tr "errors.email-does-not-match-invitation"))) + (set-field-error! :email {:message (tr "errors.email-does-not-match-invitation")}) [:restriction :registration-disabled] - (st/emit! (ntf/error (tr "errors.registration-disabled"))) + (set-field-error! :email {:message (tr "errors.registration-disabled")}) [:restriction :email-domain-is-not-allowed] - (st/emit! (ntf/error (tr "errors.email-domain-not-allowed"))) + (set-field-error! :email {:message (tr "errors.email-domain-not-allowed")}) [:restriction :email-has-permanent-bounces] - (st/emit! (ntf/error (tr "errors.email-has-permanent-bounces" (:email edata)))) + (set-field-error! :email {:message (tr "errors.email-has-permanent-bounces" (:email edata))}) [:restriction :email-has-complaints] - (st/emit! (ntf/error (tr "errors.email-has-permanent-bounces" (:email edata)))) + (set-field-error! :email {:message (tr "errors.email-has-permanent-bounces" (:email edata))}) [:validation :email-already-exists] - (st/emit! (ntf/error (tr "errors.email-already-exists"))) + (set-field-error! :email {:message (tr "errors.email-already-exists")}) [:validation :email-as-password] - (st/emit! (ntf/error (tr "errors.email-as-password"))) + (set-field-error! :password {:message (tr "errors.email-as-password")}) [:validation :weak-password] - (let [details (:details edata) - items (when (seq details) - (->> details - (map #(str "
  • " (tr %) "
    • ")) - (str/join ""))) - detail (when items - (str "
      " items "
    "))] - (st/emit! (ntf/show {:content (tr "errors.weak-password") - :detail detail - :is-html true - :type :toast - :level :error}))) + (let [options (when (seq (:details edata)) + (mapv tr (:details edata)))] + (set-field-error! :password {:message (tr "errors.weak-password") + :options options})) (do (when-let [explain (get edata :explain)] @@ -128,7 +125,7 @@ (mf/deps on-success-callback) (fn [params] (if (fn? on-success-callback) - (on-success-callback (:email params)) + (on-success-callback params) (cond (some? (:invitation-token params)) (let [token (:invitation-token params)] diff --git a/frontend/src/app/main/ui/static.cljs b/frontend/src/app/main/ui/static.cljs index dfa1bacfce..b9174d7172 100644 --- a/frontend/src/app/main/ui/static.cljs +++ b/frontend/src/app/main/ui/static.cljs @@ -125,13 +125,22 @@ success-register (mf/use-fn (fn [data] - (reset! register-token (:token data)) - (reset! current-section :register-validate))) + (cond + (:invitation-token data) + (st/emit! (rt/nav :auth-verify-token {:token (:invitation-token data)})) + + (:is-active data) + (st/emit! (rt/reload true)) + + :else + (do + (reset! user-email (:email data)) + (reset! current-section :register-email-sent))))) register-email-sent (mf/use-fn - (fn [email] - (reset! user-email email) + (fn [data] + (reset! user-email (if (string? data) data (:email data))) (reset! current-section :register-email-sent))) recovery-email-sent diff --git a/frontend/src/app/main/ui/viewer/login.cljs b/frontend/src/app/main/ui/viewer/login.cljs index 6ede841b73..a0fc4c6bb5 100644 --- a/frontend/src/app/main/ui/viewer/login.cljs +++ b/frontend/src/app/main/ui/viewer/login.cljs @@ -9,6 +9,7 @@ (:require [app.common.logging :as log] [app.main.data.modal :as modal] + [app.main.router :as rt] [app.main.store :as st] [app.main.ui.auth.login :refer [login-dialog*]] [app.main.ui.auth.recovery-request :refer [recovery-request-page*]] @@ -68,8 +69,17 @@ success-register (fn [data] - (reset! register-token (:token data)) - (set-current-section :register-validate))] + (cond + (:invitation-token data) + (st/emit! (rt/nav :auth-verify-token {:token (:invitation-token data)})) + + (:is-active data) + (st/emit! (modal/hide)) + + :else + (do + (reset! user-email (:email data)) + (set-current-section :email-sent))))] [:div {:class (stl/css :modal-overlay)} [:div {:class (stl/css :modal-container)} From 992170a9a42694162bb64d73c5914e778a950ac2 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 23 Sep 2026 16:28:11 +0000 Subject: [PATCH 21/22] :bug: Remove OpenCode V1 plugin support Remove the legacy @opencode-ai/plugin import and V1 server export so the local plugin loads under OpenCode V2 without project dependencies. Add a focused smoke test for the V2 export and tool registration, and update the related script memories. AI-assisted-by: space-bunny-free --- .opencode/plugins/penpot.js | 85 ++---------------------- .opencode/test/penpot.test.js | 32 +++++++++ .serena/memories/scripts/paren-repair.md | 8 +-- .serena/memories/scripts/psql.md | 9 ++- 4 files changed, 45 insertions(+), 89 deletions(-) create mode 100644 .opencode/test/penpot.test.js diff --git a/.opencode/plugins/penpot.js b/.opencode/plugins/penpot.js index c231907396..c2e17c9f6b 100644 --- a/.opencode/plugins/penpot.js +++ b/.opencode/plugins/penpot.js @@ -1,22 +1,9 @@ -// Penpot opencode plugin: custom tools for Penpot development. +// Penpot OpenCode V2 plugin: custom tools for Penpot development. // -// Dual V1 + V2 implementation from a single file: -// - OpenCode V1 (>= 1.18.29) calls the default export's `server()` and uses -// the returned `tool` map (built with the `tool()` helper from -// `@opencode-ai/plugin`). -// - OpenCode V2 reads the default export's `id` and `setup()` and ignores -// `server()`. Tools are registered via `ctx.tool.transform()` with JSON -// Schema inputs, and `execute` returns `{ content }`. -// See https://opencode.ai/v2/docs/build/plugins/migrate-v1 -// -// NOTE: the V2 side intentionally does NOT -// `import { Plugin } from "@opencode/plugin"`. At runtime `Plugin.define` is -// the identity function, so a plain `{ id, setup }` object is equivalent, and -// skipping the import keeps this plugin dependency-free -// (`.opencode/package.json` is gitignored, so a new dependency declared there -// would not travel with this file). +// Tools are registered with `ctx.tool.transform()` and JSON Schema inputs. +// Keep this plugin dependency-free so the auto-discovered local plugin loads +// without project npm dependencies. -import { tool } from "@opencode-ai/plugin" import path from "path" import { spawn } from "child_process" @@ -106,64 +93,6 @@ function executeParenRepair({ files, code }, directory) { }) } -// --- V1 tool definitions (OpenCode V1 calls `server()` below) --- - -const penpotPsqlTool = tool({ - description: - "Execute a SQL command against the Penpot database. Uses the defaults from scripts/psql.", - - args: { - sql: tool.schema - .string() - .describe("SQL command to execute"), - - test: tool.schema - .boolean() - .describe("Use the penpot_test database") - .optional(), - }, - - async execute(args, context) { - return executePsql(args.sql, args.test === true, context.worktree) - }, -}) - -const parenRepairTool = tool({ - description: - "Fix mismatched parentheses/braces in Clojure files (.clj, .cljs, .cljc) then reformat with cljfmt.", - - args: { - // A string is used instead of an array so OpenCode displays it - // in the generic tool invocation. - files: tool.schema - .string() - .describe( - "Comma-separated file paths to fix, for example: frontend/src/app/config.cljs, backend/src/core.clj", - ) - .optional(), - - code: tool.schema - .string() - .describe("Code string to fix via stdin") - .optional(), - }, - - async execute(args, context) { - return executeParenRepair(args, context.worktree) - }, -}) - -async function server() { - return { - tool: { - "paren-repair": parenRepairTool, - "penpot-psql": penpotPsqlTool, - }, - } -} - -// --- V2 setup (OpenCode V2 calls `setup()` and ignores `server()`) --- - const penpotPsqlInputSchema = { type: "object", properties: { @@ -199,10 +128,7 @@ const parenRepairInputSchema = { } async function setup(ctx) { - // Plugin instance location. This is not the location of every session the - // tools may run for, but it is the closest V2 equivalent of the V1 - // per-execution `context.worktree` (the repo checkout the plugin loaded - // from), which is what both tools need as cwd / script base. + // Use the plugin instance location as the working directory for both tools. const directory = ctx.location.directory ?? ctx.location.project?.canonical @@ -240,5 +166,4 @@ async function setup(ctx) { export default { id: "penpot", setup, - server, } diff --git a/.opencode/test/penpot.test.js b/.opencode/test/penpot.test.js new file mode 100644 index 0000000000..c87d1c3540 --- /dev/null +++ b/.opencode/test/penpot.test.js @@ -0,0 +1,32 @@ +import assert from "node:assert/strict" +import test from "node:test" + +import plugin from "../plugins/penpot.js" + +test("exports only the OpenCode V2 plugin contract", () => { + assert.equal(plugin.id, "penpot") + assert.equal("server" in plugin, false) +}) + +test("registers the Penpot tools during setup", async () => { + const tools = [] + const context = { + location: { directory: "/tmp/opencode/penpot-plugin-test" }, + tool: { + async transform(apply) { + apply({ + add(tool) { + tools.push(tool) + }, + }) + }, + }, + } + + await plugin.setup(context) + + assert.deepEqual( + tools.map((tool) => tool.name).sort(), + ["paren-repair", "penpot-psql"], + ) +}) diff --git a/.serena/memories/scripts/paren-repair.md b/.serena/memories/scripts/paren-repair.md index 2774058fbc..a5c290ee60 100644 --- a/.serena/memories/scripts/paren-repair.md +++ b/.serena/memories/scripts/paren-repair.md @@ -27,11 +27,11 @@ bb scripts/paren-repair --help ``` `bb` must be invoked from the repo root so the path `scripts/paren-repair` resolves. -## Native Tool Available (opencode) +## Native Tool Available (OpenCode V2) -A native opencode tool `paren-repair` is available (defined in -`.opencode/plugins/penpot.js`, which registers it for both opencode V1 via -`server()` and opencode V2 via `setup()`). The LLM can call it directly with: +A native OpenCode V2 tool `paren-repair` is available. It is defined in +`.opencode/plugins/penpot.js` and registered through `setup()`. The LLM can +call it directly with: - `files`: comma-separated file paths to fix (a string, not an array) - `code`: Code string to fix via stdin diff --git a/.serena/memories/scripts/psql.md b/.serena/memories/scripts/psql.md index 68d7b66d0f..810905f7c6 100644 --- a/.serena/memories/scripts/psql.md +++ b/.serena/memories/scripts/psql.md @@ -25,12 +25,11 @@ scripts/psql --host myhost --user myuser --db mydb `scripts/psql` must be invoked from the repo root so the path resolves. -## Native Tool Available (opencode) +## Native Tool Available (OpenCode V2) -A native opencode tool `penpot-psql` is available (defined in -`.opencode/plugins/penpot.js`, which registers it for both opencode V1 via -`server()` and opencode V2 via `setup()`). The LLM can call it directly -with: +A native OpenCode V2 tool `penpot-psql` is available. It is defined in +`.opencode/plugins/penpot.js` and registered through `setup()`. The LLM can +call it directly with: - `sql`: SQL command string to execute - `test`: Boolean flag to use the `penpot_test` database From d6abd2fecd07da48f8a473578898cc09b4b6d3d0 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 23 Sep 2026 16:40:39 +0000 Subject: [PATCH 22/22] :wrench: Add explicit issue-to-PR linking command Add a link-issue command that creates GitHub's Development reference and verifies both sides. Keep Closes in descriptions for context, but make the API link the source of truth, including for merged PRs. Add tests for successful links, missing verification, output, and failures. Update the PR workflow memories and create-pr skill to use the command. AI-assisted-by: space-bunny-free --- .agents/skills/create-pr/SKILL.md | 18 +- .serena/memories/scripts/gh.md | 12 ++ .serena/memories/workflow/creating-commits.md | 2 +- .serena/memories/workflow/creating-issues.md | 14 +- .serena/memories/workflow/creating-prs.md | 8 +- scripts/gh.py | 147 +++++++++++++++++ scripts/test_gh.py | 155 ++++++++++++++++++ 7 files changed, 342 insertions(+), 14 deletions(-) create mode 100644 scripts/test_gh.py diff --git a/.agents/skills/create-pr/SKILL.md b/.agents/skills/create-pr/SKILL.md index f285574bb8..1f18900d0f 100644 --- a/.agents/skills/create-pr/SKILL.md +++ b/.agents/skills/create-pr/SKILL.md @@ -73,7 +73,8 @@ one. Title or description fixes belong to Update mode. Write the title and body following `mem:workflow/creating-prs` (title format, description structure, writing principles) and `mem:workflow/creating-commits` (commit type emojis). Derive the title and body from the commits and, when -there is one, from the issue body. Reference the issue with `Closes #NNNN`. +there is one, from the issue body. Include `Closes #NNNN` for readable context, +but do not treat it as the link; use `python3 scripts/gh.py link-issue ` after creating the PR. Repeat the `AI-assisted-by:` trailer in the body, once per model that worked on the branch, so the PR states the assistance where a reviewer reads it. The @@ -92,6 +93,13 @@ gh pr create --repo penpot/penpot --base "" --title "" \ repository default, which is wrong for a branch cut from `staging`. `--project "Main"` is required by `mem:workflow/creating-prs`. +If an issue is present, run the explicit assignment and verification command +from `mem:workflow/creating-prs` before reporting success: + +```bash +python3 scripts/gh.py link-issue <ISSUE_NUMBER> <PR_NUMBER> +``` + ### 5. Report Report the PR URL and stop. @@ -108,6 +116,14 @@ gh pr edit <NUMBER> --repo penpot/penpot --title "<TITLE>" --body-file /tmp/pr-b gh pr view <NUMBER> --repo penpot/penpot --json title,body ``` +If the updated body contains `Closes #NNNN`, run the explicit assignment +command from `mem:workflow/creating-prs` and require its verification to +succeed: + +```bash +python3 scripts/gh.py link-issue <ISSUE_NUMBER> <PR_NUMBER> +``` + 4. Report and stop. ## User context diff --git a/.serena/memories/scripts/gh.md b/.serena/memories/scripts/gh.md index 58f73e997d..ef061edd60 100644 --- a/.serena/memories/scripts/gh.md +++ b/.serena/memories/scripts/gh.md @@ -9,6 +9,7 @@ repository via GraphQL and REST APIs through the authenticated `gh` CLI. - Finding issues with no milestone. - Fetching PR details by number or by milestone. - Comparing milestone issues against CHANGES.md to find missing entries. +- Explicitly linking a GitHub issue to a pull request and verifying both sides. - Listing or inspecting GitHub Security Advisories (GHSA). ## Prerequisites @@ -73,6 +74,17 @@ python3 scripts/gh.py prs --milestone "2.16.0" --state all **Output**: JSON array to stdout; progress to stderr. +### `link-issue` + +Explicitly assign a GitHub issue to a pull request and verify the relationship from both sides: + +```bash +python3 scripts/gh.py link-issue <ISSUE_NUMBER> <PR_NUMBER> +# Short alias: python3 scripts/gh.py link <ISSUE_NUMBER> <PR_NUMBER> +``` + +The command resolves both node IDs, calls `addCloseIssueReferences`, and checks the issue's manually linked PRs and the PR's closing issue references. It is safe to rerun, works for merged PRs, and does not close an issue retroactively. JSON goes to stdout; progress and errors go to stderr; a missing link exits non-zero. + ### `advisories` List or inspect GitHub Security Advisories for the repository. diff --git a/.serena/memories/workflow/creating-commits.md b/.serena/memories/workflow/creating-commits.md index c3373ba29e..d632c9a471 100644 --- a/.serena/memories/workflow/creating-commits.md +++ b/.serena/memories/workflow/creating-commits.md @@ -50,4 +50,4 @@ that breaks them is wrong, even if the code is right. ## Referencing Issues -Use `Closes #NNNN` (not `Fixes #NNNN`) to link a commit to a GitHub issue. +Use `Closes #NNNN` (not `Fixes #NNNN`) in commit and PR body text for readable context. The explicit issue-to-PR link is assigned with `python3 scripts/gh.py link-issue <ISSUE_NUMBER> <PR_NUMBER>` as described in `mem:workflow/creating-prs`; do not rely on the keyword to create that link. diff --git a/.serena/memories/workflow/creating-issues.md b/.serena/memories/workflow/creating-issues.md index b1b4e9ade4..92f950a1b3 100644 --- a/.serena/memories/workflow/creating-issues.md +++ b/.serena/memories/workflow/creating-issues.md @@ -271,21 +271,13 @@ the issue was sourced. ### Link the PR to the issue -Append `Closes #<ISSUE_NUMBER>` to the PR body: +Add `Closes #<ISSUE_NUMBER>` to the PR body for readable context, then run the explicit assignment command from `mem:workflow/creating-prs`: ```bash -gh pr view <PR_NUMBER> --repo penpot/penpot --json body --jq '.body' > /tmp/pr-body.md -printf "\n\nCloses #<ISSUE_NUMBER>\n" >> /tmp/pr-body.md -gh pr edit <PR_NUMBER> --repo penpot/penpot --body-file /tmp/pr-body.md - -# Verify -gh pr view <PR_NUMBER> --repo penpot/penpot --json body \ - --jq '.body | test("Closes #<ISSUE_NUMBER>")' +python3 scripts/gh.py link-issue <ISSUE_NUMBER> <PR_NUMBER> ``` -**Note:** If the PR is already merged, `Closes` won't auto-close the issue — -it only creates the "Development" sidebar link. This is the desired -behavior since the issue is a tracking artifact. +The command creates the GitHub Development link and verifies it from both the issue and PR. It is safe to rerun and does not close an issue retroactively when the PR is already merged. Do not rely on the body keyword as the assignment operation. ### Clean up diff --git a/.serena/memories/workflow/creating-prs.md b/.serena/memories/workflow/creating-prs.md index fa94a9afe0..5d5b132a6b 100644 --- a/.serena/memories/workflow/creating-prs.md +++ b/.serena/memories/workflow/creating-prs.md @@ -36,7 +36,7 @@ See `mem:workflow/creating-commits` for emoji codes. Squash merge uses the PR ti Include concise sections covering: - what changed and why; -- related GitHub issues or Taiga stories (`Closes #NNNN`, `Relates to #NNNN`, `Taiga #NNNN`); +- related GitHub issues or Taiga stories (`Closes #NNNN` for issues resolved by the PR, `Relates to #NNNN` for context, `Taiga #NNNN`); use the explicit issue assignment process below for each `Closes` issue; - screenshots or recordings for UI-visible changes; - testing performed and residual risk; - breaking changes or migration notes, if any. @@ -77,6 +77,12 @@ The "Note:" line is required at the top. Adjust if this is a manual (non-AI) PR. - ❌ Migration notes unless breaking changes - ❌ Regression fixes introduced during the PR (they're part of the development process, not the feature) +## Explicit Issue Assignment + +- For each GitHub issue that a PR resolves, run `python3 scripts/gh.py link-issue <ISSUE_NUMBER> <PR_NUMBER>` after creating or editing the PR. Do not rely on `Closes #NNNN` in the body; it is only human-readable context. +- The command calls `addCloseIssueReferences`, verifies the relationship from both the issue and PR, and exits non-zero if either side is missing. It is safe to rerun and also works for an already merged PR; it does not close an issue retroactively. +- Skip this process for `Relates to #NNNN` and Taiga references, which do not represent a closing relationship. + ## Before Opening - Follow `mem:workflow/creating-commits` for commits diff --git a/scripts/gh.py b/scripts/gh.py index f8b4f74d45..df4a4682e6 100755 --- a/scripts/gh.py +++ b/scripts/gh.py @@ -8,6 +8,7 @@ Subcommands: issues List issues in a milestone (or unassigned with milestone=none) prs Fetch details for one or more PRs (by number or milestone) advisories List or inspect GitHub security advisories + link-issue Explicitly link a GitHub issue to a pull request Usage: python3 scripts/gh.py issues <milestone-title> (default: state=closed) @@ -27,6 +28,7 @@ Usage: python3 scripts/gh.py advisories (list all advisories) python3 scripts/gh.py advisories --severity critical (filter by severity) python3 scripts/gh.py advisories GHSA-xvj6-fh9w-gjw7 (single advisory detail) + python3 scripts/gh.py link-issue 11235 11243 Prerequisites: - gh CLI authenticated (gh auth status) @@ -77,6 +79,141 @@ def run_gh_rest(path: str) -> Any: return json.loads(result.stdout) +# ───────────────────────────────────────────── +# Subcommand: link-issue +# ───────────────────────────────────────────── + +GQL_LINK_TARGETS_QUERY = """\ +query($owner: String!, $repo: String!, $issueNumber: Int!, $prNumber: Int!) { + repository(owner: $owner, name: $repo) { + issue(number: $issueNumber) { id number } + pullRequest(number: $prNumber) { id number } + } +} +""" + +GQL_ADD_CLOSE_ISSUE_REFERENCES = """\ +mutation($issueId: ID!, $pullRequestIds: [ID!]!) { + addCloseIssueReferences(input: {issueId: $issueId, pullRequestIds: $pullRequestIds}) { + issue { id number } + } +} +""" + +GQL_VERIFY_ISSUE_LINK_QUERY = """\ +query($owner: String!, $repo: String!, $issueNumber: Int!, $prNumber: Int!) { + repository(owner: $owner, name: $repo) { + issue(number: $issueNumber) { + number + state + closedByPullRequestsReferences( + includeClosedPrs: true + userLinkedOnly: true + first: 100 + ) { + nodes { number state url } + } + } + pullRequest(number: $prNumber) { + number + state + closingIssuesReferences(first: 100) { + nodes { number state url } + } + } + } +} +""" + + +def link_issue_to_pr(issue_number: int, pr_number: int) -> dict: + """Add and verify an explicit GitHub issue-to-PR link.""" + if issue_number <= 0 or pr_number <= 0: + raise ValueError("issue and pull request numbers must be positive") + + variables = { + "owner": OWNER, + "repo": REPO_NAME, + "issueNumber": issue_number, + "prNumber": pr_number, + } + target_data = run_gh_graphql(GQL_LINK_TARGETS_QUERY, variables) + repository = target_data.get("repository") or {} + issue = repository.get("issue") or {} + pull_request = repository.get("pullRequest") or {} + if not issue.get("id"): + raise RuntimeError(f"issue #{issue_number} was not found in {REPO}") + if not pull_request.get("id"): + raise RuntimeError(f"pull request #{pr_number} was not found in {REPO}") + + mutation_data = run_gh_graphql( + GQL_ADD_CLOSE_ISSUE_REFERENCES, + { + "issueId": issue["id"], + "pullRequestIds": [pull_request["id"]], + }, + ) + mutation_result = mutation_data.get("addCloseIssueReferences") or {} + linked_issue = mutation_result.get("issue") or {} + if linked_issue.get("number") != issue_number: + raise RuntimeError(f"GitHub did not link issue #{issue_number}") + + verification_data = run_gh_graphql(GQL_VERIFY_ISSUE_LINK_QUERY, variables) + repository = verification_data.get("repository") or {} + issue = repository.get("issue") or {} + pull_request = repository.get("pullRequest") or {} + if not issue or not pull_request: + raise RuntimeError("GitHub did not return both link targets during verification") + + issue_links = [ + node + for node in issue["closedByPullRequestsReferences"]["nodes"] + if node.get("number") == pr_number + ] + pr_links = [ + node + for node in pull_request["closingIssuesReferences"]["nodes"] + if node.get("number") == issue_number + ] + if not issue_links or not pr_links: + raise RuntimeError( + f"issue #{issue_number} and pull request #{pr_number} are not linked" + ) + + return { + "linked": True, + "issue": { + "number": issue["number"], + "state": issue["state"], + "linked_pull_requests": issue_links, + }, + "pull_request": { + "number": pull_request["number"], + "state": pull_request["state"], + "linked_issues": pr_links, + }, + } + + +def cmd_link_issue(args: argparse.Namespace) -> None: + """Handle the ``link-issue`` subcommand.""" + print( + f"Linking issue #{args.issue_number} to pull request #{args.pr_number}...", + file=sys.stderr, + ) + try: + result = link_issue_to_pr(args.issue_number, args.pr_number) + except (ValueError, RuntimeError) as error: + print(f"ERROR: {error}", file=sys.stderr) + sys.exit(1) + + print( + f"Verified issue #{args.issue_number} -> pull request #{args.pr_number}", + file=sys.stderr, + ) + print(json.dumps(result, indent=2)) + + # ───────────────────────────────────────────── # Shared: milestone lookup # ───────────────────────────────────────────── @@ -764,6 +901,16 @@ def main() -> None: ) p_prs.set_defaults(func=cmd_prs) + # --- link-issue --- + p_link = sub.add_parser( + "link-issue", + aliases=["link"], + help="Explicitly link an issue to a pull request and verify both sides", + ) + p_link.add_argument("issue_number", type=int, help="Issue number") + p_link.add_argument("pr_number", type=int, help="Pull request number") + p_link.set_defaults(func=cmd_link_issue) + # --- advisories --- p_adv = sub.add_parser("advisories", help="List or inspect GitHub security advisories") p_adv.add_argument( diff --git a/scripts/test_gh.py b/scripts/test_gh.py new file mode 100644 index 0000000000..83e80ea5fd --- /dev/null +++ b/scripts/test_gh.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 +"""Tests for scripts/gh.py. + +Run with: + + python3 scripts/test_gh.py +""" + +import contextlib +import importlib.machinery +import importlib.util +import io +import json +import pathlib +import sys +import types +import unittest +from unittest.mock import patch + + +# Loading scripts/gh.py should not emit scripts/__pycache__/. +sys.dont_write_bytecode = True + +SCRIPT_PATH = pathlib.Path(__file__).resolve().parent / "gh.py" + + +def load_gh(): + """Load scripts/gh.py as a module without running its CLI.""" + loader = importlib.machinery.SourceFileLoader("gh_helper", str(SCRIPT_PATH)) + spec = importlib.util.spec_from_loader("gh_helper", loader) + module = importlib.util.module_from_spec(spec) + loader.exec_module(module) + return module + + +gh = load_gh() + + +class LinkIssueTests(unittest.TestCase): + def setUp(self): + self.target_response = { + "repository": { + "issue": {"id": "issue-id", "number": 11235}, + "pullRequest": {"id": "pr-id", "number": 11243}, + } + } + self.mutation_response = { + "addCloseIssueReferences": { + "issue": {"id": "issue-id", "number": 11235} + } + } + self.verification_response = { + "repository": { + "issue": { + "number": 11235, + "state": "OPEN", + "closedByPullRequestsReferences": { + "nodes": [ + { + "number": 11243, + "state": "MERGED", + "url": "https://github.com/penpot/penpot/pull/11243", + } + ] + }, + }, + "pullRequest": { + "number": 11243, + "state": "MERGED", + "closingIssuesReferences": { + "nodes": [ + { + "number": 11235, + "state": "OPEN", + "url": "https://github.com/penpot/penpot/issues/11235", + } + ] + }, + }, + } + } + + @patch.object(gh, "run_gh_graphql") + def test_link_issue_to_pr_adds_and_verifies_reference(self, run_graphql): + run_graphql.side_effect = [ + self.target_response, + self.mutation_response, + self.verification_response, + ] + + result = gh.link_issue_to_pr(11235, 11243) + + self.assertTrue(result["linked"]) + self.assertEqual( + result["issue"]["linked_pull_requests"][0]["number"], + 11243, + ) + self.assertEqual( + result["pull_request"]["linked_issues"][0]["number"], + 11235, + ) + self.assertEqual(run_graphql.call_count, 3) + self.assertEqual( + run_graphql.call_args_list[1].args[1], + {"issueId": "issue-id", "pullRequestIds": ["pr-id"]}, + ) + + @patch.object(gh, "run_gh_graphql") + def test_link_issue_to_pr_fails_when_verification_is_missing(self, run_graphql): + self.verification_response["repository"]["issue"][ + "closedByPullRequestsReferences" + ]["nodes"] = [] + run_graphql.side_effect = [ + self.target_response, + self.mutation_response, + self.verification_response, + ] + + with self.assertRaisesRegex(RuntimeError, "are not linked"): + gh.link_issue_to_pr(11235, 11243) + + @patch.object(gh, "link_issue_to_pr") + def test_cmd_link_issue_outputs_verified_result(self, link_issue): + expected = { + "linked": True, + "issue": {"number": 11235, "state": "OPEN"}, + "pull_request": {"number": 11243, "state": "MERGED"}, + } + link_issue.return_value = expected + args = types.SimpleNamespace(issue_number=11235, pr_number=11243) + stdout = io.StringIO() + stderr = io.StringIO() + + with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr): + gh.cmd_link_issue(args) + + self.assertEqual(json.loads(stdout.getvalue()), expected) + self.assertIn("Verified issue #11235", stderr.getvalue()) + link_issue.assert_called_once_with(11235, 11243) + + @patch.object(gh, "link_issue_to_pr", side_effect=RuntimeError("link missing")) + def test_cmd_link_issue_fails_when_verification_is_missing(self, _link_issue): + args = types.SimpleNamespace(issue_number=11235, pr_number=11243) + stderr = io.StringIO() + + with contextlib.redirect_stderr(stderr): + with self.assertRaises(SystemExit) as error: + gh.cmd_link_issue(args) + + self.assertEqual(error.exception.code, 1) + self.assertIn("link missing", stderr.getvalue()) + + +if __name__ == "__main__": + unittest.main(verbosity=2)