From 114928ea5f5f9e30656bb13e848670f33ab801f6 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Tue, 20 Oct 2026 15:33:37 +0000 Subject: [PATCH 1/3] :bug: Fix the key reported when a redis DEL times out `app.redis/del` logged `:key key`, but `key` is not bound in that `let`: it resolved to `clojure.core/key`, so the line carried `#object[clojure.core$key ...]` instead of the keys that failed to be deleted. Log `:keys keys`, which is always the normalized vector, so a single key and a collection report the same shape. The tests pin the line: the keys, the hint, the error level, the logger and the timeout as cause, plus the `nil` the caller gets. The prod-infra memory now notes that client-side redis timeouts mean the JVM is paused, not that the server is slow. AI-assisted-by: deepseek-v4.1-flash --- .serena/memories/prod-infra/core.md | 2 +- backend/src/app/redis.clj | 2 +- backend/test/backend_tests/redis_test.clj | 76 +++++++++++++++++++++++ 3 files changed, 78 insertions(+), 2 deletions(-) create mode 100644 backend/test/backend_tests/redis_test.clj diff --git a/.serena/memories/prod-infra/core.md b/.serena/memories/prod-infra/core.md index 5dfaeebed7..0ab9ecde8e 100644 --- a/.serena/memories/prod-infra/core.md +++ b/.serena/memories/prod-infra/core.md @@ -5,7 +5,7 @@ Backend (`app.config`, `PENPOT_*` env vars) is parameterized; deployments choose ## Services - **PostgreSQL**: durable store. Profiles, teams, files, sessions, audit, `storage_object` metadata, the `task` queue, `scheduled_task` cron registry, migrations. File-data also lives here when the file-data backend is `legacy-db`/`db`. One shared DB across all backends. -- **Redis (Valkey-compatible)**: per-backend message bus and cache. Concrete uses: msgbus Pub/Sub for collaborative-editing broadcasts and team/profile-org notifications fired by RPC handlers (`app.rpc.notifications`, `files_update`, `teams`, `websocket`); file-summary cache gated by `enable-redis-cache`; rate-limit counters; and the dispatcher→runner work hand-off list `penpot.worker.queue::`. `PENPOT_REDIS_URI`. +- **Redis (Valkey-compatible)**: per-backend message bus and cache. Concrete uses: msgbus Pub/Sub for collaborative-editing broadcasts and team/profile-org notifications fired by RPC handlers (`app.rpc.notifications`, `files_update`, `teams`, `websocket`); file-summary cache gated by `enable-redis-cache`; rate-limit counters; and the dispatcher→runner work hand-off list `penpot.worker.queue::`. `PENPOT_REDIS_URI`. A burst of client-side `RedisCommandTimeoutException` (2s on `DEL`/`SET`, 10s on `BLPOP`) with the exporter healthy is the JVM paused or starved, not Redis: the timeout fires because nothing reads the reply. - **Object storage**: backends `:s3` and `:fs`. S3 in prod; devenv uses RustFS. Holds uploaded media, file-data when the file-data backend is `storage`, exports. Backend-side details (resolve, dedup, bucket set, object lifecycle, and file-data backends): `mem:backend/storage`. - **SMTP mailer**: invitations, password resets, email verification (sent via the `:sendmail` worker task). - **LDAP** (optional auth provider): helpers in `app.auth.*`, gated by `enable-login-with-ldap`. diff --git a/backend/src/app/redis.clj b/backend/src/app/redis.clj index be9e331dda..fba9268193 100644 --- a/backend/src/app/redis.clj +++ b/backend/src/app/redis.clj @@ -375,7 +375,7 @@ (try (-del conn keys) (catch RedisCommandTimeoutException cause - (l/err :hint "timeout on del redis key" :key key :cause cause) + (l/err :hint "timeout on del redis key" :keys keys :cause cause) nil)))) (defn ping diff --git a/backend/test/backend_tests/redis_test.clj b/backend/test/backend_tests/redis_test.clj new file mode 100644 index 0000000000..df3c8e15a8 --- /dev/null +++ b/backend/test/backend_tests/redis_test.clj @@ -0,0 +1,76 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS INC Sucursal en España SL + +(ns backend-tests.redis-test + "The redis wrapper swallows a command timeout and returns nil, so the log + line is the only trace of the command that did not reach the server. These + tests pin what that line has to carry." + (:require + [app.common.logging :as l] + [app.redis :as rds] + [clojure.test :as t]) + (:import + io.lettuce.core.RedisCommandTimeoutException)) + +(def ^:private timeout-hint "timeout on del redis key") + +(defn- timeout-connection + "A connection whose DEL always exceeds the command timeout, the way Lettuce + answers when the response does not arrive in time." + [] + (reify rds/IDefaultConnection + (-del [_ _keys] + (throw (RedisCommandTimeoutException. "DEL. Command timed out after 2 second(s)"))))) + +(defn- capture-records + "Run `f` with every log record collected, newest last. The logging API emits + asynchronously, so a record may land after `f` returns; callers wait for the + one they need." + [f] + (let [records (atom [])] + (add-watch l/log-record ::capture + (fn [_ _ _ record] + (swap! records conj record))) + (try + (f records) + (finally + (remove-watch l/log-record ::capture))))) + +(defn- timeout-record + "Wait until the record that reports a timed out DEL is captured." + [records timeout] + (let [deadline (+ (System/currentTimeMillis) timeout)] + (loop [] + (or (some (fn [record] + (when (= timeout-hint (get (::l/props record) :hint)) + record)) + @records) + (when (<= (System/currentTimeMillis) deadline) + (Thread/sleep 10) + (recur)))))) + +(t/deftest del-logs-the-keys-that-timed-out + ;; The server never saw the command, so the operator has no other way to know + ;; which key stayed behind. + (capture-records + (fn [records] + (t/is (nil? (rds/del (timeout-connection) ["file-summary:1" "file-summary:2"]))) + (let [record (timeout-record records 2000)] + (t/is (some? record) "a timed out DEL must leave a log line") + (t/is (= ["file-summary:1" "file-summary:2"] + (get (::l/props record) :keys))) + (t/is (instance? RedisCommandTimeoutException (::l/cause record))) + (t/is (= :error (::l/level record))) + (t/is (= "app.redis" (::l/logger record))))))) + +(t/deftest del-logs-a-single-key-as-a-collection + ;; `del` accepts one key or a collection and always hands a collection to the + ;; driver; the log keeps the same shape so it does not depend on the call. + (capture-records + (fn [records] + (rds/del (timeout-connection) "file-summary:1") + (t/is (= ["file-summary:1"] + (get (::l/props (timeout-record records 2000)) :keys)))))) From 3bab9fa0aaad4edb485db6942ec4bc61a16015a9 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Tue, 20 Oct 2026 20:28:06 +0000 Subject: [PATCH 2/3] :books: Note the host variable needs three updates The prod-infra memory now says that the `$host` multi-variable needs its `query`, its `options` list and its `allValue` regex updated together. A host present only in `query` is dropped when `All` is selected, so the panels that filter with `instance=~"$host..."` stop showing it. Panel 104 is the visible symptom. AI-assisted-by: minimax-m3.1-flash-free --- .serena/memories/prod-infra/grafana.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.serena/memories/prod-infra/grafana.md b/.serena/memories/prod-infra/grafana.md index 533b09516f..14324b1f7f 100644 --- a/.serena/memories/prod-infra/grafana.md +++ b/.serena/memories/prod-infra/grafana.md @@ -20,6 +20,7 @@ Keep this memory free of internal names: no host names, no deployed versions, no ## Main dashboard - `Production status`, uid `penpot-production-status`. Sections: general state and deploy, `RPC y CLIMIT`, PostgreSQL and pool, Redis/msgbus, tasks, JVM/process. Variables `$env` and `$host` (custom), no auto refresh. Every section opens collapsed except `RPC y CLIMIT`, which is the one read on a normal day. + - `$host` is a custom multi-variable: its `options` list and its `allValue` regex must both enumerate every host. A host added to `query` but missing from `options`/`allValue` is silently dropped when `All` is selected, so panels that filter with `instance=~"$host..."` stop showing it (panel 104 is the visible symptom). Update all three places together when adding a host. - Per-host investigation dashboards are separate, and the alert rules deeplink into them. - The PostgreSQL section ends with a block of vacuum panels: two stats (autovacuum workers active on the primary, and the launcher alive), a Loki timeseries of finished vacuums per interval and table, and then the `Vacuum TXID age` table, which is the last panel of the section. The worker stat is an approximation — it counts the `pg_stat_activity` bucket described above, because the exporter publishes no `backend_type` — and the Loki panel is retrospective by construction. A per-table, in-flight view (table, phase, progress) needs a `pg_stat_progress_vacuum` collector added to the exporter itself. From 021492a57772ff3178b8c21f215a1806c54b5ee3 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Tue, 20 Oct 2026 20:36:24 +0000 Subject: [PATCH 3/3] :bug: Make migration 0156 compatible with PostgreSQL 15 `SET STORAGE DEFAULT` needs PostgreSQL 16, but the official docker images pin `postgres:15`, so migration 0156 failed on those deployments. The migration now uses `SET STORAGE EXTENDED`, the storage mode of the type of every column it touches (text, text[], jsonb and bytea), which leaves the catalog in the same state on any supported version. The documented PostgreSQL floor goes back to >=15, the reason for raising it was that syntax. Verified on the test database: the 62 columns reset and no table column keeps the legacy EXTERNAL storage. The backend memory now records that migrations must stay within PostgreSQL 15 syntax. AI-assisted-by: minimax-m3.1-flash-free --- .serena/memories/backend/core.md | 2 + .../sql/0156-reset-pg-storage-defaults.sql | 136 +++++++++--------- docs/technical-guide/configuration.md | 2 +- 3 files changed, 72 insertions(+), 68 deletions(-) diff --git a/.serena/memories/backend/core.md b/.serena/memories/backend/core.md index 42e7c7b756..7ed309bd18 100644 --- a/.serena/memories/backend/core.md +++ b/.serena/memories/backend/core.md @@ -41,6 +41,8 @@ Backend RPC command areas without focused memories include access tokens, binfil Database migrations live in `backend/src/app/migrations/`; pure SQL migrations are under `backend/src/app/migrations/sql/`. SQL filenames conventionally start with a sequence and verb/table description, e.g. `0026-mod-profile-table-add-is-active-field`. Applied migrations are tracked in the `migrations` table. +Migrations must stay within PostgreSQL 15 syntax: the official docker images (`docker/images/docker-compose.yaml`) pin `postgres:15`, so anything newer breaks those deployments (`SET STORAGE DEFAULT` is PostgreSQL 16, for instance). The documented floor is >=15. + For interactive PostgreSQL access with correct dev defaults, use `scripts/psql`; to dump the current DDL schema, use `scripts/db-schema` (see `mem:scripts/psql`). diff --git a/backend/src/app/migrations/sql/0156-reset-pg-storage-defaults.sql b/backend/src/app/migrations/sql/0156-reset-pg-storage-defaults.sql index 54bd98ace5..46a7cce54b 100644 --- a/backend/src/app/migrations/sql/0156-reset-pg-storage-defaults.sql +++ b/backend/src/app/migrations/sql/0156-reset-pg-storage-defaults.sql @@ -12,118 +12,120 @@ -- already stored TOAST values keep their strategy until the row is -- rewritten, new writes use the default. -- --- NOTE: DEFAULT restores the storage mode of the column's data type --- (EXTENDED for every column below: text, text[], jsonb and bytea). It --- requires PostgreSQL 16 or newer, which is the minimum version the --- project supports (see the database section of the technical guide). --- RESET STORAGE is not valid syntax for a column. +-- NOTE: `SET STORAGE DEFAULT` would express the same thing (it restores +-- the storage mode of the column's data type), but it only exists since +-- PostgreSQL 16 and the project also runs on PostgreSQL 15. Here every +-- column below has a type whose default storage mode is EXTENDED +-- (text, text[], jsonb and bytea), so `SET STORAGE EXTENDED` reaches the +-- same catalog state on any supported version. RESET STORAGE is not +-- valid syntax for a column. ALTER TABLE access_token - ALTER COLUMN name SET STORAGE DEFAULT, - ALTER COLUMN perms SET STORAGE DEFAULT, - ALTER COLUMN token SET STORAGE DEFAULT; + ALTER COLUMN name SET STORAGE EXTENDED, + ALTER COLUMN perms SET STORAGE EXTENDED, + ALTER COLUMN token SET STORAGE EXTENDED; ALTER TABLE audit_log - ALTER COLUMN context SET STORAGE DEFAULT, - ALTER COLUMN name SET STORAGE DEFAULT, - ALTER COLUMN props SET STORAGE DEFAULT, - ALTER COLUMN source SET STORAGE DEFAULT, - ALTER COLUMN type SET STORAGE DEFAULT; + ALTER COLUMN context SET STORAGE EXTENDED, + ALTER COLUMN name SET STORAGE EXTENDED, + ALTER COLUMN props SET STORAGE EXTENDED, + ALTER COLUMN source SET STORAGE EXTENDED, + ALTER COLUMN type SET STORAGE EXTENDED; ALTER TABLE comment - ALTER COLUMN content SET STORAGE DEFAULT; + ALTER COLUMN content SET STORAGE EXTENDED; ALTER TABLE comment_thread - ALTER COLUMN page_name SET STORAGE DEFAULT, - ALTER COLUMN participants SET STORAGE DEFAULT; + ALTER COLUMN page_name SET STORAGE EXTENDED, + ALTER COLUMN participants SET STORAGE EXTENDED; ALTER TABLE file - ALTER COLUMN data SET STORAGE DEFAULT, - ALTER COLUMN data_backend SET STORAGE DEFAULT, - ALTER COLUMN features SET STORAGE DEFAULT, - ALTER COLUMN name SET STORAGE DEFAULT; + ALTER COLUMN data SET STORAGE EXTENDED, + ALTER COLUMN data_backend SET STORAGE EXTENDED, + ALTER COLUMN features SET STORAGE EXTENDED, + ALTER COLUMN name SET STORAGE EXTENDED; ALTER TABLE file_change - ALTER COLUMN changes SET STORAGE DEFAULT, - ALTER COLUMN data SET STORAGE DEFAULT, - ALTER COLUMN features SET STORAGE DEFAULT, - ALTER COLUMN label SET STORAGE DEFAULT; + ALTER COLUMN changes SET STORAGE EXTENDED, + ALTER COLUMN data SET STORAGE EXTENDED, + ALTER COLUMN features SET STORAGE EXTENDED, + ALTER COLUMN label SET STORAGE EXTENDED; ALTER TABLE file_data_fragment - ALTER COLUMN metadata SET STORAGE DEFAULT; + ALTER COLUMN metadata SET STORAGE EXTENDED; ALTER TABLE file_media_object - ALTER COLUMN mtype SET STORAGE DEFAULT, - ALTER COLUMN name SET STORAGE DEFAULT; + ALTER COLUMN mtype SET STORAGE EXTENDED, + ALTER COLUMN name SET STORAGE EXTENDED; ALTER TABLE file_thumbnail - ALTER COLUMN data SET STORAGE DEFAULT, - ALTER COLUMN props SET STORAGE DEFAULT; + ALTER COLUMN data SET STORAGE EXTENDED, + ALTER COLUMN props SET STORAGE EXTENDED; ALTER TABLE global_complaint_report - ALTER COLUMN content SET STORAGE DEFAULT, - ALTER COLUMN type SET STORAGE DEFAULT; + ALTER COLUMN content SET STORAGE EXTENDED, + ALTER COLUMN type SET STORAGE EXTENDED; ALTER TABLE profile - ALTER COLUMN auth_backend SET STORAGE DEFAULT, - ALTER COLUMN email SET STORAGE DEFAULT, - ALTER COLUMN fullname SET STORAGE DEFAULT, - ALTER COLUMN lang SET STORAGE DEFAULT, - ALTER COLUMN password SET STORAGE DEFAULT, - ALTER COLUMN props SET STORAGE DEFAULT, - ALTER COLUMN theme SET STORAGE DEFAULT; + ALTER COLUMN auth_backend SET STORAGE EXTENDED, + ALTER COLUMN email SET STORAGE EXTENDED, + ALTER COLUMN fullname SET STORAGE EXTENDED, + ALTER COLUMN lang SET STORAGE EXTENDED, + ALTER COLUMN password SET STORAGE EXTENDED, + ALTER COLUMN props SET STORAGE EXTENDED, + ALTER COLUMN theme SET STORAGE EXTENDED; ALTER TABLE profile_complaint_report - ALTER COLUMN content SET STORAGE DEFAULT, - ALTER COLUMN type SET STORAGE DEFAULT; + ALTER COLUMN content SET STORAGE EXTENDED, + ALTER COLUMN type SET STORAGE EXTENDED; ALTER TABLE project - ALTER COLUMN name SET STORAGE DEFAULT; + ALTER COLUMN name SET STORAGE EXTENDED; ALTER TABLE scheduled_task - ALTER COLUMN cron_expr SET STORAGE DEFAULT, - ALTER COLUMN id SET STORAGE DEFAULT; + ALTER COLUMN cron_expr SET STORAGE EXTENDED, + ALTER COLUMN id SET STORAGE EXTENDED; ALTER TABLE server_error_report - ALTER COLUMN content SET STORAGE DEFAULT; + ALTER COLUMN content SET STORAGE EXTENDED; ALTER TABLE server_prop - ALTER COLUMN content SET STORAGE DEFAULT, - ALTER COLUMN id SET STORAGE DEFAULT; + ALTER COLUMN content SET STORAGE EXTENDED, + ALTER COLUMN id SET STORAGE EXTENDED; ALTER TABLE storage_object - ALTER COLUMN backend SET STORAGE DEFAULT, - ALTER COLUMN metadata SET STORAGE DEFAULT; + ALTER COLUMN backend SET STORAGE EXTENDED, + ALTER COLUMN metadata SET STORAGE EXTENDED; ALTER TABLE task - ALTER COLUMN error SET STORAGE DEFAULT, - ALTER COLUMN label SET STORAGE DEFAULT, - ALTER COLUMN name SET STORAGE DEFAULT, - ALTER COLUMN props SET STORAGE DEFAULT, - ALTER COLUMN queue SET STORAGE DEFAULT, - ALTER COLUMN status SET STORAGE DEFAULT; + ALTER COLUMN error SET STORAGE EXTENDED, + ALTER COLUMN label SET STORAGE EXTENDED, + ALTER COLUMN name SET STORAGE EXTENDED, + ALTER COLUMN props SET STORAGE EXTENDED, + ALTER COLUMN queue SET STORAGE EXTENDED, + ALTER COLUMN status SET STORAGE EXTENDED; ALTER TABLE team - ALTER COLUMN name SET STORAGE DEFAULT; + ALTER COLUMN name SET STORAGE EXTENDED; ALTER TABLE team_font_variant - ALTER COLUMN font_family SET STORAGE DEFAULT, - ALTER COLUMN font_style SET STORAGE DEFAULT; + ALTER COLUMN font_family SET STORAGE EXTENDED, + ALTER COLUMN font_style SET STORAGE EXTENDED; ALTER TABLE team_invitation - ALTER COLUMN email_to SET STORAGE DEFAULT, - ALTER COLUMN role SET STORAGE DEFAULT; + ALTER COLUMN email_to SET STORAGE EXTENDED, + ALTER COLUMN role SET STORAGE EXTENDED; ALTER TABLE usage_quote - ALTER COLUMN target SET STORAGE DEFAULT; + ALTER COLUMN target SET STORAGE EXTENDED; ALTER TABLE webhook - ALTER COLUMN error_code SET STORAGE DEFAULT, - ALTER COLUMN mtype SET STORAGE DEFAULT, - ALTER COLUMN secret_key SET STORAGE DEFAULT, - ALTER COLUMN uri SET STORAGE DEFAULT; + ALTER COLUMN error_code SET STORAGE EXTENDED, + ALTER COLUMN mtype SET STORAGE EXTENDED, + ALTER COLUMN secret_key SET STORAGE EXTENDED, + ALTER COLUMN uri SET STORAGE EXTENDED; ALTER TABLE webhook_delivery - ALTER COLUMN error_code SET STORAGE DEFAULT, - ALTER COLUMN req_data SET STORAGE DEFAULT, - ALTER COLUMN rsp_data SET STORAGE DEFAULT; + ALTER COLUMN error_code SET STORAGE EXTENDED, + ALTER COLUMN req_data SET STORAGE EXTENDED, + ALTER COLUMN rsp_data SET STORAGE EXTENDED; diff --git a/docs/technical-guide/configuration.md b/docs/technical-guide/configuration.md index 59939a522b..374c5e408c 100644 --- a/docs/technical-guide/configuration.md +++ b/docs/technical-guide/configuration.md @@ -484,7 +484,7 @@ PENPOT_SECRET_KEY: my-super-secure-key ### Database -Penpot only supports PostgreSQL and we highly recommend >=16 version. If you are using official +Penpot only supports PostgreSQL and we highly recommend >=15 version. If you are using official docker images this is already solved for you. Essential database configuration: