From 097ff41e0344842b53ca7cb0e338cb6dab7dc2de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?andr=C3=A9s=20gonz=C3=A1lez?= Date: Fri, 13 Nov 2026 11:59:12 +0100 Subject: [PATCH 01/15] :sparkles: Send telemetry for WebGL renderer failures (#12987) * :sparkles: Send telemetry for WebGL renderer failures WebGL renderer failures left no trace in the audit log, so we could not tell why people switch WebGL off. - Send webgl-unavailable when the "WebGL is not available" modal shows, with the init error. - Send webgl-context-lost, webgl-context-restored and webgl-reload-failed from the WASM renderer. - Add the active renderer (wasm or svg) to handled-exception, unhandled-exception, exception-page and open-workspace-file. - Add environment details to the WebGL events and to enable/disable-webgl-rendering: normalized GPU family and backend, max texture size, device memory, CPU cores and whether the app runs inside Electron. The full GPU renderer string is not sent, as it can help fingerprint a device. Closes #12984 AI-assisted-by: claude-opus-5-5 * :zap: Avoid throwaway WebGL contexts in renderer telemetry WebGL events no longer create a context to read GPU details; the settings, menu and unavailable modal create one at most once. AI-assisted-by: claude-opus-5-5 * :bug: Refresh remembered GPU details on each renderer init A restored context can run on another GPU, so read it on every init instead of only the first one. AI-assisted-by: claude-opus-5-5 * :zap: Never create a WebGL context for renderer telemetry WebGL events send GPU details only when the renderer has already read them on init, instead of probing a throwaway context. AI-assisted-by: claude-opus-5-5 --------- Co-authored-by: Elena Torro --- frontend/src/app/main/data/workspace.cljs | 3 +- frontend/src/app/main/errors.cljs | 4 +- frontend/src/app/main/features.cljs | 7 ++ .../src/app/main/ui/settings/options.cljs | 10 +-- .../src/app/main/ui/workspace/main_menu.cljs | 6 +- .../app/main/ui/workspace/viewport_wasm.cljs | 17 ++++- frontend/src/app/render_wasm/api.cljs | 20 +++++- frontend/src/app/render_wasm/api/webgl.cljs | 71 +++++++++++++++++++ .../frontend_tests/errors_governor_test.cljs | 12 ++++ 9 files changed, 139 insertions(+), 11 deletions(-) diff --git a/frontend/src/app/main/data/workspace.cljs b/frontend/src/app/main/data/workspace.cljs index 7581866dc6..cc9c9c7954 100644 --- a/frontend/src/app/main/data/workspace.cljs +++ b/frontend/src/app/main/data/workspace.cljs @@ -314,7 +314,8 @@ ::ev/name "open-workspace-file" ::ev/origin "workspace" :file-id file-id - :team-id team-id))))))) + :team-id team-id + :renderer (features/get-active-renderer state)))))))) (defn- restore-pending-changes "Applies the edits still waiting to be saved to a freshly loaded `file`. diff --git a/frontend/src/app/main/errors.cljs b/frontend/src/app/main/errors.cljs index 5f48fd0c9e..fceac0d344 100644 --- a/frontend/src/app/main/errors.cljs +++ b/frontend/src/app/main/errors.cljs @@ -16,6 +16,7 @@ [app.main.data.nitrate :as dnt] [app.main.data.notifications :as ntf] [app.main.data.workspace :as-alias dw] + [app.main.features :as features] [app.main.repo :as rp] [app.main.router :as rt] [app.main.store :as st] @@ -410,7 +411,8 @@ :hint hint :href (rt/get-current-href) :report report - :occurrences occurrences}))) + :occurrences occurrences + :renderer (features/get-active-renderer @st/state)}))) (defn submit-report "Report the error report to the audit log subsystem, subject to the diff --git a/frontend/src/app/main/features.cljs b/frontend/src/app/main/features.cljs index b5fcfdc528..7ced696941 100644 --- a/frontend/src/app/main/features.cljs +++ b/frontend/src/app/main/features.cljs @@ -85,6 +85,13 @@ "feature not supported") (contains? (:features state) feature)) +(defn get-active-renderer + "Name of the renderer active in `state`, for telemetry: `wasm` (WebGL) or + `svg`, or nil before the features are initialized." + [state] + (when (contains? state :features) + (if (active-feature? state "render-wasm/v1") "wasm" "svg"))) + (defn active-features? "Given a state and a set of features, check if the features are all enabled." ([state a] diff --git a/frontend/src/app/main/ui/settings/options.cljs b/frontend/src/app/main/ui/settings/options.cljs index b841f3567d..00898df09c 100644 --- a/frontend/src/app/main/ui/settings/options.cljs +++ b/frontend/src/app/main/ui/settings/options.cljs @@ -20,6 +20,7 @@ [app.main.ui.ds.foundations.typography :as t] [app.main.ui.ds.foundations.typography.heading :refer [heading*]] [app.main.ui.ds.foundations.typography.text :refer [text*]] + [app.render-wasm.api.webgl :as webgl] [app.util.dom :as dom] [app.util.i18n :as i18n :refer [tr]] [app.util.theme :as theme] @@ -101,10 +102,11 @@ handle-render-change (mf/use-fn (fn [enabled?] - (st/emit! (ev/event {::ev/name (if enabled? - "enable-webgl-rendering" - "disable-webgl-rendering") - ::ev/origin "settings"}) + (st/emit! (ev/event (assoc (webgl/get-environment) + ::ev/name (if enabled? + "enable-webgl-rendering" + "disable-webgl-rendering") + ::ev/origin "settings")) (du/update-profile-props {:renderer (if enabled? :wasm :svg)}) (ntf/success (if enabled? (tr "webgl.toast.webgl-render-enabled") diff --git a/frontend/src/app/main/ui/workspace/main_menu.cljs b/frontend/src/app/main/ui/workspace/main_menu.cljs index 79d2b80a20..b05f7eaab3 100644 --- a/frontend/src/app/main/ui/workspace/main_menu.cljs +++ b/frontend/src/app/main/ui/workspace/main_menu.cljs @@ -42,6 +42,7 @@ [app.main.ui.ds.foundations.assets.icon :as i :refer [icon*]] [app.main.ui.hooks.resize :as r] [app.plugins.register :as preg] + [app.render-wasm.api.webgl :as webgl] [app.util.dom :as dom] [app.util.i18n :as i18n :refer [tr]] [app.util.keyboard :as kbd] @@ -965,8 +966,9 @@ (dom/reload-current-window true)) (fn [_] (st/emit! (ntf/error (tr "errors.generic")))))) - (st/emit! (ev/event {::ev/name ev-name - ::ev/origin "workspace:menu"}) + (st/emit! (ev/event (assoc (webgl/get-environment) + ::ev/name ev-name + ::ev/origin "workspace:menu")) (ptk/data-event ::ev/force-persist {}))))) open-plugins-manager diff --git a/frontend/src/app/main/ui/workspace/viewport_wasm.cljs b/frontend/src/app/main/ui/workspace/viewport_wasm.cljs index cfa1a3d38d..1cb2ad6c88 100644 --- a/frontend/src/app/main/ui/workspace/viewport_wasm.cljs +++ b/frontend/src/app/main/ui/workspace/viewport_wasm.cljs @@ -16,6 +16,7 @@ [app.common.types.component :as ctk] [app.common.types.shape :as cts] [app.common.types.shape.layout :as ctl] + [app.main.data.event :as ev] [app.main.data.modal :as modal] [app.main.data.workspace :as dw] [app.main.data.workspace.transforms :as dwt] @@ -58,6 +59,7 @@ [app.main.ui.workspace.viewport.viewport-ref :as vp-ref :refer [create-viewport-ref]] [app.main.ui.workspace.viewport.widgets :as widgets] [app.render-wasm.api :as wasm.api] + [app.render-wasm.api.webgl :as webgl] [app.render-wasm.rulers-state :as rs] [app.util.debug :as dbg] [app.util.text-editor :as ted] @@ -70,6 +72,10 @@ ;; --- Viewport +;; Retries after the first failed WebGL context init before showing the +;; "WebGL unavailable" modal. +(def ^:private ^:const max-init-retries 3) + (defn- apply-modifiers-to-objects [objects modifiers] (->> modifiers @@ -460,21 +466,28 @@ (let [timeout-id-ref (volatile! nil) unmounted? (volatile! false) modal-shown? (volatile! false) + init-error (volatile! nil) show-unavailable (fn [] (when-not (or @unmounted? @modal-shown?) (vreset! modal-shown? true) (reset! canvas-init? false) - (st/emit! (modal/show {:type :webgl-unavailable})))) + (st/emit! (modal/show {:type :webgl-unavailable}) + (ev/event (assoc (webgl/get-environment) + ::ev/name "webgl-unavailable" + ::ev/origin "workspace" + :hint (or @init-error "no webgl2 context")))))) try-init (fn try-init [retries] (when-not @unmounted? + (vreset! init-error nil) (let [init? (try (wasm.api/init-canvas-context canvas) (catch :default e (js/console.error "Error initializing canvas context:" e) + (vreset! init-error (ex-message e)) false))] (cond init? @@ -490,7 +503,7 @@ (->> @wasm.api/module (p/fmap (fn [ready?] (when ready? - (try-init 3))))) + (try-init max-init-retries))))) (fn [] (vreset! unmounted? true) (when-let [timeout-id @timeout-id-ref] diff --git a/frontend/src/app/render_wasm/api.cljs b/frontend/src/app/render_wasm/api.cljs index aab9321c6c..d9ad59efc5 100644 --- a/frontend/src/app/render_wasm/api.cljs +++ b/frontend/src/app/render_wasm/api.cljs @@ -33,6 +33,7 @@ [app.common.types.text :as txt] [app.common.uuid :as uuid] [app.config :as cf] + [app.main.data.event :as-alias ev] [app.main.data.helpers :as dsh] [app.main.data.notifications :as ntf] [app.main.data.render-wasm :as drw] @@ -2389,6 +2390,17 @@ (set-render-options! new-dpr) (resize-viewbox css-w css-h))))) +(defn- emit-webgl-event + "Send a WebGL audit event with the environment details. Built as a plain + data event so this namespace does not load `app.main.data.event`, which + the worker build cannot run." + [event-name props] + (st/emit! (ptk/data-event ::ev/event + (merge (webgl/get-environment) + props + {::ev/name event-name + ::ev/origin "workspace"})))) + (defn- on-webgl-context-lost [event] (dom/prevent-default event) @@ -2405,7 +2417,8 @@ :type :toast :level :warning :timeout 5000})) - (st/emit! (drw/context-lost))) + (st/emit! (drw/context-lost)) + (emit-webgl-event "webgl-context-lost" {})) (defn- on-webgl-context-restored [event] @@ -2422,6 +2435,7 @@ ;; only after the full reload so layout/viewport sync runs ;; against a live renderer with `wasm/ready?` true. (st/emit! (drw/context-restored)) + (emit-webgl-event "webgl-context-restored" {}) (listen-tiles-render-complete-once! end-context-loss-overlay!) (st/async-emit! (ntf/show {:content (tr "webgl.webgl-context-recovered.toast") @@ -2472,6 +2486,7 @@ (let [{:keys [context handle]} result] (set! wasm/gl-context-handle handle) (set! wasm/gl-context context) + (webgl/remember-gpu-info context) ;; `init-context` already ran `_init` through `_set_browser` in ;; order. Publish the handles, then mark live before sizing so the @@ -2609,6 +2624,9 @@ (p/catch (fn [cause] (ug/dispatch! (ug/event "penpot:wasm:reload-failed")) + (emit-webgl-event "webgl-reload-failed" + {:hint (ex-message cause) + :code (some-> (ex-data cause) :code name)}) (reset! wasm/reloading? false) (reset! wasm/context-lost? true) (clear-canvas) diff --git a/frontend/src/app/render_wasm/api/webgl.cljs b/frontend/src/app/render_wasm/api/webgl.cljs index 3fe8a7b0ba..cde2622727 100644 --- a/frontend/src/app/render_wasm/api/webgl.cljs +++ b/frontend/src/app/render_wasm/api/webgl.cljs @@ -11,6 +11,7 @@ [app.common.math :as mth] [app.common.render-wasm.helpers :as h] [app.common.render-wasm.wasm :as wasm] + [cuerdas.core :as str] [promesa.core :as p])) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -314,3 +315,73 @@ void main() { (resolve false))) (set! (.-src img) uri)))) (p/resolved false))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Environment +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defn- get-gpu-family + "Normalize a GPU renderer string to a GPU family. Telemetry sends the + family, not the full string, which can help fingerprint a device." + [renderer] + (let [renderer (str/lower renderer) + has? (fn [& parts] (some #(str/includes? renderer %) parts))] + (cond + (has? "swiftshader") "swiftshader" + (has? "llvmpipe" "softpipe") "llvmpipe" + (has? "microsoft basic render") "microsoft-basic-render" + (has? "apple") "apple" + (has? "nvidia" "geforce" "quadro") "nvidia" + (has? "amd" "radeon") "amd" + (has? "intel") "intel" + (has? "adreno") "adreno" + (has? "mali") "mali" + (has? "powervr") "powervr" + :else "other"))) + +(defn- get-gpu-backend + "Graphics API behind the WebGL implementation (ANGLE names it in the + renderer string), or nil when the string does not say." + [renderer] + (let [renderer (str/lower renderer) + has? (fn [& parts] (some #(str/includes? renderer %) parts))] + (cond + (has? "direct3d11" "d3d11") "d3d11" + (has? "direct3d9" "d3d9") "d3d9" + (has? "metal") "metal" + (has? "vulkan") "vulkan" + (has? "opengl") "opengl"))) + +(defn- read-gpu-info + "GPU details of a live WebGL context, or nil when it is lost." + [^js gl] + (when-not (.isContextLost gl) + (let [^js ext (.getExtension gl "WEBGL_debug_renderer_info") + renderer (or (when ext + (.getParameter gl (.-UNMASKED_RENDERER_WEBGL ext))) + (.getParameter gl (.-RENDERER gl)))] + (when (string? renderer) + {:gpu-family (get-gpu-family renderer) + :gpu-backend (get-gpu-backend renderer) + :max-texture-size (.getParameter gl (.-MAX_TEXTURE_SIZE gl))})))) + +(defonce ^:private gpu-info + (volatile! nil)) + +(defn remember-gpu-info + "Keep the GPU details of `gl`, a context the renderer just created, so + later events can send them after that context is lost." + [gl] + (when-let [info (try (read-gpu-info gl) (catch :default _ nil))] + (vreset! gpu-info info))) + +(defn get-environment + "Environment details for WebGL telemetry: GPU family, GPU backend and max + texture size when already remembered, device memory, CPU cores and + whether the app runs inside Electron. Never creates a WebGL context." + [] + (let [^js navigator js/navigator] + (assoc @gpu-info + :device-memory (.-deviceMemory navigator) + :cpu-cores (.-hardwareConcurrency navigator) + :electron (str/includes? (.-userAgent navigator) "Electron/")))) diff --git a/frontend/test/frontend_tests/errors_governor_test.cljs b/frontend/test/frontend_tests/errors_governor_test.cljs index a51f775a43..035188c5ff 100644 --- a/frontend/test/frontend_tests/errors_governor_test.cljs +++ b/frontend/test/frontend_tests/errors_governor_test.cljs @@ -174,6 +174,18 @@ (t/is (= 1 (count events))) (t/is (= 1 (:occurrences (deref (first events))))))) +(t/deftest submit-report-carries-the-active-renderer + (let [submit (fn [state hint] + (with-redefs [st/state (atom state)] + (capture-reports! + #(errors/submit-report :event-name "handled-exception" + :report "report" + :hint hint + :cause (error-cause :hint hint)))))] + (t/is (= "wasm" (:renderer (deref (first (submit {:features #{"render-wasm/v1"}} "a")))))) + (t/is (= "svg" (:renderer (deref (first (submit {:features #{}} "b")))))) + (t/is (nil? (:renderer (deref (first (submit {} "c")))))))) + (t/deftest invalid-report-does-not-consume-a-reservation (let [cause (error-cause :type :network :hint "boom") events (capture-reports! From bb4556b70500a2d4573a3382f2a7a4e6e0317570 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marina=20L=C3=B3pez?= Date: Fri, 13 Nov 2026 13:07:05 +0100 Subject: [PATCH 02/15] :bug: Persist promotional banner collapse (#13040) --- backend/src/app/rpc/commands/profile.clj | 1 + .../src/app/main/ui/dashboard/subscription.cljs | 14 +++++++++----- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/backend/src/app/rpc/commands/profile.clj b/backend/src/app/rpc/commands/profile.clj index 3bd86b459b..2a9ee61118 100644 --- a/backend/src/app/rpc/commands/profile.clj +++ b/backend/src/app/rpc/commands/profile.clj @@ -70,6 +70,7 @@ [:onboarding-questions {:optional true} [:map-of :keyword :string]] [:onboarding-questions-answered {:optional true} ::sm/boolean] [:nitrate-onboarding-viewed {:optional true} ::sm/boolean] + [:promotional-banner-collapsed {:optional true} ::sm/boolean] [:v2-info-shown {:optional true} ::sm/boolean] [:release-notes-viewed {:optional true} [::sm/text {:max 100}]] diff --git a/frontend/src/app/main/ui/dashboard/subscription.cljs b/frontend/src/app/main/ui/dashboard/subscription.cljs index b7434d5279..4e94c9ce78 100644 --- a/frontend/src/app/main/ui/dashboard/subscription.cljs +++ b/frontend/src/app/main/ui/dashboard/subscription.cljs @@ -9,6 +9,7 @@ [app.main.data.event :as ev] [app.main.data.modal :as modal] [app.main.data.nitrate :as dnt] + [app.main.data.profile :as dprof] [app.main.refs :as refs] [app.main.router :as rt] [app.main.store :as st] @@ -131,11 +132,13 @@ :is-highlighted false}])))) (mf/defc nitrate-enterprise-banner* - [{:keys [on-click button-label]}] - (let [collapsed* (mf/use-state false) - collapsed? (deref collapsed*) + [{:keys [on-click button-label profile]}] + (let [collapsed? (true? (get-in profile [:props :promotional-banner-collapsed])) toggle-collapse - (mf/use-fn #(swap! collapsed* not))] + (mf/use-fn + (mf/deps collapsed?) + #(st/emit! (dprof/update-profile-props + {:promotional-banner-collapsed (not collapsed?)})))] [:div {:class (stl/css-case :nitrate-banner true :nitrate-enterprise-banner true @@ -269,7 +272,8 @@ ;; Banner for Professional/Unlimited users outside an organization (when show-unlock-features? [:> nitrate-enterprise-banner* - {:on-click handle-click + {:profile profile + :on-click handle-click :button-label (if (:subscription profile) (tr "subscription.dashboard.banner.upgrade-nitrate") (tr "nitrate.form.try-free"))}])) From 0e4be213200ab5d2324aeea37905ca5f9fa31109 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20Moya?= Date: Fri, 13 Nov 2026 13:53:28 +0100 Subject: [PATCH 03/15] :bug: Ensure component name always matches root instance name (#12794) * :bug: Remove svg extension when creating a component from a svg file * :bug: Normalize root names to always match the component name * :sparkles: Remove any extension ignoring case, not only .svg * :wrench: Add some fixes and checks * :wrench: Improve rename detection --- common/src/app/common/files/builder.cljc | 24 +++-- .../src/app/common/files/shapes_builder.cljc | 3 +- common/src/app/common/logic/libraries.cljc | 87 +++++++++++++++---- common/src/app/common/path_names.cljc | 14 +++ .../files/shapes_builder_test.cljc | 17 ++++ .../test/common_tests/files_builder_test.cljc | 29 +++++++ .../logic/comp_creation_test.cljc | 32 +++++++ .../common_tests/logic/comp_sync_test.cljc | 65 ++++++++++++++ common/test/common_tests/path_names_test.cljc | 8 ++ .../src/app/main/data/workspace/media.cljs | 7 +- .../data/workspace_media_test.cljs | 73 +++++++++++++++- 11 files changed, 333 insertions(+), 26 deletions(-) diff --git a/common/src/app/common/files/builder.cljc b/common/src/app/common/files/builder.cljc index 2e02166a93..c5b769121f 100644 --- a/common/src/app/common/files/builder.cljc +++ b/common/src/app/common/files/builder.cljc @@ -86,7 +86,7 @@ (defn- unique-name [name state] (let [container-id (::current-page-id state) - unames (dm/get-in state [:unames container-id])] + unames (dm/get-in state [::unames container-id])] (d/unique-name name (or unames #{})))) (defn- clear-names [file] @@ -462,12 +462,24 @@ frame-id (or frame-id (get state ::current-frame-id)) + name + (or name "anonymous") + + path + (d/nilv path "") + + ;; The main instance root name must match the component's path+name, so + ;; the frame is renamed if needed. This keeps the builder output valid + ;; under the :invalid-main-instance-name integrity check. + normalized-name + (cpn/merge-path-item path name) + change1 (d/without-nils {:type :add-component :id component-id - :name (or name "anonmous") - :path (d/nilv path "") + :name name + :path path :main-instance-id frame-id :main-instance-page page-id :variant-id variant-id @@ -481,11 +493,13 @@ [{:type :set :attr :component-root :val true} {:type :set :attr :main-instance :val true} {:type :set :attr :component-id :val component-id} - {:type :set :attr :component-file :val file-id}]}] + {:type :set :attr :component-file :val file-id} + {:type :set :attr :name :val normalized-name}]}] (-> state (commit-change change1) - (commit-change change2)))) + (commit-change change2) + (track-used-name normalized-name)))) (defn add-tokens-lib [state tokens-lib] diff --git a/common/src/app/common/files/shapes_builder.cljc b/common/src/app/common/files/shapes_builder.cljc index 9577fb528d..61826488b7 100644 --- a/common/src/app/common/files/shapes_builder.cljc +++ b/common/src/app/common/files/shapes_builder.cljc @@ -18,6 +18,7 @@ [app.common.geom.shapes.common :as gsc] [app.common.geom.shapes.transforms :as gst] [app.common.math :as mth] + [app.common.path-names :as cpn] [app.common.schema :as sm :refer [max-safe-int min-safe-int]] [app.common.svg :as csvg] [app.common.types.color :as clr] @@ -196,7 +197,7 @@ (let [[vb-x vb-y vb-width vb-height] (svg-dimensions svg-data) unames (cfh/get-used-names objects) - svg-name (str/replace (:name svg-data) ".svg" "") + svg-name (cpn/remove-extension (:name svg-data)) svg-data (-> svg-data (assoc :x (mth/round diff --git a/common/src/app/common/logic/libraries.cljc b/common/src/app/common/logic/libraries.cljc index 16bf550dcf..3a0c2949ff 100644 --- a/common/src/app/common/logic/libraries.cljc +++ b/common/src/app/common/logic/libraries.cljc @@ -1146,26 +1146,69 @@ initial-root? (:component-root shape-inst) + ;; The component name is normalized (path separators with one space at + ;; each side), so the name synced to the main instance root must be + ;; normalized too. Otherwise the root would not match the component's + ;; path+name and the file would fail :invalid-main-instance-name. + rename? (and (some? component) + (contains? (:touched shape-inst) :name-group)) + original-name (:name shape-inst) + clean-name (when rename? + (cpn/clean-path original-name)) + shape-inst (cond-> shape-inst + (some? clean-name) + (assoc :name clean-name)) + root-inst shape-inst root-main (ctf/get-component-root library component) - changes (cond-> changes - (and component (contains? (:touched shape-inst) :name-group)) - (generate-rename-component (:component-id shape-inst) (:name shape-inst) library))] + ;; When the synced shape is itself the component's main instance root, + ;; `find-remote-shape` returns nil and the recursive sync never reaches + ;; it, so it must be renamed explicitly to keep it matching the + ;; component's path+name. + main-root? (and (some? component) + (= (:id shape-inst) (:main-instance-id component))) - (if component - (generate-sync-shape-inverse-recursive changes - container - shape-inst - component - library - file - libraries - shape-main - root-inst - root-main - initial-root?) - changes))) + changes (cond-> changes + (some? clean-name) + (generate-rename-component (:component-id shape-inst) clean-name library)) + + changes (if component + (generate-sync-shape-inverse-recursive changes + container + shape-inst + component + library + file + libraries + shape-main + root-inst + root-main + initial-root?) + changes) + + changes (cond-> changes + (and (some? clean-name) + main-root? + (not= clean-name original-name)) + (-> (update :redo-changes conj + (make-change container + {:type :mod-obj + :id (:id shape-inst) + :local-change? true + :operations [{:type :set + :attr :name + :val clean-name}]})) + (update :undo-changes conj + (make-change container + {:type :mod-obj + :id (:id shape-inst) + :local-change? true + :operations [{:type :set + :attr :name + :val original-name}]}))))] + + changes)) (defn- generate-sync-shape-inverse-recursive [changes container shape-inst component library file libraries shape-main root-inst root-main initial-root?] @@ -2694,10 +2737,20 @@ [path name] (cpn/split-group-name name) + ;; The name of the main instance root must match the path+name of the + ;; component. The component name is normalized here (via split-group-name), + ;; so the root shape name must be normalized too. Otherwise creating a + ;; component from a shape with an unnormalized name (e.g. "Group/Subgroup") + ;; would leave a main instance failing :invalid-main-instance-name. + normalized-name (cpn/merge-path-item path name) + [root-shape updated-shapes] - (ctn/convert-shape-in-component root objects file-id) + (ctn/convert-shape-in-component (assoc root :name normalized-name) objects file-id) changes (-> changes + (cond-> (not= normalized-name (:name root)) + (pcb/update-shapes [(:id root)] #(assoc % :name normalized-name) + {:ignore-touched true})) (pcb/add-component (:id root-shape) path name diff --git a/common/src/app/common/path_names.cljc b/common/src/app/common/path_names.cljc index 3f5fd28603..a81b6cfaee 100644 --- a/common/src/app/common/path_names.cljc +++ b/common/src/app/common/path_names.cljc @@ -147,6 +147,20 @@ Some naming conventions: [(subs s 0 (inc last-period)) (subs s (inc last-period))] [s ""])) +(defn remove-extension + "Removes the extension (the text after the last period, including the + period itself) from a name: + 'photo.svg' -> 'photo' + 'icon.SVG' -> 'icon' + 'my.svg.icon.svg' -> 'my.svg.icon' + 'archive.tar.gz' -> 'archive.tar' + 'design' -> 'design'" + [name] + (let [[base ext] (split-by-last-period name)] + (if (str/empty? ext) + name + (subs base 0 (dec (count base)))))) + ;; Tree building functions -------------------------------------------------- (defn- attach-remaining-path diff --git a/common/test/common_tests/files/shapes_builder_test.cljc b/common/test/common_tests/files/shapes_builder_test.cljc index 614ff7cf77..44e6560f36 100644 --- a/common/test/common_tests/files/shapes_builder_test.cljc +++ b/common/test/common_tests/files/shapes_builder_test.cljc @@ -91,3 +91,20 @@ ;; stroke-style is not set at import time (nil means default solid rendering) (t/is (nil? (:stroke-style stroke))))))) + +;; Regression for https://github.com/penpot/penpot/issues/12773: the SVG +;; root shape name must have any extension stripped (not only ".svg"), +;; so the component name and the main instance root name match. +(t/deftest create-svg-shapes-strips-name-extension + (let [svg-data {:name "my.svg.icon.SVG" + :tag :svg + :attrs {:xmlns "http://www.w3.org/2000/svg" + :width "24" :height "24" + :viewBox "0 0 24 24"} + :content [{:tag :rect + :attrs {:x "0" :y "0" :width "24" :height "24"} + :content []}]} + [root _children] (sb/create-svg-shapes svg-data {:x 0 :y 0} {} nil nil #{} false)] + + (t/is (= "my.svg.icon" (:name root))))) + diff --git a/common/test/common_tests/files_builder_test.cljc b/common/test/common_tests/files_builder_test.cljc index 23891a583b..5a35829d56 100644 --- a/common/test/common_tests/files_builder_test.cljc +++ b/common/test/common_tests/files_builder_test.cljc @@ -7,6 +7,8 @@ (ns common-tests.files-builder-test (:require [app.common.files.builder :as fb] + [app.common.files.validate :as cfv] + [app.common.path-names :as cpn] [app.common.uuid :as uuid] [clojure.test :as t])) @@ -87,3 +89,30 @@ :height 10})) shape (fb/get-shape state shape-id)] (t/is (= "Group / Subgroup / Name" (:name shape))))) + +(t/deftest add-component-renames-frame-to-match-component + (t/testing "the main instance frame name is aligned with the component's path+name" + (let [file-id (uuid/next) + page-id (uuid/next) + frame-id (uuid/next) + component-id (uuid/next) + state (-> (fb/create-state) + (fb/add-file {:id file-id :name "Test file"}) + (fb/add-page {:id page-id :name "Page 1"}) + (fb/add-board {:id frame-id + :name "Wrong / Frame / Name" + :x 0 + :y 0 + :width 10 + :height 10}) + (fb/add-component {:component-id component-id + :name "Component" + :path "Group / Subgroup"})) + shape (fb/get-shape state frame-id) + file-data (get-in state [::fb/files file-id :data]) + component (get-in file-data [:components component-id]) + errors (cfv/validate-file {:id file-id :data file-data} {})] + + (t/is (= "Group / Subgroup / Component" (:name shape))) + (t/is (= (:name shape) (cpn/merge-path-item (:path component) (:name component)))) + (t/is (nil? errors))))) diff --git a/common/test/common_tests/logic/comp_creation_test.cljc b/common/test/common_tests/logic/comp_creation_test.cljc index 95f158e2a3..eb0fb36d01 100644 --- a/common/test/common_tests/logic/comp_creation_test.cljc +++ b/common/test/common_tests/logic/comp_creation_test.cljc @@ -9,6 +9,7 @@ [app.common.data :as d] [app.common.files.changes-builder :as pcb] [app.common.files.shapes-helpers :as cfsh] + [app.common.files.validate :as cfv] [app.common.geom.point :as gpt] [app.common.logic.libraries :as cll] [app.common.logic.shapes :as cls] @@ -91,6 +92,37 @@ (t/is (ctk/main-instance? root)) (t/is (ctk/main-instance-of? (:id root) (:id page) component)))) +(t/deftest test-add-component-normalizes-root-name + (t/testing "main instance root name matches the component even if the source name is not normalized" + (let [;; ==== Setup + file (-> (thf/sample-file :file1) + (ths/add-sample-shape :frame1 :type :frame :name "Group/Subgroup / Name")) + + page (thf/current-page file) + frame1 (ths/get-shape file :frame1) + + ;; ==== Action + [_ component-id changes] + (cll/generate-add-component (pcb/empty-changes) + [frame1] + (:objects page) + (:id page) + (:id file) + nil) + + file' (thf/apply-changes file changes) + + ;; ==== Get + component (thc/get-component-by-id file' component-id) + root (ths/get-shape-by-id file' (:main-instance-id component)) + errors (cfv/validate-file file' {})] + + ;; ==== Check + (t/is (= "Group / Subgroup" (:path component))) + (t/is (= "Name" (:name component))) + (t/is (= "Group / Subgroup / Name" (:name root))) + (t/is (nil? errors))))) + (t/deftest test-add-component-from-several-shapes (let [;; ==== Setup file (-> (thf/sample-file :file1) diff --git a/common/test/common_tests/logic/comp_sync_test.cljc b/common/test/common_tests/logic/comp_sync_test.cljc index a929c7a616..f44d526b33 100644 --- a/common/test/common_tests/logic/comp_sync_test.cljc +++ b/common/test/common_tests/logic/comp_sync_test.cljc @@ -8,16 +8,20 @@ (:require [app.common.data :as d] [app.common.files.changes-builder :as pcb] + [app.common.files.validate :as cfv] [app.common.geom.point :as gpt] [app.common.geom.shapes :as gsh] [app.common.logic.libraries :as cll] [app.common.logic.shapes :as cls] + [app.common.path-names :as cpn] [app.common.test-helpers.components :as thc] [app.common.test-helpers.compositions :as tho] [app.common.test-helpers.files :as thf] [app.common.test-helpers.ids-map :as thi] [app.common.test-helpers.shapes :as ths] [app.common.types.component :as ctk] + [app.common.types.components-list :as ctkl] + [app.common.types.container :as ctn] [app.common.types.shape-tree :as ctst] [clojure.test :as t])) @@ -539,3 +543,64 @@ ;; A position-only change in the main component must not propagate to copies ;; and therefore must produce no redo-changes. (t/is (empty? (:redo-changes sync-changes))))) + +(defn- run-sync-inverse + "Run the inverse sync on `shape-tag`, which has been renamed to an + unnormalized name, and return the resulting main root and component + names, and the updated file." + [shape-tag] + (let [file (-> (thf/sample-file :file1) + (tho/add-simple-component-with-copy :component1 + :main-root + :main-child + :copy-root) + (tho/change-shape shape-tag #(update % :touched (fnil conj #{}) :name-group)) + (ths/update-shape shape-tag :name "Group/Subgroup")) + page (thf/current-page file) + container (ctn/make-container page :page) + changes (-> (pcb/empty-changes) + (pcb/with-container container) + (cll/generate-sync-shape-inverse (:data file) + {(:id file) file} + container + (thi/id shape-tag))) + file' (thf/apply-changes file changes) + main-root' (ths/get-shape file' :main-root) + component' (ctkl/get-component (:data file') (thi/id :component1))] + [(:name main-root') + (cpn/merge-path-item (:path component') (:name component')) + file'])) + +(t/deftest test-sync-inverse-normalizes-copy-name + (t/testing "syncing an unnormalized copy name keeps the component and its main root matching" + (let [file (-> (thf/sample-file :file1) + (tho/add-simple-component-with-copy :component1 + :main-root + :main-child + :copy-root) + (tho/change-shape :copy-root #(update % :touched (fnil conj #{}) :name-group)) + (ths/update-shape :copy-root :name "Group/Subgroup")) + page (thf/current-page file) + container (ctn/make-container page :page) + changes (-> (pcb/empty-changes) + (pcb/with-container container) + (cll/generate-sync-shape-inverse (:data file) + {(:id file) file} + container + (thi/id :copy-root))) + file' (thf/apply-changes file changes) + main-root' (ths/get-shape file' :main-root) + component' (ctkl/get-component (:data file') (thi/id :component1))] + + ;; ==== Check + (t/is (= "Group / Subgroup" (:name main-root'))) + (t/is (= "Group / Subgroup" (cpn/merge-path-item (:path component') (:name component')))) + (t/is (nil? (cfv/validate-file file' {})))))) + +(t/deftest test-sync-inverse-normalizes-main-name + (t/testing "syncing an unnormalized main name keeps the component and the main root matching" + (let [[main-root-name component-name file'] (run-sync-inverse :main-root)] + ;; ==== Check + (t/is (= "Group / Subgroup" main-root-name)) + (t/is (= "Group / Subgroup" component-name)) + (t/is (nil? (cfv/validate-file file' {})))))) diff --git a/common/test/common_tests/path_names_test.cljc b/common/test/common_tests/path_names_test.cljc index 04cf0650b8..b6dde08be0 100644 --- a/common/test/common_tests/path_names_test.cljc +++ b/common/test/common_tests/path_names_test.cljc @@ -9,6 +9,14 @@ [app.common.path-names :as cpn] [clojure.test :as t])) +(t/deftest remove-extension + (t/is (= "photo" (cpn/remove-extension "photo.svg"))) + (t/is (= "icon" (cpn/remove-extension "icon.SVG"))) + (t/is (= "my.svg.icon" (cpn/remove-extension "my.svg.icon.svg"))) + (t/is (= "archive.tar" (cpn/remove-extension "archive.tar.gz"))) + (t/is (= "design" (cpn/remove-extension "design"))) + (t/is (= "" (cpn/remove-extension ".svg")))) + (t/deftest split-group-name (t/is (= ["foo" "bar"] (cpn/split-group-name "foo/bar"))) (t/is (= ["" "foo"] (cpn/split-group-name "foo"))) diff --git a/frontend/src/app/main/data/workspace/media.cljs b/frontend/src/app/main/data/workspace/media.cljs index f3a395a504..2ef99f761f 100644 --- a/frontend/src/app/main/data/workspace/media.cljs +++ b/frontend/src/app/main/data/workspace/media.cljs @@ -14,6 +14,7 @@ [app.common.files.shapes-builder :as sb] [app.common.math :as mth] [app.common.media :as media] + [app.common.path-names :as cpn] [app.common.schema :as sm] [app.common.types.container :as ctn] [app.common.types.fills :as types.fills] @@ -435,10 +436,12 @@ fdata (dsh/lookup-file-data state file-id) page (dsh/get-page fdata page-id) - pos (dsh/get-viewport-center state)] + pos (dsh/get-viewport-center state) + + name (cpn/remove-extension (:name svg-data))] (->> (create-shapes-svg file-id (:objects page) pos svg-data) - (rx/map (partial add-shapes-and-component it fdata page (:name svg-data)))))))) + (rx/map (partial add-shapes-and-component it fdata page name))))))) (defn upload-media-components [params] diff --git a/frontend/test/frontend_tests/data/workspace_media_test.cljs b/frontend/test/frontend_tests/data/workspace_media_test.cljs index cabdd54abb..03e80547c3 100644 --- a/frontend/test/frontend_tests/data/workspace_media_test.cljs +++ b/frontend/test/frontend_tests/data/workspace_media_test.cljs @@ -8,12 +8,24 @@ "Integration tests for the chunked-upload logic in app.main.data.workspace.media." (:require + [app.common.path-names :as cpn] + [app.common.test-helpers.files :as cthf] + [app.common.test-helpers.ids-map :as cthi] [app.common.uuid :as uuid] [app.config :as cf] [app.main.data.workspace.media :as media] + [app.main.store :as st] [beicon.v2.core :as rx] [cljs.test :as t :include-macros true] - [frontend-tests.helpers.http :as http])) + [frontend-tests.helpers.async :as async] + [frontend-tests.helpers.http :as http] + [frontend-tests.helpers.mock :as mock] + [frontend-tests.helpers.state :as ths] + [potok.v2.core :as ptk] + [tubax.core :as tubax])) + +(t/use-fixtures :each + {:before cthi/reset-idmap!}) ;; --------------------------------------------------------------------------- ;; Local helpers @@ -187,3 +199,62 @@ (http/restore-fetch! orig) (t/is (= 3 @chunk-calls)) (done)))))))) + +;; --------------------------------------------------------------------------- +;; SVG component creation: any file extension must be stripped from both +;; the component name and the main instance root shape name. +;; --------------------------------------------------------------------------- + +(def ^:private svg-content + (str "" + "" + "")) + +;; The `sax` XML parser used by `tubax/xml->clj` is not available in the +;; ESM test build, so the parse step is mocked with an equivalent tree. +;; Everything else (name handling included) runs for real. +(def ^:private parsed-svg + {:tag :svg + :attrs {:xmlns "http://www.w3.org/2000/svg" + :width "100" :height "100" + :viewBox "0 0 100 100"} + :content [{:tag :rect + :attrs {:x "0" :y "0" :width "100" :height "100"} + :content []}]}) + +(t/deftest ^:async upload-svg-component-strips-svg-extension + (t/testing "creating a component from an uploaded SVG removes the extension from both names" + ;; Any extension must be stripped (not only ".svg"), including + ;; uppercase ones, extensions in the middle of the name, and + ;; filenames with no extension at all. + (doseq [[filename expected] + [["my-component.svg" "my-component"] + ["icon.SVG" "icon"] + ["my.svg.icon.svg" "my.svg.icon"] + ["design" "design"]]] + (let [file (cthf/sample-file :file1) + store (ths/setup-store file) + blob (js/File. #js [svg-content] + filename + #js {:type "image/svg+xml"})] + (await + (mock/with-mocks* + {st/state store + st/emit! (mock/stub (fn [& events] (apply ptk/emit! store events))) + tubax/xml->clj (mock/stub (fn [_source] parsed-svg))} + ;; The SVG is placed at the viewport center; without a vbox the + ;; geometry setup would receive nil coordinates. + (ptk/emit! store #(assoc % :workspace-local {:vbox {:x 0 :y 0 :width 800 :height 600}})) + (ptk/emit! store (media/upload-media-components {:file-id (:id file) + :blobs [blob]})) + (await (async/wait-for #(seq (get-in @store [:files (:id file) :data :components])) + "component created")) + (let [data (get-in @store [:files (:id file) :data]) + component (first (vals (:components data))) + root (get-in data [:pages-index (:main-instance-page component) + :objects (:main-instance-id component)])] + (t/is (some? component) (str "component not created for " filename)) + (t/is (some? root) (str "root not created for " filename)) + (t/is (= expected (:name component))) + (t/is (= expected (:name root))) + (t/is (= expected (cpn/merge-path-item (:path component) (:name component))))))))))) From b2c4b58b27599b016a66ff0420e8622bbe732cc0 Mon Sep 17 00:00:00 2001 From: Eva Marco Date: Fri, 13 Nov 2026 14:11:30 +0100 Subject: [PATCH 04/15] :bug: Keep the pages panel scroll when switching pages (#13004) Switching pages swaps the workspace content for a loader until the new page is ready, so the whole left sidebar mounts again. The pages list started at the top and the selected page scrolled into view at the bottom of the panel, even when it was already visible. The sidebar scroll store now lives in workspace-inner*, which stays mounted across page switches. The pages panel saves its scroll there and restores it before the selected page scrolls into view. The layers, assets and tokens panels keep their positions across page switches too. Closes #13002 AI-assisted-by: claude-opus-5-5 --- frontend/src/app/main/ui/workspace.cljs | 17 +++++++++----- .../src/app/main/ui/workspace/sidebar.cljs | 13 ++++++----- .../app/main/ui/workspace/sidebar/scroll.cljs | 10 ++++----- .../main/ui/workspace/sidebar/sitemap.cljs | 22 +++++++++++++++++-- 4 files changed, 44 insertions(+), 18 deletions(-) diff --git a/frontend/src/app/main/ui/workspace.cljs b/frontend/src/app/main/ui/workspace.cljs index 4b37d284a9..b9958a3230 100644 --- a/frontend/src/app/main/ui/workspace.cljs +++ b/frontend/src/app/main/ui/workspace.cljs @@ -54,7 +54,7 @@ (mf/defc workspace-content* {::mf/private true} - [{:keys [file layout page wglobal]}] + [{:keys [file layout page wglobal scroll-store]}] (let [palete-size (mf/use-state nil) selected (mf/deref refs/selected-shapes) @@ -123,6 +123,7 @@ :file-id (get file :id) :page-id page-id :file file + :scroll-store scroll-store :selected selected :section options-mode :drawing-tool (get drawing :tool)}])])) @@ -173,9 +174,14 @@ (mf/defc workspace-inner* {::mf/private true} [{:keys [page-id file-id file layout wglobal]}] - (let [page-ref (mf/with-memo [file-id page-id] - (make-page-ref file-id page-id)) - page (mf/deref page-ref)] + (let [page-ref (mf/with-memo [file-id page-id] + (make-page-ref file-id page-id)) + page (mf/deref page-ref) + + ;; NOTE: lives here because the sidebar unmounts on every page + ;; switch (the loader replaces `workspace-content*` until the + ;; new page is ready), so the scroll positions must outlive it + scroll-store* (mf/use-var {})] (mf/with-effect [] (let [focus-out #(st/emit! (dw/workspace-focus-lost)) @@ -191,7 +197,8 @@ [:> workspace-content* {:file file :page page :wglobal wglobal - :layout layout}] + :layout layout + :scroll-store scroll-store*}] [:> workspace-loader*]))) (mf/defc workspace* diff --git a/frontend/src/app/main/ui/workspace/sidebar.cljs b/frontend/src/app/main/ui/workspace/sidebar.cljs index c8a0ece18e..aed6882aea 100644 --- a/frontend/src/app/main/ui/workspace/sidebar.cljs +++ b/frontend/src/app/main/ui/workspace/sidebar.cljs @@ -101,6 +101,7 @@ [:article {:class (stl/css :layers-tab)} [:> sitemap* {:layout layout :height sitemap-height + :scroll-store scroll-store :collapsed sitemap-collapsed? :on-toggle-collapsed on-toggle-sitemap-collapsed}] @@ -118,11 +119,10 @@ (mf/defc left-sidebar* {::mf/memo true} - [{:keys [layout file tokens-lib tokens-status active-tokens resolved-active-tokens]}] + [{:keys [layout file tokens-lib tokens-status active-tokens resolved-active-tokens scroll-store]}] (let [options-mode (mf/deref refs/options-mode-global) project (mf/deref refs/project) file-id (get file :id) - scroll-store* (mf/use-var {}) design-tokens? (features/use-feature "design-tokens/v1") mode-inspect? (= options-mode :inspect) @@ -216,7 +216,7 @@ :assets [:> assets-toolbox* {:size (- width 58) :file-id file-id - :scroll-store scroll-store*}] + :scroll-store scroll-store}] :tokens [:> tokens-sidebar-tab* @@ -224,12 +224,12 @@ :tokens-status tokens-status :active-tokens active-tokens :resolved-active-tokens resolved-active-tokens - :scroll-store scroll-store*}] + :scroll-store scroll-store}] :layers [:> layers-content* {:layout layout :width width - :scroll-store scroll-store*}])]])]])) + :scroll-store scroll-store}])]])]])) ;; --- Right Sidebar (Component) @@ -362,7 +362,7 @@ [:> options-toolbox* props]))]]]])) (mf/defc sidebar* - [{:keys [layout file file-id page-id section drawing-tool selected]}] + [{:keys [layout file file-id page-id section drawing-tool selected scroll-store]}] (let [tokens-lib (mf/deref refs/tokens-lib) @@ -407,6 +407,7 @@ [:> left-sidebar* {:layout layout :file file :page-id page-id + :scroll-store scroll-store :tokens-lib tokens-lib :tokens-status tokens-status :active-tokens active-tokens-force-set diff --git a/frontend/src/app/main/ui/workspace/sidebar/scroll.cljs b/frontend/src/app/main/ui/workspace/sidebar/scroll.cljs index 1902c9798f..386ed63b73 100644 --- a/frontend/src/app/main/ui/workspace/sidebar/scroll.cljs +++ b/frontend/src/app/main/ui/workspace/sidebar/scroll.cljs @@ -7,11 +7,11 @@ (ns app.main.ui.workspace.sidebar.scroll "Scroll position save/restore for sidebar panels. - Panels unmount on every tab switch, so positions that must survive it - cannot live in component state. The store is created once in - `left-sidebar*` (which stays mounted while tabs swap, unlike - `layers-content*` and the toolboxes) with `mf/use-var` and threaded to - the panels as a prop. Shape: {[panel id] scroll-top}." + Panels unmount on every tab switch, and the whole sidebar unmounts on + every page switch, so positions that must survive it cannot live in + component state. The store is created once in `workspace-inner*` (which + stays mounted while pages and tabs swap) with `mf/use-var` and threaded + to the panels as a prop. Shape: {[panel id] scroll-top}." (:require [app.util.dom :as dom] [app.util.timers :as tm] diff --git a/frontend/src/app/main/ui/workspace/sidebar/sitemap.cljs b/frontend/src/app/main/ui/workspace/sidebar/sitemap.cljs index 3f1ec3ab8d..818d002dd8 100644 --- a/frontend/src/app/main/ui/workspace/sidebar/sitemap.cljs +++ b/frontend/src/app/main/ui/workspace/sidebar/sitemap.cljs @@ -23,6 +23,7 @@ [app.main.ui.ds.foundations.assets.icon :as i :refer [icon*]] [app.main.ui.hooks :as hooks] [app.main.ui.notifications.badge :refer [badge-notification]] + [app.main.ui.workspace.sidebar.scroll :as sc] [app.render-wasm.api :as wasm.api] [app.util.dom :as dom] [app.util.i18n :as i18n :refer [tr]] @@ -291,10 +292,16 @@ ;; --- Sitemap Toolbox (mf/defc sitemap* - [{:keys [height collapsed on-toggle-collapsed]}] + [{:keys [height collapsed on-toggle-collapsed scroll-store]}] (let [file (mf/deref ref:file-with-pages) file-id (get file :id) project-id (get file :project-id) + content-ref (mf/use-ref nil) + + on-scroll (mf/use-fn + (mf/deps scroll-store file-id) + (fn [event] + (sc/save-scroll! scroll-store [:sitemap file-id] event))) on-create (mf/use-fn (mf/deps file-id project-id) @@ -306,6 +313,15 @@ read-only? (mf/use-ctx ctx/workspace-read-only?) permissions (mf/use-ctx ctx/permissions)] + ;; NOTE: a layout effect, so the saved position is back before the + ;; selected page item runs its `scroll-into-view-if-needed!` effect. + ;; With nothing saved the list is left alone, so the selected page + ;; still scrolls into view when the file opens. + (mf/with-layout-effect [file-id collapsed] + (when-not ^boolean collapsed + (when-some [saved (get @scroll-store [:sitemap file-id])] + (sc/restore-scroll! (mf/ref-val content-ref) saved)))) + [:div {:class (stl/css :sitemap) :style {:--height (dm/str height "px")}} @@ -326,6 +342,8 @@ :icon i/add}])] (when-not ^boolean collapsed - [:div {:class (stl/css :sitemap-content)} + [:div {:class (stl/css :sitemap-content) + :ref content-ref + :on-scroll on-scroll} [:> pages-list* {:key (dm/str (:id file)) :file file}]])])) From bde53b46961f36090f20e8b34e3c4f4f203153f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marina=20L=C3=B3pez?= Date: Fri, 13 Nov 2026 14:16:30 +0100 Subject: [PATCH 05/15] :bug: Hide team plan for standalone enterprise owners (#13047) --- .../app/main/ui/dashboard/subscription.cljs | 17 +++++--- frontend/src/app/main/ui/dashboard/team.cljs | 4 +- .../frontend_tests/ui/subscription_test.cljs | 39 ++++++++++++++++--- 3 files changed, 48 insertions(+), 12 deletions(-) diff --git a/frontend/src/app/main/ui/dashboard/subscription.cljs b/frontend/src/app/main/ui/dashboard/subscription.cljs index 4e94c9ce78..61c96506bc 100644 --- a/frontend/src/app/main/ui/dashboard/subscription.cljs +++ b/frontend/src/app/main/ui/dashboard/subscription.cljs @@ -35,11 +35,18 @@ "enterprise" (get-subscription-type (:subscription team)))) +(defn get-profile-subscription-type + [profile] + (if (dnt/is-valid-license? profile) + (dm/get-in profile [:subscription :type]) + (get-subscription-type (dm/get-in profile [:props :subscription])))) + (defn show-team-plan? - [is-owner team] + [is-owner team profile] (not (and is-owner (nil? (dm/get-in team [:organization :id])) - (= "enterprise" (get-subscription-type (:subscription team)))))) + (contains? #{"enterprise" "nitrate"} + (get-profile-subscription-type profile))))) (mf/defc cta-power-up* [{:keys [top-title top-description bottom-description bottom-button bottom-button-href has-dropdown is-highlighted]}] @@ -303,7 +310,7 @@ (let [nitrate? (dnt/is-valid-license? profile) nitrate-license (:subscription profile) subscription (-> profile :props :subscription) - subscription-type (if nitrate? (:type nitrate-license) (get-subscription-type subscription)) + subscription-type (get-profile-subscription-type profile) subscription-is-trial (= "trialing" (:status (if nitrate? nitrate-license subscription))) go-to-subscription (mf/use-fn #(st/emit! (rt/nav :settings-subscription)))] [:button {:class (stl/css :nitrate-current-plan) @@ -323,7 +330,7 @@ "enterprise" (tr "subscription.current-plan.enterprise"))]])) (mf/defc team* - [{:keys [is-owner team]}] + [{:keys [is-owner team profile]}] (let [subscription (:subscription team) subscription-type (get-subscription-type subscription) team-plan-type (get-team-plan-type team) @@ -341,7 +348,7 @@ href (str "payments/subscriptions/show?returnUrl=" href)] (st/emit! (rt/nav-raw :href href)))))] - (when (show-team-plan? is-owner team) + (when (show-team-plan? is-owner team profile) [:div {:class (stl/css :team)} [:div {:class (stl/css :team-label)} (tr "subscription.dashboard.team-plan")] diff --git a/frontend/src/app/main/ui/dashboard/team.cljs b/frontend/src/app/main/ui/dashboard/team.cljs index 4d74599557..0055548533 100644 --- a/frontend/src/app/main/ui/dashboard/team.cljs +++ b/frontend/src/app/main/ui/dashboard/team.cljs @@ -1779,4 +1779,6 @@ (tr "labels.num-of-files" (i18n/c (:files stats)))]]] (when (contains? cfg/flags :subscriptions) - [:> team* {:is-owner (:is-owner permissions) :team team}])]]])) + [:> team* {:is-owner (:is-owner permissions) + :team team + :profile profile}])]]])) diff --git a/frontend/test/frontend_tests/ui/subscription_test.cljs b/frontend/test/frontend_tests/ui/subscription_test.cljs index 7517720806..f85628dd15 100644 --- a/frontend/test/frontend_tests/ui/subscription_test.cljs +++ b/frontend/test/frontend_tests/ui/subscription_test.cljs @@ -6,6 +6,7 @@ (ns frontend-tests.ui.subscription-test (:require + [app.main.data.nitrate :as dnt] [app.main.ui.dashboard.subscription :as ds] [cljs.test :as t :include-macros true])) @@ -25,16 +26,42 @@ (t/deftest owner-enterprise-plan-is-hidden-for-team-outside-organization (t/is (false? (ds/show-team-plan? true {:subscription {:type "enterprise" - :status "active"}})))) + :status "active"}} + {:props {:subscription {:type "enterprise" + :status "active"}}})))) (t/deftest team-plan-is-visible-for-other-owner-and-organization-cases - (t/is (true? (ds/show-team-plan? true - {:subscription {:type "professional" - :status "active"}}))) + (doseq [owner-plan ["professional" "unlimited"]] + (t/is (true? (ds/show-team-plan? true + {:subscription {:type owner-plan + :status "active"}} + {:props {:subscription {:type owner-plan + :status "active"}}})))) (t/is (true? (ds/show-team-plan? true {:subscription {:type "enterprise" :status "active"} - :organization {:id "org-1"}}))) + :organization {:id "org-1"}} + {:props {:subscription {:type "enterprise" + :status "active"}}}))) (t/is (true? (ds/show-team-plan? false {:subscription {:type "enterprise" - :status "active"}})))) + :status "active"}} + {:props {:subscription {:type "enterprise" + :status "active"}}})))) + +(t/deftest owner-enterprise-plan-is-detected-even-when-team-subscription-falls-back + (t/is (false? (ds/show-team-plan? true + {:subscription {:type "professional" + :status "active"}} + {:props {:subscription {:type "enterprise" + :status "trialing"}}})))) + +(t/deftest nitrate-owner-plan-hides-team-plan-when-team-is-outside-organization + (with-redefs [dnt/is-valid-license? (constantly true)] + (t/is (false? (ds/show-team-plan? true + {:subscription {:type "professional" + :status "active"}} + {:subscription {:type "nitrate" + :status "active"} + :props {:subscription {:type "professional" + :status "active"}}}))))) From f3c1230b798f4e62d83f13923b7cb3061b4e548d Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Fri, 13 Nov 2026 17:33:42 +0100 Subject: [PATCH 06/15] :bug: Allow OIDC registration with password login disabled (#12974) * :bug: Allow OIDC registration with password login disabled The register-profile command rejected every registration without the login-with-password flag, including SSO ones, while the OIDC callback already allowed :oidc-registration as an alternative, so new OIDC users were stuck at the register screen with a 400 registration-disabled error. The gate now branches on the verified token backend: password tokens keep the previous checks, SSO tokens only require :registration or :oidc-registration through a predicate shared with the callback, so the two gates cannot drift apart again. Flags and invitation checks are split into single-responsibility helpers, and tokens without a backend claim are rejected loudly by schema instead of defaulting silently. Covered by regression tests for the SSO matrix, canceled invitations, the missing-backend case and the no-password-fallback, plus a configuration doc note. Closes #12964 AI-assisted-by: Muse Spark 1.3 Contributor * :bug: Gate SSO registration by provider type, accept invitations Follow up the review on #12974. The SSO registration gate now treats a team invitation token as a substitute for :registration / :oidc-registration, in both the OIDC callback and register-profile, mirroring the password flow: the callback redirects to the register screen when an invitation is present, and register-profile revalidates the invitation row under a row lock before creating the profile when no SSO flag is set, raising :invalid-token instead of creating an account that cannot join a team. The SSO branch is restricted to OIDC-typed backends (oidc-provider-types); a prepared-register token carrying any other backend is rejected with :invalid-token instead of silently taking the SSO rules. The flags and invitation checks are shared by one check-registration helper in register-profile; the prepare command runs flags, then the email-mismatch restriction, then the invitation row lookup, so the password login flag fails closed first. The schema defs are private, the create-profile default uses penpot-backend, the OIDC callback test decodes the redirect token to assert the :prepared-register claims, and the docs and the auth memory state the invitation path for SSO registration. Regression tests cover invitation-only registration, canceled invitations with and without SSO flags, unknown backends, and the double-fault precedence in prepare-register. AI-assisted-by: inkling-2-small-free * :bug: Require invitation email match for SSO registration Close the hole left by treating any invitation token as a substitute for the SSO registration flags. register-profile now requires the invitation member-email to match the prepared-register email, raising :email-does-not-match-invitation otherwise, mirroring the password flow. The OIDC callback verifies the team-invitation JWT and the email before redirecting to register instead of trusting raw presence. Covered by the requested mismatch regression test plus callback tests for mismatch and garbage tokens. AI-assisted-by: Muse Spark 1.3 Contributor * :recycle: Share registration gate between prepare and register-profile After the email-match fix, check-registration runs flags, email and invitation row in the same order as validate-register-attempt!. The prepare command now calls the shared helper instead of inlining the trio, and the stale docstring rationale is gone, so future invitation rule changes land in one place. AI-assisted-by: Muse Spark 1.3 Contributor --- .../auth-permissions-product-domains.md | 2 +- backend/src/app/auth/oidc.clj | 30 ++- backend/src/app/rpc/commands/auth.clj | 124 ++++++---- backend/test/backend_tests/auth_oidc_test.clj | 70 ++++++ .../test/backend_tests/rpc_profile_test.clj | 215 +++++++++++++++++- docs/technical-guide/configuration.md | 6 +- 6 files changed, 404 insertions(+), 43 deletions(-) diff --git a/.serena/memories/backend/auth-permissions-product-domains.md b/.serena/memories/backend/auth-permissions-product-domains.md index 9d7c5d6725..a4cba14fe5 100644 --- a/.serena/memories/backend/auth-permissions-product-domains.md +++ b/.serena/memories/backend/auth-permissions-product-domains.md @@ -12,7 +12,7 @@ - Lockout and RPC rate limits share the `app.http.errors/handle-error :rate-limit` HTTP path: status 429, body `{:type :rate-limit :code ... :hint ... :ttl ...}`, and any supplied `::http/headers` preserved. Account lockout raises `:code :account-locked` and, when it carries a non-nil `:ttl` (seconds), the handler adds `retry-after`. The RPC limiter (`app.rpc.rlimit`) raises `:code :request-blocked` and sets `retry-after` itself in `::http/headers` (seconds until the longest rejecting limit resets), alongside `x-rate-limit-remaining`/`x-rate-limit-reset`. CORS (`app.http.middleware/with-cors-headers`) exposes `content-type`, `retry-after`, and both `x-rate-limit-*` headers. External flags: `enable-account-lockout`, `enable-rpc-rlimit`. - Logout may return an OIDC provider redirect URI when the session claims include provider/session data and the provider has a logout URI. - Invitation tokens are verified through token issuers and only accepted when the token member id/email matches the authenticated profile; otherwise login proceeds without consuming the invitation. -- When public registration is disabled, invitation-based password registration also requires a matching, non-expired `team_invitation` row; registration revalidates it under a row lock before creating the profile. +- When public registration is disabled, invitation-based password registration also requires a matching, non-expired `team_invitation` row; registration revalidates it under a row lock before creating the profile. SSO registration (OIDC and compatible providers) follows the OIDC callback gate: it requires `:registration` or `:oidc-registration`, or a team invitation token issued for the registering email (an invitation substitutes the flags only when `member-email` matches the OIDC/`prepared-register` email via `clean-email`, as in the password flow, which raises `:email-does-not-match-invitation` otherwise). The OIDC callback verifies the invitation JWT and the email before redirecting to register, and `register-profile` revalidates the invitation under a row lock when no SSO flag is set; it never requires password login. Only OIDC-typed backends (`oidc-provider-types` in `app.auth.oidc`) take the SSO registration gate; any other `:backend` in a prepared-register token is rejected as `:invalid-token`. - HTTP/session parsing details such as cookie/header precedence, JWT session token versions, and SameSite behavior are in `mem:backend/subtleties`. ## Permission model diff --git a/backend/src/app/auth/oidc.clj b/backend/src/app/auth/oidc.clj index c4e79f8251..3d3daa0d8d 100644 --- a/backend/src/app/auth/oidc.clj +++ b/backend/src/app/auth/oidc.clj @@ -1062,6 +1062,32 @@ :organization-id organization-id :organization-name organization-name})))))) +(def oidc-provider-types + "Backends authenticating through an OIDC-compatible provider. + Only these backends take the SSO registration gate." + #{"oidc" "google" "github" "gitlab"}) + +(defn sso-registration-allowed? + "SSO registration is allowed when public registration or explicit + OIDC registration is enabled. Shared by the callback gate and the + `register-profile` token gate — keep one definition." + [] + (or (contains? cf/flags :registration) + (contains? cf/flags :oidc-registration))) + +(defn- valid-invitation-for? + "Returns true when `invitation-token` is a verifiable + `:team-invitation` JWT issued for `email`. Never throws: a garbage + token means there is no valid invitation." + [cfg invitation-token email] + (try + (boolean + (when (and (some? invitation-token) (some? email)) + (let [invitation (tokens/verify cfg {:token invitation-token :iss :team-invitation})] + (= (profile/clean-email (:member-email invitation)) + (profile/clean-email email))))) + (catch Throwable _ false))) + (defn- callback-handler [cfg {:keys [params] :as request}] (if-let [error (get params :error)] @@ -1094,8 +1120,8 @@ (redirect-with-error "email-domain-not-allowed") :else - (if (or (contains? cf/flags :registration) - (contains? cf/flags :oidc-registration)) + (if (or (sso-registration-allowed?) + (valid-invitation-for? cfg (:invitation-token info) (:email info))) (redirect-to-register cfg info provider) (redirect-with-error "registration-disabled"))) diff --git a/backend/src/app/rpc/commands/auth.clj b/backend/src/app/rpc/commands/auth.clj index 9a4d466f4b..899af9ec68 100644 --- a/backend/src/app/rpc/commands/auth.clj +++ b/backend/src/app/rpc/commands/auth.clj @@ -46,6 +46,10 @@ (def schema:token [::sm/word-string {:max 6000}]) +(def ^:private penpot-backend + "Auth backend identifier for the password flow." + "penpot") + (defn- elapsed-verify-threshold? [profile] (let [elapsed (ct/diff (:modified-at profile) (ct/now)) @@ -226,25 +230,72 @@ ;; ---- COMMAND: Prepare Register -(defn- validate-registration-flags! - ([cfg invitation] - (validate-registration-flags! cfg invitation false)) - ([cfg invitation lock-invitation?] - (when-not (contains? cf/flags :login-with-password) - (ex/raise :type :restriction - :code :registration-disabled - :hint "registration disabled")) +(defn- check-registration-flags + "The `:backend` is the one embedded in the verified + `:prepared-register` token: the password flow backend, one of the + `oidc/oidc-provider-types` backends for SSO flows, or any other + value, which is rejected loudly." + [backend invitation] + (cond + (= backend penpot-backend) + (do + (when-not (contains? cf/flags :login-with-password) + (ex/raise :type :restriction + :code :registration-disabled + :hint "registration disabled")) - (when-not (contains? cf/flags :registration) - (if (nil? invitation) - (ex/raise :type :restriction - :code :registration-disabled - :hint "registration disabled") - (let [opts (when lock-invitation? {::db/for-update true})] - (when (nil? (teams-invitations/active-invitation cfg invitation opts)) - (ex/raise :type :validation - :code :invalid-token - :hint "no active invitation associated with the token"))))))) + (when-not (or (contains? cf/flags :registration) + (some? invitation)) + (ex/raise :type :restriction + :code :registration-disabled + :hint "registration disabled"))) + + (contains? oidc/oidc-provider-types backend) + ;; SSO backends share the OIDC callback gate + ;; (`oidc/sso-registration-allowed?`): registration is allowed with + ;; `:registration` or `:oidc-registration`, without requiring + ;; password login. An invitation token is a substitute for the + ;; flags, as in the password flow. + (when-not (or (oidc/sso-registration-allowed?) + (some? invitation)) + (ex/raise :type :restriction + :code :registration-disabled + :hint "registration disabled")) + + :else + (ex/raise :type :validation + :code :invalid-token + :hint "unsupported registration backend"))) + +(defn- check-registration-invitation + "Verifies that the invitation is still active. No-op unless a live + invitation is required: a backend whose registration flags do not + allow public registration, with an invitation at hand." + [cfg backend invitation lock-invitation?] + (let [flag-allows? (if (= backend penpot-backend) + (contains? cf/flags :registration) + (oidc/sso-registration-allowed?))] + (when (and (some? invitation) + (not flag-allows?)) + (let [opts (when lock-invitation? {::db/for-update true})] + (when (nil? (teams-invitations/active-invitation cfg invitation opts)) + (ex/raise :type :validation + :code :invalid-token + :hint "no active invitation associated with the token")))))) + +(defn- check-registration + "Shared registration gate (register-profile and prepare-register): + validates the flags, the invitation email and the team invitation + row in order." + [cfg backend invitation email lock-invitation?] + (check-registration-flags backend invitation) + (when (and (some? invitation) + (not= (profile/clean-email (:member-email invitation)) + (profile/clean-email email))) + (ex/raise :type :restriction + :code :email-does-not-match-invitation + :hint "email should match the invitation")) + (check-registration-invitation cfg backend invitation lock-invitation?)) (defn- validate-register-attempt! [cfg params] @@ -254,23 +305,7 @@ {:token (:invitation-token params) :iss :team-invitation}))] - (when (or (not (contains? cf/flags :login-with-password)) - (and (not (contains? cf/flags :registration)) - (not invitation?))) - (ex/raise :type :restriction - :code :registration-disabled - :hint "registration disabled")) - - (when (and invitation? - (not= (profile/clean-email (:email params)) - (profile/clean-email (:member-email invitation)))) - (ex/raise :type :restriction - :code :email-does-not-match-invitation - :hint "email should match the invitation")) - - (when (and invitation? - (not (contains? cf/flags :registration))) - (validate-registration-flags! cfg invitation))) + (check-registration cfg penpot-backend invitation (:email params) false)) (when (and (email.blacklist/enabled? cfg) (email.blacklist/contains? cfg (:email params))) @@ -342,7 +377,7 @@ :fullname fullname :password (:password params) :invitation-token (:invitation-token params) - :backend "penpot" + :backend penpot-backend :iss :prepared-register :exp (ct/in-future {:days 7}) :props props} @@ -414,7 +449,7 @@ locale (when (and (string? locale) (not (str/blank? locale))) locale) - backend (:backend params "penpot") + backend (:backend params penpot-backend) is-demo (:is-demo params false) is-muted (:is-muted params false) is-active (:is-active params false) @@ -500,12 +535,25 @@ :token vtoken :extra-data ptoken})))) +(def ^:private schema:prepared-register-claims + [:map {:title "PreparedRegisterClaims"} + [:backend ::sm/text] + [:email ::sm/email]]) + +(def ^:private valid-prepared-register-claims? + (sm/lazy-validator schema:prepared-register-claims)) + (defn register-profile [{:keys [::db/conn] :as cfg} {:keys [token] :as params}] (let [claims (tokens/verify cfg {:token token :iss :prepared-register}) + _ (when-not (valid-prepared-register-claims? claims) + (ex/raise :type :validation + :code :invalid-token + :hint "prepared-register token contains unexpected data")) + backend (:backend claims) invitation (when-let [token (:invitation-token claims)] (tokens/verify cfg {:token token :iss :team-invitation})) - _ (validate-registration-flags! cfg invitation true) + _ (check-registration cfg backend invitation (:email claims) true) params (cond-> claims (:accept-newsletter-updates params) (update :props assoc :newsletter-updates true)) diff --git a/backend/test/backend_tests/auth_oidc_test.clj b/backend/test/backend_tests/auth_oidc_test.clj index 5f86bbfe50..03a0638d75 100644 --- a/backend/test/backend_tests/auth_oidc_test.clj +++ b/backend/test/backend_tests/auth_oidc_test.clj @@ -447,6 +447,76 @@ loc (redirect-location result)] (t/is (.contains loc "error=registration-disabled"))))))) +(t/deftest callback-with-invitation-and-no-flags-redirects-to-register + ;; Review of #12974: a verifiable invitation issued for the OIDC + ;; email acts as a substitute for the SSO registration flags, so the + ;; user reaches the register screen instead of the + ;; registration-disabled error. The JWT is verified and the email is + ;; compared here; the invitation row is revalidated under a row lock + ;; in `register-profile`. + (let [cfg (dissoc base-cfg :app.email/blacklist :app.email/whitelist) + invitation (tokens/generate cfg {:iss :team-invitation + :exp (ct/in-future {:hours 1}) + :member-email "u@e.com"}) + state (make-state-token cfg {:invitation-token invitation}) + request (default-request cfg :state state)] + (binding [cf/config {:public-uri "http://localhost:3449"} + cf/flags #{}] + (with-redefs [app.auth.oidc/resolve-provider (constantly {:type "oidc" :id "oidc"}) + app.auth.oidc/get-info (constantly {:email "u@e.com" :fullname "U" + :backend "oidc" :email-verified false + :invitation-token invitation + :props {}}) + app.auth.oidc/get-profile (constantly nil)] + (let [result (#'oidc/callback-handler cfg request) + loc (redirect-location result) + claims (tokens/decode cfg (location-token loc))] + (t/is (.contains loc "screen=auth-register-validate")) + (t/is (.contains loc "token=")) + (t/is (= :prepared-register (:iss claims)) + "the redirect token must be a :prepared-register token") + (t/is (= invitation (:invitation-token claims)) + "the invitation token must travel inside the prepared-register claims")))))) + +(t/deftest callback-with-invitation-email-mismatch-redirects-to-registration-disabled + ;; Review of #12974: an invitation issued for another email must not + ;; unlock registration with closed registration. + (let [cfg (dissoc base-cfg :app.email/blacklist :app.email/whitelist) + invitation (tokens/generate cfg {:iss :team-invitation + :exp (ct/in-future {:hours 1}) + :member-email "alice@example.com"}) + state (make-state-token cfg {:invitation-token invitation}) + request (default-request cfg :state state)] + (binding [cf/config {:public-uri "http://localhost:3449"} + cf/flags #{}] + (with-redefs [app.auth.oidc/resolve-provider (constantly {:type "oidc" :id "oidc"}) + app.auth.oidc/get-info (constantly {:email "attacker@example.com" :fullname "A" + :backend "oidc" :email-verified false + :invitation-token invitation + :props {}}) + app.auth.oidc/get-profile (constantly nil)] + (let [result (#'oidc/callback-handler cfg request) + loc (redirect-location result)] + (t/is (.contains loc "error=registration-disabled"))))))) + +(t/deftest callback-with-garbage-invitation-redirects-to-registration-disabled + ;; Review of #12974: raw presence is not enough — an unverifiable + ;; invitation string must not reach the register screen. + (let [cfg (dissoc base-cfg :app.email/blacklist :app.email/whitelist) + state (make-state-token cfg {:invitation-token "inv-123"}) + request (default-request cfg :state state)] + (binding [cf/config {:public-uri "http://localhost:3449"} + cf/flags #{}] + (with-redefs [app.auth.oidc/resolve-provider (constantly {:type "oidc" :id "oidc"}) + app.auth.oidc/get-info (constantly {:email "u@e.com" :fullname "U" + :backend "oidc" :email-verified false + :invitation-token "inv-123" + :props {}}) + app.auth.oidc/get-profile (constantly nil)] + (let [result (#'oidc/callback-handler cfg request) + loc (redirect-location result)] + (t/is (.contains loc "error=registration-disabled"))))))) + (t/deftest callback-profile-blocked (let [cfg (dissoc base-cfg :app.email/blacklist :app.email/whitelist) state (make-state-token cfg {}) diff --git a/backend/test/backend_tests/rpc_profile_test.clj b/backend/test/backend_tests/rpc_profile_test.clj index d2bc861ba0..ce59a3a2b1 100644 --- a/backend/test/backend_tests/rpc_profile_test.clj +++ b/backend/test/backend_tests/rpc_profile_test.clj @@ -954,6 +954,10 @@ (t/is (= :email-does-not-match-invitation (:code edata)))))) (t/deftest prepare-register-with-invitation-and-disabled-registration-and-wrong-email + ;; The email check runs before the invitation row lookup, so a + ;; mismatched email is reported even when the invitation row is gone. + ;; Flags stay `#{:login-with-password}`: password registration is + ;; allowed only via invitation. (with-redefs [app.config/flags #{:login-with-password}] (let [itoken (tokens/generate th/*system* {:iss :team-invitation @@ -961,7 +965,6 @@ :role :editor :team-id uuid/zero :member-email "user2@example.com"}) - data {::th/type :prepare-register-profile :invitation-token itoken :email "user@example.com" @@ -1013,6 +1016,216 @@ (t/is (= :registration-disabled (-> out :error ex-data :code))) (t/is (nil? (th/db-get :profile {:email email}))))))))) +(defn- create-oidc-register-token + "Mint a `:prepared-register` token shaped like the ones the OIDC + callback produces in `redirect-to-register` (no real provider needed)." + [email extra] + (tokens/generate th/*system* + (merge {:iss :prepared-register + :exp (ct/in-future "48h") + :backend "oidc" + :email email + :fullname "OIDC User" + :email-verified true + :props {}} + extra))) + +(t/deftest register-profile-allows-oidc-registration-with-password-login-disabled + ;; Issue #12964: with password login and public registration disabled, + ;; a new user arriving from OIDC must still be able to register when + ;; `oidc-registration` is enabled. + (with-redefs [app.config/flags #{:login-with-oidc :oidc-registration}] + (let [email "oidc-user@example.com" + token (create-oidc-register-token email nil) + out (th/command! {::th/type :register-profile + :token token}) + result (:result out) + mdata (meta result)] + (t/is (th/success? out)) + ;; A session is minted: the user lands on the dashboard. + (t/is (seq (:app.rpc/response-transform-fns mdata))) + (let [profile (th/db-get :profile {:email email})] + (t/is (some? profile)) + (t/is (= "oidc" (:auth-backend profile))) + (t/is (true? (:is-active profile)))) + ;; No password fallback while password login is disabled. + (let [login-out (th/command! {::th/type :login-with-password + :email email + :password "Whatever1!"})] + (t/is (not (th/success? login-out))) + (t/is (= :restriction (-> login-out :error ex-data :type))) + (t/is (= :login-disabled (-> login-out :error ex-data :code))))))) + +(t/deftest register-profile-rejects-oidc-without-registration-flags + (with-redefs [app.config/flags #{:login-with-oidc}] + (let [email "oidc-rejected@example.com" + token (create-oidc-register-token email nil) + out (th/command! {::th/type :register-profile + :token token})] + (t/is (not (th/success? out))) + (t/is (= :registration-disabled (-> out :error ex-data :code))) + (t/is (nil? (th/db-get :profile {:email email})))))) + +(t/deftest register-profile-allows-oidc-with-public-registration-and-password-login-disabled + (with-redefs [app.config/flags #{:login-with-oidc :registration}] + (let [email "oidc-public@example.com" + token (create-oidc-register-token email nil) + out (th/command! {::th/type :register-profile + :token token})] + (t/is (th/success? out)) + (let [profile (th/db-get :profile {:email email})] + (t/is (some? profile)) + (t/is (= "oidc" (:auth-backend profile))))))) + +(t/deftest register-profile-allows-oidc-with-invitation-and-registration-disabled + (let [email "oidc-invited@example.com" + fixture (create-invitation-fixture email (ct/in-future "48h"))] + (with-redefs [app.config/flags #{:login-with-oidc :oidc-registration}] + (let [itoken (create-test-invitation-token fixture email) + token (create-oidc-register-token email {:invitation-token itoken}) + out (th/command! {::th/type :register-profile + :token token}) + result (:result out) + mdata (meta result)] + (t/is (th/success? out)) + ;; Invitation branch: the token is echoed back and a session is + ;; minted so the frontend can complete the team-invitation flow. + (t/is (string? (:invitation-token result))) + (t/is (seq (:app.rpc/response-transform-fns mdata))) + (let [profile (th/db-get :profile {:email email})] + (t/is (some? profile)) + (t/is (= "oidc" (:auth-backend profile)))))))) + +(t/deftest register-profile-allows-oidc-with-canceled-invitation + ;; The invitation row is only consulted when no SSO registration + ;; flag is set: with `:oidc-registration` (as here), registration + ;; succeeds even when the invitation was canceled, and team + ;; membership fails later at `verify-token`. + (let [email "oidc-canceled@example.com" + fixture (create-invitation-fixture email (ct/in-future "48h"))] + (with-redefs [app.config/flags #{:login-with-oidc :oidc-registration}] + (let [itoken (create-test-invitation-token fixture email) + token (create-oidc-register-token email {:invitation-token itoken})] + (th/db-delete! :team-invitation + {:team-id (:id (:team fixture)) + :email-to email}) + (let [out (th/command! {::th/type :register-profile + :token token}) + result (:result out) + mdata (meta result)] + (t/is (th/success? out)) + (t/is (string? (:invitation-token result))) + (t/is (seq (:app.rpc/response-transform-fns mdata))) + (let [profile (th/db-get :profile {:email email})] + (t/is (some? profile)) + (t/is (= "oidc" (:auth-backend profile)))) + ;; The team is not joined: with the invitation row gone, + ;; accepting it fails. + (let [vout (th/command! {::th/type :verify-token + :token (:invitation-token result)})] + (t/is (not (th/success? vout))) + (t/is (= :validation (-> vout :error ex-data :type))) + (t/is (= :invalid-token (-> vout :error ex-data :code))))))))) + +(t/deftest register-profile-rejects-prepared-token-without-backend + ;; The `:backend` claim is mandatory: a prepared-register token + ;; without it is rejected loudly instead of falling into any + ;; registration branch. + (with-redefs [app.config/flags #{:login-with-oidc :oidc-registration}] + (let [token (tokens/generate th/*system* + {:iss :prepared-register + :exp (ct/in-future "48h") + :email "no-backend@example.com" + :fullname "No Backend" + :email-verified true + :props {}}) + out (th/command! {::th/type :register-profile + :token token})] + (t/is (not (th/success? out))) + (t/is (= :validation (-> out :error ex-data :type))) + (t/is (= :invalid-token (-> out :error ex-data :code))) + (t/is (nil? (th/db-get :profile {:email "no-backend@example.com"})))))) + +(t/deftest register-profile-rejects-non-oidc-backend-token + ;; The SSO registration gate only applies to backends of OIDC type; + ;; a token carrying any other backend is rejected loudly instead of + ;; silently taking the SSO branch. + (with-redefs [app.config/flags #{:login-with-oidc :oidc-registration}] + (let [token (tokens/generate th/*system* + {:iss :prepared-register + :exp (ct/in-future "48h") + :backend "ldap" + :email "unknown-backend@example.com" + :fullname "Unknown Backend" + :email-verified true + :props {}}) + out (th/command! {::th/type :register-profile + :token token})] + (t/is (not (th/success? out))) + (t/is (= :validation (-> out :error ex-data :type))) + (t/is (= :invalid-token (-> out :error ex-data :code))) + (t/is (nil? (th/db-get :profile {:email "unknown-backend@example.com"})))))) + +(t/deftest register-profile-allows-oidc-with-invitation-and-no-registration-flags + ;; Review of #12974: an invitation token is a substitute for the + ;; SSO registration flags, as it is for the password flow. With no + ;; `:registration` nor `:oidc-registration` flag, a user arriving + ;; through a team invitation can still register via OIDC. + (let [email "oidc-invited-noflags@example.com" + fixture (create-invitation-fixture email (ct/in-future "48h"))] + (with-redefs [app.config/flags #{:login-with-oidc}] + (let [itoken (create-test-invitation-token fixture email) + token (create-oidc-register-token email {:invitation-token itoken}) + out (th/command! {::th/type :register-profile + :token token}) + result (:result out) + mdata (meta result)] + (t/is (th/success? out)) + ;; Invitation branch: the token is echoed back and a session is + ;; minted so the frontend can complete the team-invitation flow. + (t/is (string? (:invitation-token result))) + (t/is (seq (:app.rpc/response-transform-fns mdata))) + (let [profile (th/db-get :profile {:email email})] + (t/is (some? profile)) + (t/is (= "oidc" (:auth-backend profile)))))))) + +(t/deftest register-profile-rejects-oidc-invitation-email-mismatch-with-no-registration-flags + ;; Review of #12974: the invitation is a substitute for the SSO + ;; registration flags only when it was issued for the registering + ;; email. A forwarded invitation for another email must not unlock + ;; registration with closed registration. + (let [invited-email "oidc-invited-other@example.com" + attacker-mail "oidc-attacker-noflags@example.com" + fixture (create-invitation-fixture invited-email (ct/in-future "48h"))] + (with-redefs [app.config/flags #{:login-with-oidc}] + (let [itoken (create-test-invitation-token fixture invited-email) + token (create-oidc-register-token attacker-mail {:invitation-token itoken}) + out (th/command! {::th/type :register-profile + :token token})] + (t/is (not (th/success? out))) + (t/is (= :restriction (-> out :error ex-data :type))) + (t/is (= :email-does-not-match-invitation (-> out :error ex-data :code))) + (t/is (nil? (th/db-get :profile {:email attacker-mail}))))))) + +(t/deftest register-profile-rejects-oidc-with-canceled-invitation-and-no-registration-flags + ;; Review of #12974: with no registration flag set, the invitation + ;; row is revalidated under a row lock before creating the profile. + ;; A canceled invitation stops signup and no account is created. + (let [email "oidc-canceled-noflags@example.com" + fixture (create-invitation-fixture email (ct/in-future "48h"))] + (with-redefs [app.config/flags #{:login-with-oidc}] + (let [itoken (create-test-invitation-token fixture email) + token (create-oidc-register-token email {:invitation-token itoken})] + (th/db-delete! :team-invitation + {:team-id (:id (:team fixture)) + :email-to email}) + (let [out (th/command! {::th/type :register-profile + :token token})] + (t/is (not (th/success? out))) + (t/is (= :validation (-> out :error ex-data :type))) + (t/is (= :invalid-token (-> out :error ex-data :code))) + (t/is (nil? (th/db-get :profile {:email email})))))))) + (t/deftest verify-token-redirects-invitation-based-on-member-id (with-redefs [app.config/flags #{:login-with-password}] (let [owner (th/create-profile* 1 {:is-active true}) diff --git a/docs/technical-guide/configuration.md b/docs/technical-guide/configuration.md index ebade88db7..9715bc2ce3 100644 --- a/docs/technical-guide/configuration.md +++ b/docs/technical-guide/configuration.md @@ -270,7 +270,11 @@ PENPOT_OIDC_USER_INFO_SOURCE: __Since version 2.1.2__ Allows users to register and login with oidc without having to previously -register with another method. +register with another method. It also works with password login disabled +(`disable-login-with-password`): OIDC registration only needs this flag, +not password login. An invitation link also works as a substitute for +this flag: an invited user can register with OIDC even with both +registration flags disabled. ```bash PENPOT_FLAGS: [...] enable-oidc-registration From d9a9d0c2bcf4bc2ffa563cb7be39682ddbb8b80d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bel=C3=A9n=20Albeza?= Date: Sat, 14 Nov 2026 18:16:55 +0100 Subject: [PATCH 07/15] :books: Update contributing guide with rules about auto-rejecting PRs (#13012) --- docs/.vscode/settings.json | 5 ----- docs/contributing-guide/code-contributions/index.njk | 7 ++++--- 2 files changed, 4 insertions(+), 8 deletions(-) delete mode 100644 docs/.vscode/settings.json diff --git a/docs/.vscode/settings.json b/docs/.vscode/settings.json deleted file mode 100644 index 1e915ede10..0000000000 --- a/docs/.vscode/settings.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "editor.rulers": [ - 80 - ] -} diff --git a/docs/contributing-guide/code-contributions/index.njk b/docs/contributing-guide/code-contributions/index.njk index 2c9ef6b517..4c62007edd 100644 --- a/docs/contributing-guide/code-contributions/index.njk +++ b/docs/contributing-guide/code-contributions/index.njk @@ -18,10 +18,11 @@ Thinking of contributing to Penpot core but not sure where to start? We’ve mad

Pull requests

If you want propose a change or bug fix with the Pull-Request system firstly you should carefully read the DCO section and format your commits accordingly.

-

If you intend to fix a bug it's fine to submit a pull request right away but we still recommend to file an issue detailing what you're fixing. This is helpful in case we don't accept that specific fix but want to keep track of the issue.

+

If you intend to fix a bug please make sure it's attached to an existing issue. Pull requests that have no issue attached will be automatically rejected. Pull requests attached to issues labeled as needs triage or internal will be autorejected as well.

If you want to implement or start working in a new feature, please open a question / discussion issue for it. No pull-request will be accepted without previous chat about the changes, independently if it is a new feature, already planned feature or small quick win.

+

Please do not submit a pull request for an issue that has already been assigned to someone else. It there is no recent activity, feel free to ask for it be assigned to you.

If is going to be your first pull request, You can learn how from this free video series.

-

We will use the easy fix mark for tag for indicate issues that are easy for beginners.

+

We will use the good first issue label to mark issues that are easy to fix for beginners.

Commit message guidelines

We have very precise rules over how our git commit messages can be formatted.

@@ -35,7 +36,7 @@ Thinking of contributing to Penpot core but not sure where to start? We’ve mad [footer] -

Where type is:

+

Where type is:

  • bug :bug: a commit that fixes a bug
  • sparkles :sparkles: a commit that adds an improvement
  • From e23ca02aa20ec0bfef31c3efeec57025191def5b Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Sat, 14 Nov 2026 19:27:18 +0100 Subject: [PATCH 08/15] :sparkles: Reference the file data storage object with a foreign key (#12769) * :sparkles: Reference the file data storage object with a foreign key The file data pointer lived in `file_data.metadata.storage-ref-id`, a JSON key. The storage GC parsed it to find live references, and nothing stopped it from physically deleting an object a live row still pointed at: a failure in that scan lost the blob. Migration 0159 moves the pointer to a nullable `storage_object_id` column on `file_data`, backfills it from the metadata key, removes the key from the rows it backfills, indexes the column and rebuilds the foreign key to `storage_object(id)` as `NO ACTION DEFERRABLE INITIALLY IMMEDIATE`. `INITIALLY IMMEDIATE` is deliberate: the GC catches the violation inside the chunk transaction, while a deferred check would fire at commit, outside it, and the sweep would never move past the blocked chunk. The column is the only source of truth. The `storage-ref-id` entry is gone from the file metadata schema, and the single point that serializes the metadata drops it, so no write can resurrect it. `upsert!` does not query the row: the pointer of the row being replaced travels in its params, and the object is touched before the row is written. Fragments and snapshots pass nil because a modified pointer always gets a new id; the offload passes its own row; `insert-file!` drops it, since an INSERT never replaces a persisted object. `update-file!` takes the file map read from the database plus the fields to change and merges them, so the pointer comes from the read map and no caller can lose it by building a partial map. A map without `:data` raises `:missing-file-data`: it is not `assert`, which is compiled out in the deployed binary. The params cannot override the pointer or retarget the write with `:id`/`:file-id`. `bfc/file->response` strips the pointer from every response (`get-file`, `create-file`, `duplicate-file`, the view-only bundle, the team access request, the comments read, the library summary, the snapshot read and the debug download). A `file-data` object is authentication-only, so a leaked id would let any logged-in profile fetch the blob from `/assets/by-id`. `get-file-fragment` resolves through the file data backend, so a fragment on cold storage no longer hands nil to `blob/decode`, and answers only the keys of its result schema. The storage GC reference scan, the restore union and `objects-gc` read the column. The offload reads `SELECT fd.*`, so its rows go through a `decode-row` that decodes jsonb metadata and drops a JSON null, which the row schema rejected; a row without a readable object raises `:missing-storage-object` with the file and row ids. A chunk blocked by the foreign key (SQLState 23503) is a real anomaly, since the reference scan should have frozen the object: it stays an error with its own hint and the chunk is retried on the next run until the reference is fixed. The release is a coordinated restart, not a rolling deploy, and not rollback-safe for the rows it writes: the previous code writes and scans the metadata pointer, so a key it writes misses the backfill and removing it loses the reference. `operations/2.20.0-file-data-storage-object-id.md` builds the column, the index and the foreign key online and runs the backfill with every instance stopped, then marks the step as applied; a fresh or small instance does the whole job in the migration. The runbook keeps the validation scan out of the stopped window and adds a lock timeout and retry to the online statements. Tests cover the foreign key shape, the backfill and its corrupt and orphan guards, the rebuild of a missing index and of a wrong, non-deferrable or deferred constraint, the deletion refused by a live reference, the metadata cleanup, the offload row normalizations, the touched objects of the unshare and the restore, the response stripping on every call site and the chunk blocked by the foreign key. The backend memories and the developer guide document the pointer, the GC behavior and the two migration paths. Closes #12768 AI-assisted-by: deepseek-v4.1-flash AI-assisted-by: inkling-2-small-free * :bug: Hold the file row lock across the migration write `get-file` applies the pending migrations in memory and, when it finds any, persists the result. That write read the row without a lock and `update-file!` is an unconditional UPDATE, so a save that committed in between was overwritten: the saved content, `revn` and `modified-at` went back, the next save from that client failed with `:revn-conflict` and the object the save left became an orphan the storage GC cannot revisit once it is frozen. The read now runs in a transaction of its own (`get-file` calls `get-file*` inside `db/tx-run!` and accepts a cfg map only), so the `file` row lock covers the re-check and the write. `get-file*` reads the row once (`sql:get-file` plus `resolve-applied-migrations`) and completes it with `attach-file-data`; only when a migration is pending it reads the file again under `FOR UPDATE`, re-checks `need-migration?` on that fresh map and persists it (`migrate-file`). The common case stays at one row read and one migrations resolution, the migration case keeps a single decoded copy in the heap, and a row that changed in between is not overwritten. A migration found on the unlocked read is applied in memory only, for the next read to persist. `migrate-file` returns the reattached map when it persists, with the `:storage-object-id` of the row the write left, and the realized map when it does not; `:realize?` is what returns the read fully plain. `fdata/upsert!` answers `{:written ... :storage-object-id ...}` instead of a boolean, `update-file!` propagates it and the migrating read puts it back in the map it returns, so a caller that persists again touches the current object instead of the replaced one. `insert-file!` keeps nil: an INSERT never replaces a persisted object. The write also compares the stored pointer with the caller's value (`IS NOT DISTINCT FROM`) and raises `:stale-file-data-pointer` when it affects no row. Every writer holds the `file` row lock, so a pointer that no longer matches is a protocol violation, not a race to retry. The expected pointer is a separate argument because the storage branch replaces `:storage-object-id` before the upsert, and the check lives in `upsert-in-database`: the `legacy-db` branches do not use that SQL and their count can legitimately be zero. The save, snapshot and restore-backup reads take `FOR UPDATE` instead of `FOR SHARE`: they inherit the lock, and the migration branch asks for the exclusive one in the same transaction, so a shared lock that upgrades deadlocks two concurrent readers. A data read rejects `:lock-for-share?` for the same reason; only row-only reads accept it. `sql:get-file-without-data` is a plain select now, not a CTE, which makes locking legal without loading the data. Realization has a single owner: `attach-file-data` only resolves and decodes, and `get-file*` realizes once per path. The applied migrations no longer travel in the `get-file-with-summary` response, cache hit and cache miss alike. `embed-assets` reads each library without `:realize?`, so its `:components` is an unloaded pointer map; `absorb-assets` walks it with `vals`, `PointerMap.seq` calls `load!` and that raised with no loader bound, so every export with `:merge-libraries` of a library with components failed. The absorb runs under the library's own loader now. The in-place binfile import (`import-file-and-overwrite*`, `::overwrite`, `::bfc/file-id`) had no product caller and cannot carry the previous `storage_object_id`, so overwriting a storage-backed row would leave its object unreclaimed. `import-files!` rejects a cfg with `::bfc/file-id` (`:not-supported`) and the helpers only that path used go with it (`invalidate-thumbnails`, `*reference-file*`, the `::overwrite` id branch of `index-object` and of `save-file!`, the `on-conflict-do-nothing?` flag of the media insert). Tests: a new `file-migration-read-test` pins the single read, the locked re-read, the concurrent save interleaving, the double migration and the read-only path; the upsert tests cover the return shape and the stale pointer; the SQL recorder they share lives in `backend-tests.helpers`, and the save, snapshot, restore and summary sequences follow the new reads. The backend memories document the read contract and the pointer. AI-assisted-by: deepseek-v4.1-flash * :bug: Harden the get-file guard and skip its read-only transaction The cfg guard of `get-file` was an `assert`, which `app.config` elides when `:backend-asserts` is off, so a deployed binary accepted a bare pool or connection while the docstring promised it raises. It throws `IllegalArgumentException` now, the class the option guards of the same namespace already use, and the existing test expects it instead of `AssertionError`. A read with `:read-only?` applies the pending migrations in memory and writes nothing, so the transaction the migration write needs only costs two round trips there. `get-file` runs it with `db/run!` instead: with a cfg that carries a connection it still joins the caller's transaction, and only the reads that would have opened their own skip it. A lock option needs a read-write transaction from the caller, because PostgreSQL rejects row locks in a read-only one; the docstring states that. Tests: the guard test expects the new class; a cfg marked `::db/read-only` reads a pending migration without persisting it (the config flag that fills that key is legacy, but the key is set by hand); and a read-only read is pinned as not opening a transaction while the same call without the option does. The backend memory documents the split and the guard. AI-assisted-by: deepseek-v4.1-flash --- .serena/memories/backend/storage.md | 19 +- .serena/memories/backend/subtleties.md | 8 +- .serena/memories/prod-infra/core.md | 8 +- backend/src/app/binfile/common.clj | 401 ++++++++++++------ backend/src/app/binfile/v3.clj | 63 +-- backend/src/app/features/fdata.clj | 245 ++++++----- backend/src/app/features/file_snapshots.clj | 16 +- backend/src/app/http/debug.clj | 4 +- backend/src/app/migrations.clj | 5 +- .../0159-add-file-data-storage-object-id.sql | 65 +++ backend/src/app/rpc/commands/comments.clj | 5 +- backend/src/app/rpc/commands/files.clj | 51 ++- backend/src/app/rpc/commands/files_create.clj | 5 +- .../src/app/rpc/commands/files_snapshot.clj | 21 +- backend/src/app/rpc/commands/files_update.clj | 10 +- backend/src/app/rpc/commands/management.clj | 4 +- .../app/rpc/commands/teams_invitations.clj | 3 + backend/src/app/rpc/commands/viewer.clj | 8 +- backend/src/app/srepl/helpers.clj | 2 +- backend/src/app/storage/gc_deleted.clj | 44 +- backend/src/app/storage/gc_touched.clj | 9 +- backend/src/app/tasks/objects_gc.clj | 9 +- backend/src/app/tasks/offload_file_data.clj | 48 ++- backend/test/backend_tests/binfile_test.clj | 168 ++++++-- .../test/backend_tests/fdata_upsert_test.clj | 83 +++- backend/test/backend_tests/file_data_test.clj | 323 ++++++++++++++ .../file_migration_read_test.clj | 365 ++++++++++++++++ .../file_pointer_snapshot_test.clj | 7 +- backend/test/backend_tests/helpers.clj | 17 +- .../test/backend_tests/http_debug_test.clj | 27 ++ .../test/backend_tests/migrations_test.clj | 217 ++++++++++ .../test/backend_tests/objects_gc_test.clj | 28 ++ .../backend_tests/offload_file_data_test.clj | 111 +++-- .../restore_storage_objects_test.clj | 6 +- .../test/backend_tests/rpc_comment_test.clj | 26 ++ .../backend_tests/rpc_file_snapshot_test.clj | 69 ++- backend/test/backend_tests/rpc_file_test.clj | 169 +++++++- .../backend_tests/rpc_management_test.clj | 33 ++ backend/test/backend_tests/rpc_team_test.clj | 7 +- .../test/backend_tests/rpc_viewer_test.clj | 35 ++ .../backend_tests/storage_gc_deleted_test.clj | 73 ++++ common/src/app/common/types/file.cljc | 1 - docs/technical-guide/developer/backend.md | 46 ++ .../2.20.0-file-data-storage-object-id.md | 285 +++++++++++++ 44 files changed, 2695 insertions(+), 454 deletions(-) create mode 100644 backend/src/app/migrations/sql/0159-add-file-data-storage-object-id.sql create mode 100644 backend/test/backend_tests/file_data_test.clj create mode 100644 backend/test/backend_tests/file_migration_read_test.clj create mode 100644 operations/2.20.0-file-data-storage-object-id.md diff --git a/.serena/memories/backend/storage.md b/.serena/memories/backend/storage.md index 0a9ca8e24c..678fa539d0 100644 --- a/.serena/memories/backend/storage.md +++ b/.serena/memories/backend/storage.md @@ -93,7 +93,7 @@ Since `put-object!` uses backend-specific operations (`impl/resolve-backend` + ` | `organization` | Organization logos uploaded by the Nitrate management API. | Yes | Public | No reference scan. A touched object is deleted. | | `tempfile` | Export files and temporary font downloads. | No | Authentication required | No reference scan. A touched object uses a two-hour deletion delay. | | `upload-session` | Chunked-upload chunks. References: `upload_session_chunk.object_id` and `upload_session_chunk.session_id` (both NO ACTION DEFERRABLE: restrict semantics, procedural deletion). | No | Authentication required | Reference scan on `upload_session_chunk.object_id`; a referenced chunk freezes, an orphan is deleted after the delay. Deferred rows keep their mappings until give-up. | -| `file-data` | Encoded file data when `file-data-backend` is `storage`. Reference metadata has `storage-ref-id`, `file-id`, and the `file_data` row ID. | Yes | Authentication required | Reference scan. | +| `file-data` | Encoded file data when `file-data-backend` is `storage`. Reference: the `file_data.storage_object_id` column (FK `NO ACTION DEFERRABLE INITIALLY IMMEDIATE`); the object metadata keeps `file-id` and the `file_data` row ID. | Yes | Authentication required | Reference scan. | | `file-data-fragment` | Compatibility value for file-data fragments. The current backend has no dedicated producer for this bucket. | No current write semantics | Public | No touched-object collector case. | | `file-change` | Compatibility value for file changes. Current snapshots store data in `file_data`, not this bucket. | No current write semantics | Authentication required | No touched-object collector case. | | `job-resource` | Storage objects owned by job rows (`job.resource_id`), written by `app.jobs.storage`. | No | Authentication required, owner-scoped | jobs-GC touch → reference scan. | @@ -121,15 +121,22 @@ Since `put-object!` uses backend-specific operations (`impl/resolve-backend` + ` - `file-data-backend` accepts `legacy-db`, `db`, or `storage`. - `legacy-db` stores main data in `file.data` and snapshots in `file_change.data`. - `db` stores encoded data in `file_data.data`. -- `storage` stores encoded data in storage subsystem with `file-data` bucket and keeps `data` nil in `file_data` table. -- The `file_data.metadata.storage-ref-id` value points to the storage object. -- `fdata/upsert!` touches a storage object from incoming metadata before it stores the new row. +- `storage` stores encoded data in storage subsystem with `file-data` bucket, keeps `data` nil in `file_data` table, and points at the object with `file_data.storage_object_id`. +- `file_data.storage_object_id` is a `NO ACTION DEFERRABLE INITIALLY IMMEDIATE` foreign key to `storage_object(id)`: the database refuses to physically delete a referenced object. The `IMMEDIATE` is load-bearing: the RI violation fires inside the `storage-gc-deleted` chunk transaction, so the chunk rolls back, is logged at error level with a generic hint plus the `constraint` and `table` the database reports (any `NO ACTION` foreign key on `storage_object` can block the delete, not only `file_data`'s) and is retried on the next run; with `INITIALLY DEFERRED` the check would fire at commit, outside the catch, and the sweep would never move past the blocked chunk. +- The column is the only source of truth. The release is a coordinated restart (no rolling deploy) when the instance uses the storage backend (`PENPOT_FILE_DATA_BACKEND=storage`) or still has rows carrying `metadata.storage-ref-id`; with the default `db` backend and no such rows the previous and the new code read the same `file_data.data` column, so a rolling deploy is safe. `metadata.storage-ref-id` is no longer written or read, and the `file-data` metadata schema does not declare it. The migration backfills the rows it can resolve (valid uuid with an existing `storage_object`) and removes the key from those rows; a corrupt or orphan pointer keeps the key, which nothing reads. A large instance runs `operations/2.20.0-file-data-storage-object-id.md`: column, index and FK online, then the backfill (only if a row still carries `metadata.storage-ref-id`) with every instance stopped, verifies, and marks the migration as applied (`INSERT INTO migrations`). The backfill must never run while the previous code serves when the storage backend is in use (or rows with the pointer remain), because that code writes and scans the metadata pointer. A rollback to the previous code cannot read the rows written by this release when the storage backend is in use. +- `fdata/upsert!` touches the `storage_object_id` it receives in the params (the pointer the persisted row had) before replacing it, so the replaced blob is reclaimed. Callers pass the column from the file map (main/restore), `nil` for a new row (fragments, snapshots, new files) or the row itself (offload); there is no query in the upsert path, which is why every caller reads the file under the row lock first (the migration-on-read path included, see `mem:backend/subtleties`). It returns `{:written :storage-object-id }`, the pointer of the object the write created (nil for `db`/`legacy-db`), and `binfile.common/update-file!` propagates it so a migrating read returns the map with the pointer of the row it persisted instead of the pre-write one. The write also compares the expected pointer with `IS NOT DISTINCT FROM` (the caller's value, not the new one) and raises `:stale-file-data-pointer` when the row changed under it, so a stale map fails instead of leaking the replaced blob; the check lives in `upsert-in-database` because the `legacy-db` branches do not use the upsert SQL. +- `binfile.common/update-file!` takes the file map read from the database plus the fields to change (`[cfg file & params]`), merges them and builds the `file_data` params from the merged map, so the pointer travels in the read map and a caller cannot lose it by building a partial map; the params cannot override `:storage-object-id`. `insert-file!` always drops it (an INSERT never replaces a persisted pointer; `duplicate-file` hands the source map there). `bfc/get-file` keeps the pointer for the write paths, and every response goes through `bfc/file->response`, which strips it and any residual `storage-ref-id` left in the metadata of the rows the migration does not backfill: each `file-data` object is authentication-only and any logged-in profile could otherwise fetch it from `/assets/by-id`. +- The in-place binfile import was removed: `import-files!` raises `:not-supported` when the cfg carries `::bfc/file-id`, so no caller can overwrite a storage-backed row without carrying its previous pointer. - File snapshots use `file_data` for snapshot data and `file_change` for snapshot metadata. -- Tiered offload (`:tiered-file-data-storage` flag + `db` backend): `file-gc` submits an `:offload-file-data` job (priority 10, delay 1000, same params) only when it actually processed the file; the handler takes the `file` row with `FOR UPDATE SKIP LOCKED` and skips (`{:skipped 1}`, no error) when the row is locked or gone, then moves every live `file_data` row of that file with `backend = 'db'` to `backend = 'storage'` via `fdata/upsert!` (one `put-object!` in bucket `file-data` per row, `data` becomes nil) inside a single all-or-nothing transaction per file. The skip only serializes because every `file_data` writer holds at least `FOR SHARE` on the `file` row. A storage failure aborts only that file and bubbles for a runner retry; orphans go to `storage-gc-touched`. A second run is a no-op because the query only matches `backend = 'db'`. +- Tiered offload (`:tiered-file-data-storage` flag + `db` backend): `file-gc` submits an `:offload-file-data` job (priority 10, delay 1000, same params) only when it actually processed the file; the handler takes the `file` row with `FOR UPDATE SKIP LOCKED` and skips (`{:skipped 1}`, no error) when the row is locked or gone, then moves every live `file_data` row of that file with `backend = 'db'` to `backend = 'storage'` via `fdata/upsert!` (one `put-object!` in bucket `file-data` per row, `data` becomes nil and `storage_object_id` points at the new object) inside a single all-or-nothing transaction per file. The skip only serializes because every `file_data` writer locks the `file` row first (the update RPC and the snapshot paths with `FOR UPDATE` through `lock-for-update?`, the migration-on-read path with `FOR UPDATE`). A storage failure aborts only that file and bubbles for a runner retry; orphans go to `storage-gc-touched`. A second run is a no-op because the query only matches `backend = 'db'`. +- The offload normalizes each `SELECT fd.*` row before `upsert!`: the row builder omits NULL columns, jsonb metadata arrives as a `PGobject`, and a JSON `null` decodes to nil and must disappear instead of travelling as `:metadata nil`, which the update schema rejects. - Offload pacing: none by deliberate decision. Sleeps only make sense when batching multiple transactions; the per-file transaction is short by construction and the scheduler run takes minutes per production metrics, so neither loop sleeps. `jobs/heartbeat` (pool-based liveness, ~1 write/s) stays on both paths. - Offload metrics (bounded labels only, never ids or error text): `penpot_file_data_offload_rows_total{type}` and `penpot_file_data_offload_bytes_total{type}` (`type`: `main`/`snapshot`/`fragment`, successful runs only) plus `penpot_file_data_offload_timing_ms{result}` (`result`: `ok`/`skip`/`error`). Helpers live in `app.jobs.metrics`; the job-def requires `::mtx/metrics`. - Writes always resolve the backend through `fdata/default-backend` (`cf/get :file-data-backend`), so post-offload edits land back in hot (`db`) storage without extra code; reads stay transparent through `fdata/resolve-file-data` + `decode-file-data` + `realize`. -- Deleting an offloaded row touches its storage object first (`objects-gc/delete-file-data-chunk` via `fdata/process-metadata`), so `storage-gc-touched` reclaims the blob with the usual delay; restoring a file unmarks the referenced objects. +- Deleting an offloaded row touches its storage object first (`objects-gc/delete-file-data-chunk` via `fdata/touch-storage-object` with the row's `storage_object_id`), so `storage-gc-touched` reclaims the blob with the usual delay; restoring a file unmarks the referenced objects. +- `storage-gc-touched` resolves the `file-data` reference with `SELECT EXISTS (SELECT 1 FROM file_data WHERE storage_object_id = ?)`, an indexed lookup that no longer parses the object metadata. +- A row with `backend='storage'` and no readable object makes `resolve-file-data` raise `:missing-storage-object` with the file and row ids instead of failing later in the storage backend. +- The `file-id` and row id kept in the storage object metadata are informational for diagnostics; nothing parses them to resolve the reference (the reference is the column). ## File Change diff --git a/.serena/memories/backend/subtleties.md b/.serena/memories/backend/subtleties.md index 5976fac9ad..b1e0c2f1fb 100644 --- a/.serena/memories/backend/subtleties.md +++ b/.serena/memories/backend/subtleties.md @@ -91,17 +91,17 @@ ## File data persistence -- File data backends are `legacy-db`, `db`, and `storage`. The storage backend keeps encoded file data in storage bucket `file-data`; the DB row stores metadata with `storage-ref-id` and nil data. -- `fdata/upsert!` touches any storage object referenced by incoming metadata before storing the new row/blob. +- File data backends are `legacy-db`, `db`, and `storage`. The storage backend keeps encoded file data in storage bucket `file-data`; the DB row stores nil data and points at the object with the `storage_object_id` column (FK `NO ACTION DEFERRABLE`). +- `fdata/upsert!` touches the storage object the persisted row pointed at before replacing it; the pointer travels in the params (`file->file-data-params` from the read file map, `nil` for a new row, the row for the offload) and the params of `update-file!` cannot override it. `binfile.common/update-file!` merges the read file map with the fields to change, so the pointer survives; `insert-file!` drops it because an INSERT never replaces a persisted object. The column is the only source of truth and the metadata no longer carries `storage-ref-id` (the migration removes it from the backfilled rows and the write path never stores it); the release is a coordinated restart when the storage backend is in use (the previous code cannot read the new rows), while the default `db` backend reads the same data column and tolerates a rolling deploy. Responses built from a file map go through `bfc/file->response`, which strips the pointer and a residual `storage-ref-id` from the metadata. - Pointer-map fragments are persisted separately as type `fragment`, and only modified pointer maps are written. A pointer keeps its id while nothing changes; content (`assoc`/`without`) and metadata (`with-meta`) changes produce a fresh id, so a fragment row is never rewritten after it is written. Re-assoc with an identical value returns the same map in array and hash maps alike, so a save keeps the fragment id of every page it does not change. - `enable-objects-map` skips unloaded pointer pages: their fragment was already wrapped when it was written, and loading it only to wrap it again costs one page read per untouched page on every save. A page with a plain fragment is wrapped when it is first loaded or edited; a data migration that enables `fdata/objects-map` must rewrite the data itself. - Snapshots share fragment rows with the live file (they are stored with pointers, not materialized). `file-gc` marks every fragment referenced neither by the current main file nor by an unpurged snapshot; the file is read unrealized and unchanged pages keep their ids via reattach, so a pass over an unchanged file writes no new fragments. - A save attaches its automatic snapshot before applying pending file migrations: the snapshot keeps the pre-migration pointers, data and migration set, and restoring it leaves the next read to migrate the file. This also holds for files without the feature. The preview (`get-file-snapshot`) applies the pending migrations in memory with the live file's libraries and never persists them: the stored version keeps its data, the payload carries the snapshot's `:migrations` (an ordered set) and `fdata/objects-map` is realized for clients that do not declare it. The historical feature set is deliberately not checked against the enabled one: a version may predate a feature change. The window is narrow: the `get-file` RPC reads with `migrate?` on and its ETag hashes `available-migrations`, so a post-deploy read cannot be a 304 from the pre-deploy state and migrates before any save in the normal flow; the pre-migration snapshot only appears when the save is the first request to touch the file after the deploy (an open tab that saves without re-reading, or a non-UI writer). `create-file-snapshot` and the restore backup read with `migrate?` on, so those versions are stored already migrated. - `:migrations` is always a `d/ordered-set`: `file-migrations/resolve-applied-migrations` builds one, `fmg/migrate` re-wraps the union after applying, and `file-snapshots/decode-snapshot` converts the `file_change` pgarray (as a plain vector, `set/difference` and `contains?` would compare indexes instead of names). A snapshot with NULL `:migrations` and an old `:version` predates the `file_change.migrations` column (SQL migration 0140): the preview regenerates the set from the stored version and applies that full chain, while `restore!` uses a fixed 67 baseline, so preview and restore can disagree for very old snapshots. -- The migration read path (`binfile.common/migrate-file`) captures `fdata/original-pointers` before realizing, reattaches unchanged positions after applying the migrations, and persists the reattached file; it returns the realized file because reads expose plain data. A migration that adds `fdata/objects-map` skips the reattach so the pages get wrapped. +- The migration read path (`binfile.common/get-file*`) reads the `file` row once (`sql:get-file`, with its data columns; with the storage backend the blob lives in the object, not in the column) plus an explicit `resolve-applied-migrations`, because a row read does not resolve them. A read with no pending migration is completed by `attach-file-data` on that map: one row read and one migrations resolution. Only when a migration is pending is the row re-read under `FOR UPDATE` on the `file` row, where `need-migration?` is re-checked on the fresh map and the migrated file persisted (`binfile.common/migrate-file`); that re-read resolves the migrations again, and `migrate-file` once more after persisting, to return the applied set. The migration path holds two raw copies of the blob (the first read and the locked re-read) but a single decoded one: the alternative, a row-only probe before the read, avoided the first raw copy at the cost of an extra row read on every open, and was removed. `get-file` runs the read inside `db/tx-run!` (joining the caller's transaction when the cfg carries a connection) so the persist commits with the read; a read-only read (`:read-only?`) takes no lock and writes nothing, so it runs on a plain connection (`db/run!`) instead of opening a transaction of its own, and with a cfg that already carries a connection it still joins the caller's. It accepts a cfg map only and throws an `IllegalArgumentException` otherwise (`assert` is elided without `:backend-asserts`); a data read rejects `:lock-for-share?` (only row-only reads accept it), because the migration branch re-reads with `FOR UPDATE`, and a lock option needs a read-write transaction from the caller because PostgreSQL rejects row locks in a read-only transaction. A row-only read (`load-data?` false) resolves the applied migrations by default too (`:load-migrations?`), because the row can be completed later and the caller may need them; a caller that does not want them passes `:load-migrations? false`. Persisting is an optimization of the read (the read already returns migrated data): a row that changed between the first read and the locked read is not overwritten, and a pending migration discovered on the unlocked read is applied in memory only (the next read persists it under the lock). This was the only writer of the file row outside the lock protocol (save, library absorption, restore, `file-gc` and offload already take it); without it, a save that committed in between was overwritten by the stale map (lost content, `revn` moving back so the next save fails with `:revn-conflict`, and the object the save left as a frozen orphan). `migrate-file` captures `fdata/original-pointers` before realizing, reattaches unchanged positions after applying the migrations, and persists the reattached file; it returns that reattached map when it persists (the pages the migration does not change keep their pointers and fragment ids) and the realized map when it does not, so a read is plain only when the caller asks with `:realize?`, and a migration that adds `fdata/objects-map` skips the reattach so the pages get wrapped. - `fdata/load-pointer` reads rows that a previous `file-gc` pass marked on purpose (explicit `::db/remove-deleted false`): the fragment is only gone once `objects-gc` purges it. The `get-file-fragment` RPC keeps rejecting marked rows. - Bind the fragment loader through `fdata/with-pointers` (reads), `fdata/with-tracked-pointers` (writes with a loader) or `fdata/with-tracked` (writes with nothing to load yet, e.g. file creation), and `fdata/with-untracked` (validation passes that must track nothing) instead of hand-rolled `binding` forms. Two direct bindings remain on purpose: the fine-grained tracked toggles of `app.rpc.commands.files-update` (validation/repair turn tracking off, restore the existing atom and read it before repairing), and `binfile.common/encode-file` (a fresh tracked set inheriting the current one). Converge them only if atom-scoped macros (`with-tracking`, `with-inherited-tracking`) are added; today the extra indirection buys nothing. -- Snapshot creation takes `FOR SHARE` on the file row: both `create-file-snapshot` and the backup read of `restore-file-snapshot` hold it from the read to the insert, so they serialize with a `file-gc` pass (`FOR UPDATE ... SKIP LOCKED`) instead of pinning fragments the pass is about to mark. `get-file-snapshot` realizes the snapshot pointers with the live file id (fragment rows are keyed by file, not by snapshot). +- Snapshot creation takes `FOR UPDATE` on the file row: both `create-file-snapshot` and the backup read of `restore-file-snapshot` hold it from the read to the insert, so they serialize with a `file-gc` pass (`FOR UPDATE ... SKIP LOCKED`) and with a concurrent save, and never pin fragments the pass is about to mark. The lock is exclusive and not `FOR SHARE` because the migration-on-read path may re-read the same row with `FOR UPDATE`: a shared lock that asks for the exclusive one deadlocks two concurrent readers (one creating a snapshot, the other saving, both seeing a pending migration). `get-file-snapshot` realizes the snapshot pointers with the live file id (fragment rows are keyed by file, not by snapshot). - `fdata/realize` combines pointer realization and object-map realization. Use it before operations that need complete in-memory file data instead of pointer placeholders. - A `file_data` row is one of three classes, told apart by `type`: `main` is the file's current state and carries `id = file_id`; `snapshot` is a point-in-time copy whose `id` is the one its `file_change` row shares; `fragment` is a piece of the pointer-map, referenced from the file's data. - Their lifecycles differ: `main` and `snapshot` live with the file (dated when it is deleted, undated when it is restored), while a `fragment` carries no date and is collected by `file_gc` once the file stops referencing it. Neither is a session buffer, so do not read a `fragment` as an ephemeral revision. diff --git a/.serena/memories/prod-infra/core.md b/.serena/memories/prod-infra/core.md index d8ced132ca..09a58d9784 100644 --- a/.serena/memories/prod-infra/core.md +++ b/.serena/memories/prod-infra/core.md @@ -43,7 +43,13 @@ Production runs the deployed toolbox, not the devenv one: - **Start from the ids, not from an inventory.** The ids come from the error or the logs. Unbounded queries over tables with tens of millions of rows (`storage_object`, `file_media_object`) do not finish in a reasonable time; bounded to an id list they are index lookups. - **One query per reference column.** Two columns in one `IN (col_a, col_b)` lose the index and scan the whole table; split them. - The read-only replica cuts long queries: see `mem:prod-infra/grafana`. -- A migration that builds an index runs inside the migration transaction, so it cannot use `CONCURRENTLY`, and a plain `CREATE INDEX` locks the table against writes for the whole build. On a large table (file saves write to `file_change` on every save) build the index out of band, before deploying, with `CREATE INDEX CONCURRENTLY` on its own (never inside a transaction block); the shipped migration uses `IF NOT EXISTS`, finds it and does nothing. A concurrent build interrupted midway leaves the index invalid: drop it and retry (check `pg_indexes` and `pg_index.indisvalid`). +- A migration that builds an index runs inside the migration transaction, so it cannot use `CONCURRENTLY`, and a plain `CREATE INDEX` locks the table against writes for the whole build. On a large table build the index out of band, before deploying, with `CREATE INDEX CONCURRENTLY` on its own (never inside a transaction block). A concurrent build interrupted midway leaves the index invalid: drop it and retry (check `pg_index` and `pg_index.indisvalid`). +- A partitioned parent cannot take `CREATE INDEX CONCURRENTLY` (`cannot create index on partitioned table ... concurrently`). Build one index per partition with `CONCURRENTLY`, then `CREATE INDEX ... ON ONLY` on the parent and `ALTER INDEX ... ATTACH PARTITION` for each child; the parent index only becomes valid when every partition is attached. +- Write the migration as plain, idempotent SQL: `ADD COLUMN IF NOT EXISTS`, `CREATE INDEX IF NOT EXISTS`, and `DROP CONSTRAINT IF EXISTS` followed by `ADD CONSTRAINT`. Do not encode shape checks as `pg_catalog` introspection: they are easy to get wrong (a missing `NOT condeferred` accepts an `INITIALLY DEFERRED` foreign key) and they duplicate what the runbook verifies by hand. `IF NOT EXISTS` matches the name, not the definition, so the out-of-band procedure verifies the index shape and completeness. +- The runbook is the contract for a large instance: it builds and verifies the out-of-band work and then marks the migration step as applied (`INSERT INTO migrations (module, step) ...`), so the body does not run at startup. Verify the column, the pending rows, the index (valid, no predicate or expression, every partition attached) and the constraint shape before inserting the row; the marker is the only thing that skips the migration. +- A reference that a GC task resolves must be `DEFERRABLE INITIALLY IMMEDIATE`, never `INITIALLY DEFERRED`: the immediate check fires inside the transaction that catches the violation and retries the chunk; the deferred one fires at commit, outside the catch. A validated foreign key belongs out of band: `ADD FOREIGN KEY` runs as an `ALTER TABLE` and it scans the whole referencing table; `NOT VALID` on a partitioned table only exists since PostgreSQL 18. +- A backfill that removes a key an old reader uses (for example `metadata.storage-ref-id`, which the previous storage GC scans) must run with the old code stopped. Otherwise the old reader loses the reference and can delete a live object, and the rows it writes in the meantime miss the backfill. +- `ADD COLUMN` takes a brief `ACCESS EXCLUSIVE` even when `IF NOT EXISTS` skips it, and `ADD FOREIGN KEY` takes `ACCESS EXCLUSIVE` on the referencing table and `SHARE ROW EXCLUSIVE` on the referenced one. A migration accepts those locks; a deployment that cannot take them runs the runbook and marks the step as applied. ## See also diff --git a/backend/src/app/binfile/common.clj b/backend/src/app/binfile/common.clj index 43dbc60c2a..b6a701b784 100644 --- a/backend/src/app/binfile/common.clj +++ b/backend/src/app/binfile/common.clj @@ -43,7 +43,6 @@ (def ^:dynamic *state* nil) (def ^:dynamic *options* nil) -(def ^:dynamic *reference-file* nil) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; DEFAULTS @@ -159,7 +158,7 @@ [index obj & attrs] (reduce (fn [index attr-fn] (let [old-id (attr-fn obj) - new-id (if (::overwrite *options*) old-id (uuid/next))] + new-id (uuid/next)] (assoc index old-id new-id))) index attrs)) @@ -209,6 +208,7 @@ p.team_id, coalesce(fd.backend, 'legacy-db') AS backend, fd.metadata AS metadata, + fd.storage_object_id AS storage_object_id, fd.data AS data FROM file AS f LEFT JOIN file_data AS fd ON (fd.file_id = f.id AND fd.id = f.id) @@ -218,27 +218,37 @@ (str sql:files-with-data " WHERE f.id = ?")) (def sql:get-file-without-data - (str "WITH files AS (" sql:files-with-data ")" - "SELECT f.id, - f.project_id, - f.created_at, - f.modified_at, - f.deleted_at, - f.name, - f.is_shared, - f.has_media_trimmed, - f.revn, - f.ignore_sync_until, - f.comment_thread_seqn, - f.features, - f.version, - f.vern, - f.team_id - FROM files AS f - WHERE f.id = ?")) + "SELECT f.id, + f.project_id, + f.created_at, + f.modified_at, + f.deleted_at, + f.name, + f.is_shared, + f.has_media_trimmed, + f.revn, + f.ignore_sync_until, + f.comment_thread_seqn, + f.features, + f.version, + f.vern, + p.team_id + FROM file AS f + INNER JOIN project AS p ON (p.id = f.project_id) + WHERE f.id = ?") (defn- migrate-file - [{:keys [::db/conn] :as cfg} {:keys [read-only?]} {:keys [id] :as file}] + "Apply the pending migrations to the file. + + With `persist-after-migration?` the migrated map is also persisted, which is + only safe from a map read under the `file` row lock (see `get-file*`). The + returned map is then the reattached one: decoded data where the pages the + migration did not change keep their pointers and their fragment ids, plus + the `:storage-object-id` of the row the write left, so a caller that + persists again touches the current object. Without `persist-after-migration?` + the realized map is returned, which is what a read-only context can do. + `get-file*` realizes the result when the caller asks for it." + [cfg {:keys [id] :as file} {:keys [persist-after-migration?]}] (fdata/with-tracked-pointers cfg id (let [libs (delay (get-resolved-file-libraries cfg file)) ;; Capture the pointer layout before realizing: the migration @@ -252,111 +262,186 @@ objects-map-added? (and (not objects-map-before?) (contains? (:features realized) "fdata/objects-map"))] - (if (or read-only? (db/read-only? conn)) - realized - (let [file (if objects-map-added? - realized - (fdata/reattach-unchanged original realized))] + (if persist-after-migration? + (let [file (if objects-map-added? + realized + (fdata/reattach-unchanged original realized)) + written (update-file! cfg file)] ;; When the file is migrated, we break the rule of not ;; performing mutations on get operations and update the file - ;; with all migrations applied. The caller still receives the - ;; realized file: the read contract is plain data. - (update-file! cfg file) - (fmigr/resolve-applied-migrations cfg realized)))))) + ;; with all migrations applied. The reattached map is also what + ;; the read returns: the caller realizes it only when it asks. + (assoc (fmigr/resolve-applied-migrations cfg file) + :storage-object-id (:storage-object-id written))) + realized)))) + +(defn- fetch-row + "Read the file row. With `load-data?` the file data columns come along, but + they are not resolved nor decoded (see `attach-file-data`). With + `load-migrations?` the applied migrations are resolved, which is what + `need-migration?` expects. A skip-locked read answers nil instead of raising + when the row is gone." + [{:keys [::db/conn] :as cfg} id + {:keys [include-deleted? + skip-locked? + lock-for-update? + lock-for-share? + throw-if-not-exists? + load-migrations? + load-data?]}] + + (let [sql (if load-data? + sql:get-file + sql:get-file-without-data) + sql (cond + lock-for-update? + (str sql " FOR UPDATE of f") + + lock-for-share? + (str sql " FOR SHARE of f") + + :else + sql) + + sql (if (and skip-locked? (or lock-for-share? lock-for-update?)) + (str sql " SKIP LOCKED") + sql) + + file (db/get-with-sql conn [sql id] {::db/throw-if-not-exists false + ::db/remove-deleted (not include-deleted?)}) + file (some-> file + (d/update-when :features db/decode-pgarray #{}) + (d/update-when :metadata fdata/decode-metadata)) + file (if (and file load-migrations?) + (fmigr/resolve-applied-migrations cfg file) + file)] + + (or file + (when-not (or skip-locked? (not throw-if-not-exists?)) + (ex/raise :type :not-found + :code :object-not-found + :hint "database object not found" + :table :file + :file-id id))))) + + +(defn- attach-file-data + "Resolve the data of an already read row and decode it when `decode?`. It + completes the map it receives instead of reading the row again. It does not + realize: `get-file*` owns that step, once per path." + [cfg file {:keys [decode?]}] + (let [file (fdata/resolve-file-data cfg file)] + (if decode? + (fdata/decode-file-data cfg file) + file))) + +(defn- fetch-file + "Read the row with its data resolved and decoded, under the requested lock." + [cfg id options] + (when-let [file (fetch-row cfg id options)] + (attach-file-data cfg file options))) + +(def ^:private + default-get-file-options + {:load-migrations? true + :throw-if-not-exists? true + :persist-after-migration? true + :load-data? true + :migrate? true + :decode? true}) (defn- get-file* - [{:keys [::db/conn] :as cfg} id - {:keys [migrate? - realize? - decode? - skip-locked? - include-deleted? - load-data? - throw-if-not-exists? - lock-for-update? - lock-for-share?] - :or {lock-for-update? false - lock-for-share? false - load-data? true - migrate? true - decode? true - include-deleted? false - throw-if-not-exists? true - realize? false} - :as options}] + [{:keys [::db/conn] :as cfg} id options] (assert (db/connection? conn) "expected cfg with valid connection") - (when (and (not load-data?) - (or lock-for-share? lock-for-update? skip-locked?)) - (throw (IllegalArgumentException. "locking is incompatible when `load-data?` is false"))) - (let [sql - (if load-data? - sql:get-file - sql:get-file-without-data) + (let [{:keys [migrate? + realize? + decode? + load-data? + lock-for-share? + read-only?] + :as options} + (merge default-get-file-options options) - sql - (cond - lock-for-update? - (str sql " FOR UPDATE of f") + read-only? + (or read-only? (db/read-only? conn))] - lock-for-share? - (str sql " FOR SHARE of f") + (when (and realize? (or (not load-data?) (not decode?))) + (throw (IllegalArgumentException. "invalid options, you only can realize when loading data and decoding"))) - :else - sql) + (when (and lock-for-share? load-data?) + (throw (IllegalArgumentException. "invalid options, `:lock-for-share?` is only allowed on row-only reads"))) - sql - (if skip-locked? - (str sql " SKIP LOCKED") - sql) + (cond + ;; The caller only wants the row: that read is the answer. + (not load-data?) + (fetch-row cfg id options) - file - (db/get-with-sql conn [sql id] - {::db/throw-if-not-exists false - ::db/remove-deleted (not include-deleted?)}) + (and migrate? decode? (not read-only?)) + (when-let [file (fetch-row cfg id options)] + (if (fmg/need-migration? file) + (when-let [file (fetch-file cfg id (assoc options :lock-for-update? true))] + (cond-> file + (fmg/need-migration? file) + (as-> $ (migrate-file cfg $ options)) - file - (-> file - (d/update-when :features db/decode-pgarray #{}) - (d/update-when :metadata fdata/decode-metadata))] + realize? + (->> (fdata/realize cfg)))) - (if file - (if load-data? - (let [file - (->> file - (fmigr/resolve-applied-migrations cfg) - (fdata/resolve-file-data cfg)) + (let [file (attach-file-data cfg file options)] + (cond-> file + realize? + (->> (fdata/realize cfg)))))) - will-migrate? - (and migrate? (fmg/need-migration? file))] + :else + (when-let [file (fetch-file cfg id options)] + (cond-> file + (and migrate? decode? (fmg/need-migration? file)) + (as-> $ (migrate-file cfg $ (assoc options :persist-after-migration? false))) - (if decode? - (cond->> (fdata/decode-file-data cfg file) - (and realize? (not will-migrate?)) - (fdata/realize cfg) - - will-migrate? - (migrate-file cfg options)) - - file)) - file) - - (when-not (or skip-locked? (not throw-if-not-exists?)) - (ex/raise :type :not-found - :code :object-not-found - :hint "database object not found" - :table :file - :file-id id))))) + realize? + (->> (fdata/realize cfg))))))) (defn get-file "Get file, resolve all features and apply migrations. + Runs inside a transaction: it joins the caller's when the cfg already + carries a connection and opens one otherwise, so the migration write + commits with the read. A read-only read (`:read-only?`) takes no lock and + writes nothing, so it runs on a plain connection instead of opening a + transaction of its own; with a cfg that already carries a connection it + still joins the caller's. Accepts a cfg map only: a bare pool or a + connection raises `IllegalArgumentException`. + Useful when you have plan to apply massive or not surgical operations on file, because it removes the overhead of lazy fetching - and decoding." + and decoding. + + Options: + - `:realize?` returns plain data instead of fragment pointers; it + requires `:load-data?` and `:decode?`. + - `:decode?` decodes the stored blob (default true). + - `:load-data?` reads the data columns (default true). A row-only read + (`false`) is the only mode that accepts `:lock-for-share?`. + - `:migrate?` applies the pending migrations on read (default true). + - `:load-migrations?` resolves the applied migrations on a row read + (default true). + - `:read-only?` applies the pending migrations in memory only and skips + the transaction. It does not take a lock. + - `:lock-for-update?`, `:skip-locked?`, `:include-deleted?` and + `:throw-if-not-exists?` are the usual read knobs. A lock needs a + read-write transaction from the caller: PostgreSQL rejects row locks in + a read-only transaction. + - `:persist-after-migration?` is internal: `get-file*` turns it off + for the in-memory migration of a read-only context. Do not pass it." [cfg file-id & {:as opts}] - (db/run! cfg get-file* file-id opts)) + (when-not (map? cfg) + (throw (IllegalArgumentException. + "expected cfg map, not a pool or a connection"))) + (if (:read-only? opts) + (db/run! cfg get-file* file-id opts) + (db/tx-run! cfg get-file* file-id opts))) (defn clean-file-features [file] @@ -726,7 +811,12 @@ (let [library-ids (get-libraries cfg [file-id]) data (reduce (fn [data library-id] (if-let [library (get-file cfg library-id :include-deleted? true)] - (cond-> (ctf/absorb-assets data (:data library)) + ;; The library is read without `:realize?`, so its + ;; `:components` may be a pointer map. Bind the + ;; library's own loader: the absorb walks the + ;; components and would raise without one. + (cond-> (fdata/with-pointers cfg library-id + (ctf/absorb-assets data (:data library))) (= library-id (cfo/get-tokens-source data)) (cfo/absorb-tokens-lib (:data library))) data)) @@ -809,18 +899,33 @@ (dissoc :metadata) (dissoc :migrations))) +(defn file->response + "Strip the internal keys of a file map before it leaves the backend in a + response. The file data storage pointer is bookkeeping for the write path: + a `file-data` object is authentication-only, so a leaked id would let any + logged-in profile fetch the blob from `/assets/by-id`. The metadata key of + the old pointer is stripped too: a row the migration does not backfill keeps + it, and the metadata schema is open, so it would travel in the response." + [file] + (-> file + (dissoc :storage-object-id) + (d/update-when :metadata #(dissoc % :storage-ref-id)))) + (defn- file->file-data-params - [{:keys [id] :as file} & {:as opts}] + [{:keys [id] :as file}] (let [created-at (or (:created-at file) (ct/now)) modified-at (or (:modified-at file) created-at)] - (d/without-nils - {:id id - :type "main" - :file-id id - :data (:data file) - :metadata (:metadata file) - :created-at created-at - :modified-at modified-at}))) + (-> (d/without-nils + {:id id + :type "main" + :file-id id + :data (:data file) + :metadata (:metadata file) + :created-at created-at + :modified-at modified-at}) + ;; The storage object the persisted row pointed at (nil for a new or + ;; imported file). `upsert!` touches it before replacing the row. + (assoc :storage-object-id (:storage-object-id file))))) (defn insert-file! "Insert a new file into the database table. Expectes a not-encoded file. @@ -844,34 +949,63 @@ :cause cause) (throw cause)))) - (->> (file->file-data-params file) + ;; An INSERT never replaces a persisted pointer: `duplicate-file` hands + ;; the source file map here, and touching the source object would be a + ;; write on a live object. + (->> (file->file-data-params (dissoc file :storage-object-id)) (fdata/upsert! cfg)) nil)) (defn update-file! - "Update an existing file on the database. Expects not encoded file." - [{:keys [::db/conn] :as cfg} {:keys [id] :as file} & {:as opts}] + "Update an existing file on the database. Expects a not-encoded file map, + as returned by `get-file`, and optionally the fields to change as keyword + args; they are merged into the file map before persisting. - (if (::reset-migrations? opts false) - (fmigr/reset-migrations! conn file) - (fmigr/upsert-migrations! conn file)) + The storage object the persisted row pointed at travels in the map + (`:storage-object-id`), and `upsert!` touches it before replacing the row. + That is why the caller must pass the map it read instead of building a + partial one: nothing is queried here. The params only change mutable fields + of the file: the pointer and the identity (`:id`, `:file-id`) always come + from the read map. - (let [file - (encode-file cfg file) + Returns the map `fdata/upsert!` answers with: `:written` and the + `:storage-object-id` of the object this write created, so a caller that + persists again touches the current object instead of the replaced one." + [{:keys [::db/conn] :as cfg} file & {:as params}] - file-params - (file->params (dissoc file :id)) + ;; A map read with `:load-data? false` has no `:data`; persisting it would + ;; wipe the file content. `ex/raise` instead of `assert`: the guard must + ;; exist in the deployed binary, where `clojure.core/assert` is compiled + ;; out. + (when-not (contains? file :data) + (ex/raise :type :internal + :code :missing-file-data + :hint "expected a file map read with data" + :file-id (:id file))) - file-data-params - (file->file-data-params file)] + (let [reset? (::reset-migrations? params false) + ;; The pointer and the identity are bookkeeping of the persisted row, + ;; not editable fields: they can only come from the read map. A pointer + ;; in the params would skip the touch and leak the replaced blob; an + ;; `:id` would retarget the UPDATE. + file (merge file (dissoc params ::reset-migrations? + :storage-object-id + :id + :file-id))] - (db/update! conn :file file-params - {:id id} - {::db/return-keys false}) + (if reset? + (fmigr/reset-migrations! conn file) + (fmigr/upsert-migrations! conn file)) - (fdata/upsert! cfg file-data-params) - nil)) + (let [file (encode-file cfg file) + file-params (file->params (dissoc file :id))] + + (db/update! conn :file file-params + {:id (:id file)} + {::db/return-keys false}) + + (fdata/upsert! cfg (file->file-data-params file))))) (defn save-file! "Applies all the final validations and perist the file, binfile @@ -884,8 +1018,7 @@ (let [file (-> file (assoc :created-at timestamp) (assoc :modified-at timestamp) - (cond-> (not (::overwrite cfg)) - (assoc :ignore-sync-until (ct/plus timestamp (ct/duration {:seconds 5})))) + (assoc :ignore-sync-until (ct/plus timestamp (ct/duration {:seconds 5}))) (update :features (fn [features] (-> (::features cfg #{}) @@ -902,9 +1035,7 @@ (when (ex/exception? result) (l/error :hint "file schema validation error" :cause result)))) - (if (::overwrite cfg) - (update-file! cfg file (assoc opts ::reset-migrations? true)) - (insert-file! cfg file opts)))) + (insert-file! cfg file opts))) (def ^:private sql:get-file-libraries "WITH RECURSIVE libs AS ( diff --git a/backend/src/app/binfile/v3.clj b/backend/src/app/binfile/v3.clj index 69cdf9fa95..4a5cf868f4 100644 --- a/backend/src/app/binfile/v3.clj +++ b/backend/src/app/binfile/v3.clj @@ -988,8 +988,7 @@ :old-id (str (:id item)) ::l/sync? true) - (db/insert! conn :file-media-object params - ::db/on-conflict-do-nothing? (::bfc/overwrite cfg)))) + (db/insert! conn :file-media-object params))) (doseq [[index item] (d/enumerate thumbnails)] (let [media-id (bfc/lookup-index (:media-id item)) @@ -1305,56 +1304,6 @@ {:file-ids file-ids :resolution resolution}))) -(defn- invalidate-thumbnails - [cfg file-id] - (let [storage (sto/resolve cfg ::db/reuse-conn true) - - sql-1 - (str "update file_tagged_object_thumbnail " - " set deleted_at = now() " - " where file_id=? returning media_id") - - sql-2 - (str "update file_thumbnail " - " set deleted_at = now() " - " where file_id=? returning media_id")] - - (run! #(sto/touch-object! storage %) - (sequence - (keep :media-id) - (concat - (db/exec! cfg [sql-1 file-id]) - (db/exec! cfg [sql-2 file-id])))))) - -(defn- import-file-and-overwrite* - [{:keys [::manifest ::bfc/file-id] :as cfg}] - - (when (not= 1 (count (:files manifest))) - (ex/raise :type :validation - :code :invalid-condition - :hint "unable to perform in-place update with binfile containing more than 1 file" - :manifest manifest)) - - (bfc/configure-import-transaction cfg) - - (let [ref-file (bfc/get-minimal-file cfg file-id ::db/for-update true) - file (first (get manifest :files)) - cfg (assoc cfg ::bfc/overwrite true)] - - (vswap! bfc/*state* update :index assoc (:id file) file-id) - - (binding [bfc/*options* cfg - bfc/*reference-file* ref-file] - - (import-storage-objects cfg) - (import-file cfg file {:current 1 :total 1}) - - (invalidate-thumbnails cfg file-id) - (bfm/apply-pending-migrations! cfg) - - {:file-ids [file-id] - :resolution {}}))) - (defn- import-files [{:keys [::bfc/timestamp ::bfc/input] :or {timestamp (ct/now)} :as cfg}] @@ -1405,9 +1354,7 @@ (tap-progress cfg {:section :manifest}) (binding [bfc/*state* (volatile! {:media [] :index {}})] - (if (::bfc/file-id cfg) - (db/tx-run! cfg import-file-and-overwrite*) - (db/tx-run! cfg import-files*))))) + (db/tx-run! cfg import-files*)))) ;; --- PUBLIC API @@ -1476,6 +1423,12 @@ (defn import-files! [{:keys [::bfc/input] :as cfg}] + (when (::bfc/file-id cfg) + (ex/raise :type :validation + :code :not-supported + :hint "in-place import is not supported" + :file-id (::bfc/file-id cfg))) + (assert (and (uuid? (::bfc/profile-id cfg)) (uuid? (::bfc/project-id cfg))) diff --git a/backend/src/app/features/fdata.clj b/backend/src/app/features/fdata.clj index da8b662514..a7af13c9f7 100644 --- a/backend/src/app/features/fdata.clj +++ b/backend/src/app/features/fdata.clj @@ -105,11 +105,16 @@ (dissoc file :legacy-data)) (defmethod resolve-file-data "storage" - [cfg {:keys [metadata] :as file}] + [cfg {:keys [storage-object-id] :as file}] (let [storage (sto/resolve cfg ::db/reuse-conn true) - ref-id (:storage-ref-id metadata) - data (->> (sto/get-object storage ref-id) - (sto/get-object-bytes storage))] + sobject (or (sto/get-object storage storage-object-id) + (ex/raise :type :internal + :code :missing-storage-object + :hint "file data row has no readable storage object" + :file-id (or (:file-id file) (:id file)) + :id (:id file) + :storage-object-id storage-object-id)) + data (->> (sto/get-object-bytes storage sobject))] (-> file (assoc :data data) (dissoc :legacy-data)))) @@ -122,8 +127,8 @@ (def ^:private sql:insert-file-data "INSERT INTO file_data (file_id, id, created_at, modified_at, deleted_at, - type, backend, metadata, data) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)") + type, backend, metadata, data, storage_object_id) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") (def ^:private sql:upsert-file-data (str sql:insert-file-data @@ -132,104 +137,130 @@ deleted_at=?, backend=?, metadata=?, - data=?")) + data=?, + storage_object_id=? + WHERE file_data.storage_object_id IS NOT DISTINCT FROM ?")) (defn- upsert-in-database - [cfg {:keys [id file-id created-at modified-at deleted-at type backend data metadata]}] + [cfg {:keys [id file-id created-at modified-at deleted-at type backend data metadata storage-object-id]} expected-storage-object-id] (let [created-at (or created-at (ct/now)) - metadata (some-> metadata db/json) - modified-at (or modified-at created-at)] + ;; The metadata pointer is dead (the column is the only source of + ;; truth). Clean it here, the single point where every branch + ;; serializes the metadata, so no write can resurrect it. + metadata (some-> metadata (dissoc :storage-ref-id) (db/json)) + modified-at (or modified-at created-at) - (db/exec-one! cfg [sql:upsert-file-data - file-id id - created-at - modified-at - deleted-at - type - backend - metadata - data - modified-at - deleted-at - backend - metadata - data]))) + result + (db/exec-one! cfg [sql:upsert-file-data + file-id id + created-at + modified-at + deleted-at + type + backend + metadata + data + storage-object-id + modified-at + deleted-at + backend + metadata + data + storage-object-id + expected-storage-object-id])] + + ;; A zero count means the row exists with another pointer: the caller + ;; read it before someone else replaced it. Every writer holds the `file` + ;; row lock, so this is a protocol violation, not a race to retry. + (when-not (pos? (db/get-update-count result)) + (ex/raise :type :internal + :code :stale-file-data-pointer + :hint "the file data row changed since it was read" + :file-id file-id + :id id)) + + result)) (defn- handle-persistence [cfg {:keys [type backend id file-id data] :as params}] - - (cond - (= backend "storage") - ;; IMPORTANT: we strongly do not reuse the main connection that can - ;; run inside a transaction because the storage upload process can - ;; fail in the middle of uploading and leave garbage on the underlying - ;; backend, if we participate in the main transaction and it aborts - ;; we will lose all registry of the pending to reconcile blobs - ;; what the storage subsystem registers in other parallel - ;; transaction - (let [storage (sto/resolve cfg) - content (sto/content data) - sobject (sto/put-object! storage - {::sto/content content - ::sto/touch true - :bucket "file-data" - :content-type "application/octet-stream" - :file-id file-id - :id id}) - metadata (-> (:metadata params) - (assoc :storage-ref-id (:id sobject))) - params (-> params - (assoc :metadata metadata) - (assoc :data nil))] - (upsert-in-database cfg params)) - - (= backend "db") - (let [metadata (dissoc (:metadata params) :storage-ref-id) - params (assoc params :metadata metadata)] - (upsert-in-database cfg params)) - - (= backend "legacy-db") + (let [expected (:storage-object-id params)] (cond - (= type "main") - (do - (db/delete! cfg :file-data - {:id id :file-id file-id :type "main"} - {::db/return-keys false}) - (db/update! cfg :file - {:data data} - {:id file-id} - {::db/return-keys false})) + (= backend "storage") + ;; IMPORTANT: we strongly do not reuse the main connection that can + ;; run inside a transaction because the storage upload process can + ;; fail in the middle of uploading and leave garbage on the underlying + ;; backend, if we participate in the main transaction and it aborts + ;; we will lose all registry of the pending to reconcile blobs + ;; what the storage subsystem registers in other parallel + ;; transaction + (let [storage (sto/resolve cfg) + content (sto/content data) + sobject (sto/put-object! storage + {::sto/content content + ::sto/touch true + :bucket "file-data" + :content-type "application/octet-stream" + :file-id file-id + :id id}) + params (-> params + (assoc :storage-object-id (:id sobject)) + (assoc :data nil))] + {:result (upsert-in-database cfg params expected) + :storage-object-id (:id sobject)}) - (= type "snapshot") - (do - (db/delete! cfg :file-data - {:id id :file-id file-id :type "snapshot"} - {::db/return-keys false}) - ;; every mutation of `file_change` moves `updated_at`: the partition - ;; runbook uses it as the watermark of the delta passes - (db/update! cfg :file-change - {:data data - :updated-at (ct/now)} - {:file-id file-id :id id} - {::db/return-keys false})) + (= backend "db") + {:result (upsert-in-database cfg (assoc params :storage-object-id nil) expected) + :storage-object-id nil} - (= type "fragment") - (upsert-in-database cfg - (-> (dissoc params :metadata) - (assoc :backend "db"))) + (= backend "legacy-db") + {:result + (cond + (= type "main") + (do + (db/delete! cfg :file-data + {:id id :file-id file-id :type "main"} + {::db/return-keys false}) + (db/update! cfg :file + {:data data} + {:id file-id} + {::db/return-keys false})) + + (= type "snapshot") + (do + (db/delete! cfg :file-data + {:id id :file-id file-id :type "snapshot"} + {::db/return-keys false}) + ;; every mutation of `file_change` moves `updated_at`: the partition + ;; runbook uses it as the watermark of the delta passes + (db/update! cfg :file-change + {:data data + :updated-at (ct/now)} + {:file-id file-id :id id} + {::db/return-keys false})) + + (= type "fragment") + (upsert-in-database cfg + (-> (dissoc params :metadata) + (assoc :backend "db") + (assoc :storage-object-id nil)) + expected) + + :else + (throw (RuntimeException. "not implemented"))) + :storage-object-id nil} :else - (throw (RuntimeException. "not implemented"))) + (throw (IllegalArgumentException. + (str "backend '" backend "' not supported")))))) - :else - (throw (IllegalArgumentException. - (str "backend '" backend "' not supported"))))) - -(defn process-metadata - [cfg metadata] - (when-let [storage-id (:storage-ref-id metadata)] - (let [storage (sto/resolve cfg ::db/reuse-conn true)] - (sto/touch-object! storage storage-id)))) +(defn touch-storage-object + "Mark a storage object as touched so the storage GC can reclaim it once + nothing references it anymore. Used when a file data row replaces or + deletes the object it pointed at. Expects a non-nil object id; callers + guard with `some->`/`some->>`." + [cfg storage-object-id] + (-> (sto/resolve cfg ::db/reuse-conn true) + (sto/touch-object! storage-object-id))) (defn- default-backend [backend] @@ -248,6 +279,10 @@ [:file-id ::sm/uuid] [:backend {:optional true} [:enum "db" "legacy-db" "storage"]] [:metadata {:optional true} ctf/schema:file-metadata] + ;; The storage object the persisted row pointed at before this write; it is + ;; the object to touch when the row replaces it. Required so a new caller + ;; must decide it explicitly (nil for a new row) instead of leaking it. + [:storage-object-id [:maybe ::sm/uuid]] [:data {:optional true} bytes?] [:created-at {:optional true} ::ct/inst] [:modified-at {:optional true} [:maybe ::ct/inst]] @@ -257,17 +292,28 @@ (sm/check-fn schema:update-params :hint "invalid params received for update")) (defn upsert! - "Create or update file data" + "Create or update a file data row. + + `:storage-object-id` is the storage object the persisted row pointed at + before this write (nil for a new row or a row that had none), not the + object the new content will live in: the write resolves that itself. The + previous object is marked as touched before the row is replaced, so the + storage GC can reclaim the blob once nothing references it. + + The function does not query the row: the caller must hand the pointer it + read. Returns a map with `:written` (whether a row was written) and the + `:storage-object-id` of the object this write created, nil for the `db` + and `legacy-db` backends." [cfg params & {:as opts}] (let [params (-> (check-update-params params) (update :backend default-backend))] - (some->> (:metadata params) - (process-metadata cfg)) + (some->> (:storage-object-id params) + (touch-storage-object cfg)) - (-> (handle-persistence cfg params) - (db/get-update-count) - (pos?)))) + (let [{:keys [result storage-object-id]} (handle-persistence cfg params)] + {:written (pos? (db/get-update-count result)) + :storage-object-id storage-object-id}))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; POINTER-MAP @@ -281,7 +327,7 @@ ;; previous pass marked must still load until `objects-gc` purges it. (let [fragment (some-> (db/get* cfg :file-data {:id id :file-id file-id :type "fragment"} - {::sql/columns [:data :backend :id :metadata] + {::sql/columns [:data :backend :file_id :id :metadata :storage_object_id] ::db/remove-deleted false}) (update :metadata decode-metadata))] @@ -308,9 +354,12 @@ (when (pmap/modified? item) (l/trc :hint "persist pointer" :file-id (str file-id) :id (str id)) (let [content (-> item deref blob/encode)] + ;; A modified pointer has a fresh id, so the fragment row is always a + ;; new insert: there is no previous storage object to touch. (upsert! cfg {:id id :file-id file-id :type "fragment" + :storage-object-id nil :data content}))))) (defmacro with-pointers diff --git a/backend/src/app/features/file_snapshots.clj b/backend/src/app/features/file_snapshots.clj index 7c1818b743..4b1d2de942 100644 --- a/backend/src/app/features/file_snapshots.clj +++ b/backend/src/app/features/file_snapshots.clj @@ -42,7 +42,8 @@ c.data AS legacy_data, fd.data AS data, coalesce(fd.backend, 'legacy-db') AS backend, - fd.metadata AS metadata + fd.metadata AS metadata, + fd.storage_object_id AS storage_object_id FROM file_change AS c LEFT JOIN file_data AS fd ON (fd.file_id = c.file_id AND fd.id = c.id @@ -65,12 +66,16 @@ (d/update-when :features db/decode-pgarray #{})))) (def ^:private sql:get-minimal-file + ;; `fd.storage_object_id` is load-bearing: `restore!` hands this map to + ;; `update-file!` so the previous main object is touched before the row is + ;; replaced. Removing it would leave the old blob unreclaimed with no error. "SELECT f.id, f.revn, f.modified_at, f.deleted_at, fd.backend AS backend, - fd.metadata AS metadata + fd.metadata AS metadata, + fd.storage_object_id AS storage_object_id FROM file AS f LEFT JOIN file_data AS fd ON (fd.file_id = f.id AND fd.id = f.id) WHERE f.id = ?") @@ -131,7 +136,8 @@ (defn get-snapshot "Get a fully decoded snapshot for read-only preview or restoration. Returns the snapshot map with decoded :data field, or nil when the file has - no snapshot with that id." + no snapshot with that id. The internal storage pointer of the row is not + part of the result." [cfg file-id snapshot-id] (let [now (ct/now)] (some->> (db/get-with-sql cfg [sql:get-snapshot file-id snapshot-id now] @@ -139,7 +145,8 @@ ::db/throw-if-not-exists false}) (decode-snapshot) (fdata/resolve-file-data cfg) - (fdata/decode-file-data cfg)))) + (fdata/decode-file-data cfg) + (bfc/file->response)))) (def ^:private sql:get-visible-snapshots (str "WITH " @@ -293,6 +300,7 @@ {:id snapshot-id :file-id (:id file) :type "snapshot" + :storage-object-id nil :data (blob/encode (:data file)) :created-at created-at :deleted-at deleted-at}) diff --git a/backend/src/app/http/debug.clj b/backend/src/app/http/debug.clj index d62520706f..768e358176 100644 --- a/backend/src/app/http/debug.clj +++ b/backend/src/app/http/debug.clj @@ -123,7 +123,8 @@ (defn- get-resolved-file [cfg file-id] - (bfc/get-file cfg file-id :migrate? false :decode? false)) + (-> (bfc/get-file cfg file-id :migrate? false :decode? false) + (bfc/file->response))) (defn prepare-download [file filename] @@ -372,7 +373,6 @@ :profile-id profile-id :project-id project-id) cfg (assoc cfg - ::bfc/overwrite false ::bfc/profile-id profile-id ::bfc/project-id project-id ::bfc/team-id (:id team) diff --git a/backend/src/app/migrations.clj b/backend/src/app/migrations.clj index b886816a1d..76bf5fb9d0 100644 --- a/backend/src/app/migrations.clj +++ b/backend/src/app/migrations.clj @@ -541,7 +541,10 @@ :fn (mg/resource "app/migrations/sql/0158-del-forbidden-file-library-rels.sql")} {:name "0158-partition-file-change" - :fn (mg/resource "app/migrations/sql/0158-partition-file-change.sql")}]) + :fn (mg/resource "app/migrations/sql/0158-partition-file-change.sql")} + + {:name "0159-add-file-data-storage-object-id" + :fn (mg/resource "app/migrations/sql/0159-add-file-data-storage-object-id.sql")}]) (defn apply-migrations! [pool name migrations] diff --git a/backend/src/app/migrations/sql/0159-add-file-data-storage-object-id.sql b/backend/src/app/migrations/sql/0159-add-file-data-storage-object-id.sql new file mode 100644 index 0000000000..3a92172edf --- /dev/null +++ b/backend/src/app/migrations/sql/0159-add-file-data-storage-object-id.sql @@ -0,0 +1,65 @@ +-- Turn the file data storage pointer into a column with a foreign key. +-- +-- Until now the pointer lived in `file_data.metadata->>'storage-ref-id'`. +-- The storage GC parsed the JSON, and nothing stopped it from deleting the +-- blob of a live row. With the column and the foreign key the database +-- refuses that delete. +-- +-- With the storage backend the release is a coordinated restart: the column is +-- the only source of truth, and the previous code cannot read the rows this +-- release writes. With the default `db` backend the new code writes the same +-- `file_data.data` column the previous code reads, so a rolling deploy is +-- safe. +-- +-- This migration runs on a fresh database and on small instances. A large +-- instance builds the column and the index out of band, runs this same +-- backfill and foreign key with the instances stopped, and marks this step as +-- applied; the end state is identical. See +-- `operations/2.20.0-file-data-storage-object-id.md`. +-- +-- `DEFERRABLE` follows the schema convention (all foreign keys of the project +-- are deferrable, so `th/database-reset` can delete table by table with +-- `SET CONSTRAINTS ALL DEFERRED`). It is `INITIALLY IMMEDIATE`, not +-- `DEFERRED`: the storage GC catches the violation inside the chunk +-- transaction and retries the chunk, and a deferred check would fire at +-- commit, outside that transaction, so the GC would never move past it. + +ALTER TABLE file_data + ADD COLUMN IF NOT EXISTS storage_object_id uuid; + +-- The CTE is MATERIALIZED so the uuid cast only sees rows that passed the +-- regex. A plain subquery can be inlined, and a corrupt value would reach the +-- cast and abort the migration. +WITH candidates AS MATERIALIZED ( + SELECT file_id, + id, + (metadata->>'storage-ref-id')::uuid AS soid + FROM file_data + WHERE backend = 'storage' + AND storage_object_id IS NULL + AND (metadata->>'storage-ref-id') ~* + '^[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}$' +) +UPDATE file_data AS fd + SET storage_object_id = c.soid, + metadata = fd.metadata - 'storage-ref-id' + FROM candidates AS c + WHERE fd.file_id = c.file_id + AND fd.id = c.id + AND EXISTS (SELECT 1 FROM storage_object AS so WHERE so.id = c.soid); + +-- The deletion check of the foreign key looks a row up by this column for +-- every deleted object; without the index it scans all partitions. +CREATE INDEX IF NOT EXISTS file_data__storage_object_id__idx + ON file_data (storage_object_id); + +-- Rebuild the constraint so the definition is always the one above. This is +-- also how a constraint created by hand with the wrong shape (CASCADE, not +-- deferrable, or `INITIALLY DEFERRED`) gets repaired. +ALTER TABLE file_data + DROP CONSTRAINT IF EXISTS file_data_storage_object_id_fkey; + +ALTER TABLE file_data + ADD CONSTRAINT file_data_storage_object_id_fkey + FOREIGN KEY (storage_object_id) REFERENCES storage_object(id) + ON DELETE NO ACTION DEFERRABLE INITIALLY IMMEDIATE; diff --git a/backend/src/app/rpc/commands/comments.clj b/backend/src/app/rpc/commands/comments.clj index a61e1cb3c1..09e24ad8ff 100644 --- a/backend/src/app/rpc/commands/comments.clj +++ b/backend/src/app/rpc/commands/comments.clj @@ -192,7 +192,10 @@ (-> file (assoc :page-name (dm/get-in data [:pages-index page-id :name])) (assoc :page-id page-id) - (dissoc :data))))) + (dissoc :data) + ;; The storage pointer is internal bookkeeping: it never leaves the + ;; backend in a response. + (bfc/file->response))))) ;; FIXME: rename (defn- get-comment-thread diff --git a/backend/src/app/rpc/commands/files.clj b/backend/src/app/rpc/commands/files.clj index 4bf2717eee..de4d98444f 100644 --- a/backend/src/app/rpc/commands/files.clj +++ b/backend/src/app/rpc/commands/files.clj @@ -43,7 +43,6 @@ [app.rpc.doc :as-alias doc] [app.rpc.helpers :as rph] [app.rpc.permissions :as perms] - [app.util.blob :as blob] [app.util.events :as events] [app.util.pointer-map :as pmap] [app.util.services :as sv] @@ -222,7 +221,11 @@ (if (and (contains? (:features file) "fdata/objects-map") (not (contains? (:features params) "fdata/objects-map"))) (feat.fdata/realize-objects cfg file) - file))))) + file) + + ;; The storage pointer is internal bookkeeping: it never leaves the + ;; backend in a response. + (bfc/file->response file))))) ;; --- COMMAND QUERY: get-file-fragment (by id) @@ -231,7 +234,7 @@ [:id ::sm/uuid] [:file-id ::sm/uuid] [:created-at ::ct/inst] - [:content ::sm/any]]) + [:data ::sm/any]]) (def schema:get-file-fragment [:map {:title "get-file-fragment"} @@ -241,8 +244,17 @@ (defn- get-file-fragment [cfg file-id fragment-id] - (some-> (db/get cfg :file-data {:file-id file-id :id fragment-id :type "fragment"}) - (update :data blob/decode))) + ;; Resolve through the file data backend: a fragment can live on cold + ;; storage, and reading its row directly would hand `nil` to `blob/decode`. + ;; Only the public fields leave the backend: the row also carries the + ;; internal storage pointer and the backend metadata. + (when-let [row (db/get cfg :file-data {:file-id file-id + :id fragment-id + :type "fragment"})] + (let [fragment (->> row + (feat.fdata/resolve-file-data cfg) + (feat.fdata/decode-file-data cfg))] + (select-keys fragment (sm/keys schema:file-fragment))))) (defn- check-fragment-scope! "Checks that the fragment is reachable from the pages authorized by @@ -556,17 +568,23 @@ (let [file (bfc/get-file cfg id :load-data? false) summary (t/decode-str result)] (-> (assoc file :library-summary summary) - (dissoc :data))))) + (dissoc :data :migrations) + (bfc/file->response))))) calculate-from-db (fn [] (let [file (bfc/get-file cfg id) result (feat.fdata/with-pointers cfg id (calculate-library-summary file))] + ;; The storage pointer is internal bookkeeping: this summary is an + ;; RPC response and must not expose it. The applied migrations go + ;; for the same reason: this endpoint is not where a client + ;; completes a read, and `schema:file` declares the key optional. (-> file (assoc :library-summary result) (dissoc :legacy-data) - (dissoc :data)))) + (dissoc :data :migrations) + (bfc/file->response)))) persist-to-cache (fn [{:keys [::rds/conn]} data cache-key] @@ -1007,13 +1025,10 @@ :library-id (str (:id ldata)) :file-id (str file-id)) - (bfc/update-file! cfg {:id file-id - :features (:features file) - :migrations (:migrations file) - :revn (inc (:revn file)) - :data (:data file) - :modified-at (ct/now) - :has-media-trimmed false})))) + (bfc/update-file! cfg file + :revn (inc (:revn file)) + :modified-at (ct/now) + :has-media-trimmed false)))) (defn- get-referenced-files [cfg library-id] @@ -1420,8 +1435,8 @@ (def ^:private sql:restore-storage-objects ;; Collect referenced storage ids first (UUID set), then join by PK. - ;; Casting `storage_object.id` to text (or OR-ing that with `id IN`) forces - ;; a scan of deleted rows; cast the JSON side instead. + ;; `file_data` points at its object with the `storage_object_id` column, so + ;; the reference is already a uuid and no JSON cast is needed. "WITH ids AS (SELECT unnest(?::uuid[]) AS file_id), refs AS ( SELECT fmo.media_id AS id @@ -1448,11 +1463,11 @@ WHERE fot.deleted_at IS NULL AND fot.media_id IS NOT NULL UNION - SELECT (fd.metadata->>'storage-ref-id')::uuid + SELECT fd.storage_object_id FROM file_data AS fd JOIN ids ON (ids.file_id = fd.file_id) WHERE fd.deleted_at IS NULL - AND (fd.metadata->>'storage-ref-id') IS NOT NULL + AND fd.storage_object_id IS NOT NULL ) UPDATE storage_object AS so SET deleted_at = null diff --git a/backend/src/app/rpc/commands/files_create.clj b/backend/src/app/rpc/commands/files_create.clj index 368f480ec7..3d1fac497a 100644 --- a/backend/src/app/rpc/commands/files_create.clj +++ b/backend/src/app/rpc/commands/files_create.clj @@ -68,7 +68,10 @@ {:modified-at (ct/now)} {:id project-id}) - (bfc/get-file cfg (:id file)))))) + ;; The storage pointer is internal bookkeeping: it never leaves the + ;; backend in a response. + (-> (bfc/get-file cfg (:id file)) + (bfc/file->response)))))) (def ^:private schema:create-file [:map {:title "create-file" :closed true} diff --git a/backend/src/app/rpc/commands/files_snapshot.clj b/backend/src/app/rpc/commands/files_snapshot.clj index ede921d523..d56691540f 100644 --- a/backend/src/app/rpc/commands/files_snapshot.clj +++ b/backend/src/app/rpc/commands/files_snapshot.clj @@ -126,12 +126,15 @@ ;; every unpurged snapshot as a liveness root, and readers ;; (get-file-snapshot, restore!, reduce-snapshots) resolve pointers ;; on demand. - ;; NOTE: the file is read with FOR SHARE inside this transaction so + ;; NOTE: the file is read with FOR UPDATE inside this transaction so ;; snapshot creation serializes with a concurrent :file-gc pass - ;; (which takes FOR UPDATE + SKIP LOCKED on the same row). Without - ;; the lock, a version pinned mid-pass could reference fragment rows - ;; the pass is recreating. - (let [file (bfc/get-file cfg file-id :lock-for-share? true) + ;; (which takes FOR UPDATE + SKIP LOCKED on the same row) and with a + ;; concurrent save. Without the lock, a version pinned mid-pass could + ;; reference fragment rows the pass is recreating. The lock is + ;; exclusive, not shared, because the migration-on-read path may + ;; re-read the same row with FOR UPDATE; holding a shared lock while + ;; asking for the exclusive one deadlocks two concurrent readers. + (let [file (bfc/get-file cfg file-id :lock-for-update? true) project (db/get-by-id cfg :project (:project-id file))] (-> cfg @@ -167,12 +170,14 @@ delay (ldel/get-deletion-delay team) ;; Create backup snapshot of the current state (committed immediately - ;; independent of the restore outcome). The read takes FOR SHARE so a + ;; independent of the restore outcome). The read takes FOR UPDATE so a ;; concurrent :file-gc pass (which holds FOR UPDATE + SKIP LOCKED on the ;; file row) either finishes first -- and we snapshot the new fragments -- ;; or skips the file and retries later. Without the lock, the backup ;; could reference fragments the pass is about to mark as deleted, - ;; breaking the backup permanently. + ;; breaking the backup permanently. The lock is exclusive because the + ;; migration-on-read path may re-read the row with FOR UPDATE: a shared + ;; lock that upgrades deadlocks two concurrent readers. ;; ;; The backup read also provides the revn the guard below compares ;; against. An edit committed before this read is captured in the @@ -181,7 +186,7 @@ backup-revn (db/tx-run! cfg (fn [cfg] - (let [file (bfc/get-file cfg file-id :lock-for-share? true)] + (let [file (bfc/get-file cfg file-id :lock-for-update? true)] (fsnap/create! cfg file {:profile-id profile-id :deleted-at (ct/in-future delay) diff --git a/backend/src/app/rpc/commands/files_update.clj b/backend/src/app/rpc/commands/files_update.clj index 6cc2846e9f..c47b018c0f 100644 --- a/backend/src/app/rpc/commands/files_update.clj +++ b/backend/src/app/rpc/commands/files_update.clj @@ -438,9 +438,15 @@ :modified-at (or timestamp (ct/now))}})))) (defn get-file - "Get not-decoded file, only decodes the features set." + "Get not-decoded file, only decodes the features set. + + The read takes the `file` row with FOR UPDATE: the save persists the + map it read, so it must hold the row from the read to the write. The + lock is exclusive, not shared, because the migration-on-read path may + re-read the same row with FOR UPDATE, and a shared lock that upgrades + deadlocks two concurrent readers." [cfg id] - (bfc/get-file cfg id :decode? false :lock-for-share? true)) + (bfc/get-file cfg id :decode? false :lock-for-update? true)) (defn persist-file! "Function responsible of persisting already encoded file. Should be diff --git a/backend/src/app/rpc/commands/management.clj b/backend/src/app/rpc/commands/management.clj index b0ac390b57..79cf51781b 100644 --- a/backend/src/app/rpc/commands/management.clj +++ b/backend/src/app/rpc/commands/management.clj @@ -88,7 +88,9 @@ fmeds)] (db/insert! conn :file-media-object params ::db/return-keys false)) - file))) + ;; The storage pointer of the source file is internal bookkeeping: the + ;; new file does not inherit it, and it never leaves the backend. + (bfc/file->response file)))) (def ^:private schema:duplicate-file diff --git a/backend/src/app/rpc/commands/teams_invitations.clj b/backend/src/app/rpc/commands/teams_invitations.clj index 75e5ce2c87..83d09d3219 100644 --- a/backend/src/app/rpc/commands/teams_invitations.clj +++ b/backend/src/app/rpc/commands/teams_invitations.clj @@ -770,6 +770,9 @@ (-> file (dissoc :data) (dissoc :deleted-at) + ;; The storage pointer is internal bookkeeping: it never leaves the + ;; backend in a response. + (bfc/file->response) (assoc :page-id (-> file :data :pages first))))) (def ^:private schema:create-team-access-request diff --git a/backend/src/app/rpc/commands/viewer.clj b/backend/src/app/rpc/commands/viewer.clj index d36915d640..d757f4018d 100644 --- a/backend/src/app/rpc/commands/viewer.clj +++ b/backend/src/app/rpc/commands/viewer.clj @@ -129,7 +129,10 @@ (defn- get-view-only-bundle [{:keys [::db/conn] :as cfg} {:keys [profile-id file-id share-id ::perms] :as params}] - (let [file (bfc/get-file cfg file-id) + (let [;; The storage pointer is internal bookkeeping: this bundle is an RPC + ;; response and must not expose it. + file (-> (bfc/get-file cfg file-id) + (bfc/file->response)) project (db/get conn :project {:id (:project-id file)} @@ -159,7 +162,8 @@ libs (->> (bfc/get-file-libraries conn file-id) (mapv (fn [{:keys [id] :as lib}] - (merge lib (bfc/get-file cfg id))))) + (-> (merge lib (bfc/get-file cfg id)) + (bfc/file->response))))) ;; A share link never grants pages of a linked library, so trim ;; each library to the components the allowed pages reference. diff --git a/backend/src/app/srepl/helpers.clj b/backend/src/app/srepl/helpers.clj index 18221f9434..5f839081b5 100644 --- a/backend/src/app/srepl/helpers.clj +++ b/backend/src/app/srepl/helpers.clj @@ -182,5 +182,5 @@ :created-by "system"})) (let [file' (update file' :revn inc)] - (bfc/update-file! system file' opts) + (bfc/update-file! system file' ::bfc/reset-migrations? (::bfc/reset-migrations? opts)) true)))) diff --git a/backend/src/app/storage/gc_deleted.clj b/backend/src/app/storage/gc_deleted.clj index aacc2d7ac3..378649d3c6 100644 --- a/backend/src/app/storage/gc_deleted.clj +++ b/backend/src/app/storage/gc_deleted.clj @@ -118,6 +118,16 @@ [(ct/now) chunk-size]])] (db/exec! conn (into [sql] params)))) +(defn- server-error + "Return the PostgreSQL server error of a foreign key violation, or nil. It + carries the constraint and the referencing table, which is what the operator + needs: any `NO ACTION` foreign key on `storage_object` can block the delete, + not only the `file_data` one." + [cause] + (when (and (instance? org.postgresql.util.PSQLException cause) + (= "23503" (.getSQLState ^java.sql.SQLException cause))) + (.getServerErrorMessage ^org.postgresql.util.PSQLException cause))) + (defn- clean-deleted "Processes the pending objects chunk by chunk with keyset pagination: each chunk is one short transaction that reads and locks the next `chunk-size` @@ -127,11 +137,16 @@ marked as deleted. A chunk that fails aborts its transaction: nothing of the chunk is deleted, - its locks are released, and it is logged and skipped. The cursor advances - to the `(deleted_at, id)` of the last row of each read chunk regardless of - the deletion result, so a failed chunk is not visited again in the same - run. There is no long-lived read transaction: each transaction lives only - as long as one chunk, letting the vacuum proceed between chunks. Rows + its locks are released, and it is logged and skipped. A chunk blocked by a + foreign key (SQLState 23503) keeps its log at error level with a hint and the + constraint and table the database reports, and is retried on the next run; + the reference must be fixed for the chunk to make progress. Any `NO ACTION` + foreign key on `storage_object` can block the delete, not only the one of + `file_data`. The cursor advances to the `(deleted_at, id)` of + the last row of each read chunk regardless of the deletion result, so a + failed chunk is not visited again in the same run. There is no long-lived + read transaction: each transaction lives only as long as one chunk, letting + the vacuum proceed between chunks. Rows locked by a concurrent run are skipped and deferred to the next run; a chunk that fails after a partial backend delete self-heals on the next run (physical deletes are idempotent). Every `throttle-every` deleted @@ -155,9 +170,22 @@ ;; afterwards is a JDBC no-op on the already ;; rolled-back connection. (db/rollback! conn) - (l/err :hint "unexpected error on chunk deletion" - :ids (map :id rows) - :cause cause) + (if-let [serr (server-error cause)] + ;; A NO ACTION foreign key refused a physical + ;; delete: a live row of another table still + ;; references an object marked for deletion. + ;; That is a real anomaly (the reference scan + ;; should have frozen it), so it stays an + ;; error and the chunk is retried on the next + ;; run until the reference is fixed. + (l/err :hint "chunk deletion blocked by a live reference; fix the reference and let the next run reclaim the chunk" + :ids (map :id rows) + :constraint (.getConstraint ^org.postgresql.util.ServerErrorMessage serr) + :table (.getTable ^org.postgresql.util.ServerErrorMessage serr) + :cause cause) + (l/err :hint "unexpected error on chunk deletion" + :ids (map :id rows) + :cause cause)) [rows {}])))))] ;; Reported only after the chunk commits: a rolled-back chunk leaves diff --git a/backend/src/app/storage/gc_touched.clj b/backend/src/app/storage/gc_touched.clj index 80e411d9da..c4e4c463e9 100644 --- a/backend/src/app/storage/gc_touched.clj +++ b/backend/src/app/storage/gc_touched.clj @@ -108,16 +108,13 @@ (def sql:exists-file-data-refs "SELECT EXISTS ( SELECT 1 FROM file_data - WHERE file_id = ? - AND id = ? - AND metadata->>'storage-ref-id' = ?::text + WHERE storage_object_id = ? ) AS has_refs") (defn- has-file-data-refs? [conn sobject] - (let [{:keys [file-id id]} (:metadata sobject)] - (-> (db/exec-one! conn [sql:exists-file-data-refs file-id id (:id sobject)]) - (get :has-refs)))) + (-> (db/exec-one! conn [sql:exists-file-data-refs (:id sobject)]) + (get :has-refs))) (def ^:private sql:mark-freeze-in-bulk "UPDATE storage_object diff --git a/backend/src/app/tasks/objects_gc.clj b/backend/src/app/tasks/objects_gc.clj index 80f212a6b1..8f4ff44a98 100644 --- a/backend/src/app/tasks/objects_gc.clj +++ b/backend/src/app/tasks/objects_gc.clj @@ -613,7 +613,7 @@ ORDER BY deleted_at ASC") (def ^:private sql:lock-file-data - "SELECT file_id, id, type, deleted_at, metadata, backend + "SELECT file_id, id, type, deleted_at, backend, storage_object_id FROM file_data WHERE (file_id, id) IN (SELECT file_id, id FROM unnest(?::uuid[], ?::uuid[]) AS u(file_id, id)) AND deleted_at IS NOT NULL @@ -630,11 +630,10 @@ (let [file-ids (db/create-array conn "uuid" (map :file-id rows)) ids (db/create-array conn "uuid" (map :id rows))] (->> (db/exec! conn [sql:lock-file-data file-ids ids threshold]) - (reduce (fn [total {:keys [file-id id type deleted-at metadata backend]}] + (reduce (fn [total {:keys [file-id id type deleted-at backend storage-object-id]}] - (some->> metadata - (fdata/decode-metadata) - (fdata/process-metadata cfg)) + (some->> storage-object-id + (fdata/touch-storage-object cfg)) (l/trc :obj "file-data" :id (str id) diff --git a/backend/src/app/tasks/offload_file_data.clj b/backend/src/app/tasks/offload_file_data.clj index e8e3eb12ff..816a272ccb 100644 --- a/backend/src/app/tasks/offload_file_data.clj +++ b/backend/src/app/tasks/offload_file_data.clj @@ -28,18 +28,33 @@ (def ^:private sql:lock-file ;; NOTE: SKIP LOCKED only serializes against writers that also lock - ;; the file row. Every file_data writer must therefore hold at least - ;; FOR SHARE on it (the update RPC via lock-for-share?, file-gc via - ;; FOR UPDATE); a writer skipping that lock would reopen the - ;; lost-update race this lock closes. + ;; the file row. Every file_data writer must therefore lock it too: + ;; the update RPC and the snapshot paths via `lock-for-update?`, and + ;; file-gc via FOR UPDATE. A writer skipping that lock would reopen + ;; the lost-update race this lock closes. "SELECT id FROM file WHERE id = ? AND deleted_at IS NULL FOR UPDATE SKIP LOCKED") +(defn- decode-row + "Normalize a `file_data` row read with `SELECT fd.*`: NULL columns are + absent (the db layer uses `next.jdbc.optional`) and jsonb metadata arrives + as a `PGobject`, while `fdata/upsert!` expects a decoded metadata map. A + JSON `null` decodes to nil, and the key has to disappear rather than travel + as `:metadata nil`, which the schema rejects." + [row] + (let [metadata (some-> (:metadata row) fdata/decode-metadata)] + (cond-> (dissoc row :metadata) + (some? metadata) (assoc :metadata metadata)))) + (defn- offload-file-data "Offload one hot-storage row to cold storage. Returns the row type and the migrated bytes for the caller to aggregate." [cfg {:keys [id file-id type data] :as fdata}] (jobs/heartbeat cfg) - (fdata/upsert! cfg (assoc fdata :backend "storage")) + ;; The pointer key is normalized here because a NULL `storage_object_id` + ;; is omitted by the row builder; `decode-row` already normalized metadata. + (fdata/upsert! cfg (assoc fdata + :backend "storage" + :storage-object-id (:storage-object-id fdata))) (let [size (if (bytes? data) (alength ^bytes data) 0)] (l/trc :hint "file data row offloaded" :file-id (str file-id) @@ -61,16 +76,19 @@ [{:keys [::db/conn] :as cfg} file-id] (if (db/exec-one! conn [sql:lock-file file-id]) (let [{:keys [rows bytes by-type]} - (reduce (fn [stats fdata] - (let [{:keys [type size]} (offload-file-data cfg fdata)] - (-> stats - (update :rows inc) - (update :bytes + size) - (update-in [:by-type type] - (fn [{r :rows b :bytes :or {r 0 b 0}}] - {:rows (inc r) :bytes (+ b size)}))))) - {:rows 0 :bytes 0 :by-type {}} - (db/plan conn [sql:get-file-data file-id]))] + (transduce + (map decode-row) + (completing + (fn [stats fdata] + (let [{:keys [type size]} (offload-file-data cfg fdata)] + (-> stats + (update :rows inc) + (update :bytes + size) + (update-in [:by-type type] + (fn [{r :rows b :bytes :or {r 0 b 0}}] + {:rows (inc r) :bytes (+ b size)})))))) + {:rows 0 :bytes 0 :by-type {}} + (db/plan conn [sql:get-file-data file-id]))] (l/trc :hint "file data offloaded" :file-id (str file-id) :offloaded rows diff --git a/backend/test/backend_tests/binfile_test.clj b/backend/test/backend_tests/binfile_test.clj index 77ec4ec380..68ca50536d 100644 --- a/backend/test/backend_tests/binfile_test.clj +++ b/backend/test/backend_tests/binfile_test.clj @@ -801,6 +801,95 @@ (t/is (cfo/token-applied? token-radius shape [:r1 :r2 :r3 :r4])) (t/is (cfo/token-applied? token-opacity shape [:opacity])))))) +(t/deftest embed-assets-loads-the-library-components + ;; The library is read without `:realize?`, so its `:components` is an + ;; unloaded pointer map: `absorb-assets` walks it with `vals`, which calls + ;; `load!`, and that raises when no loader is bound. The absorb has to run + ;; under the library's own loader. This is a pre-existing failure, not a + ;; regression of the migration-on-read work: the migration path returns the + ;; pointer already loaded by the reattach comparison. + (let [profile (th/create-profile* 1) + library (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile) + :is-shared true}) + file (th/create-file* 2 {:profile-id (:id profile) + :project-id (:default-project-id profile)}) + lib-page (first (get-in library [:data :pages])) + file-page (first (get-in file [:data :pages])) + component-id (uuid/random) + main-id (uuid/random) + copy-id (uuid/random)] + + (th/link-file-to-library* {:file-id (:id file) :library-id (:id library)}) + + ;; The library owns one component. + (update-file! + :file-id (:id library) + :profile-id (:id profile) + :changes + [{:type :add-obj + :page-id lib-page + :id main-id + :parent-id uuid/zero + :frame-id uuid/zero + :components-v2 true + :obj (cts/setup-shape + {:id main-id + :name "Board" + :frame-id uuid/zero + :parent-id uuid/zero + :type :frame + :main-instance true + :component-root true + :component-file (:id library) + :component-id component-id})} + {:type :add-component + :path "" + :name "Board" + :main-instance-id main-id + :main-instance-page lib-page + :id component-id + :anotation nil}]) + + ;; The file instances it, so the absorb has a usage to embed. + (update-file! + :file-id (:id file) + :profile-id (:id profile) + :changes + [{:type :add-obj + :page-id file-page + :id copy-id + :parent-id uuid/zero + :frame-id uuid/zero + :components-v2 true + :obj (cts/setup-shape + {:id copy-id + :name "Board" + :frame-id uuid/zero + :parent-id uuid/zero + :type :frame + :main-instance false + :component-root true + :component-file (:id library) + :component-id component-id})}]) + + (t/testing "a plain library is absorbed" + (let [file-data (-> (bfc/get-file th/*system* (:id file) :realize? true) :data) + result (bfc/embed-assets th/*system* file-data (:id file))] + (t/is (contains? (:components result) component-id)))) + + (t/testing "a library that needed a migration is absorbed too" + ;; The scenario the review reported. It works even without the loader, + ;; because `reattach-unchanged` loads the pointers it puts back; it + ;; stays pinned so a future reattach that stops loading does not + ;; silently break the absorb. + (th/db-delete! :file-migration + {:file-id (:id library) + :name "0031-migrate-stroke-width-token-attr"}) + (let [file-data (-> (bfc/get-file th/*system* (:id file) :realize? true) :data) + result (bfc/embed-assets th/*system* file-data (:id file))] + (t/is (contains? (:components result) component-id)))))) + (t/deftest export-detach-libraries-preserves-own-tokens-and-status (let [profile (th/create-profile* 1) {:keys [library file page-id shape-id token-radius token-opacity]} @@ -1410,26 +1499,23 @@ (t/is (some? file-res)) (t/is (= 1 (count (:done file-res)))))))) -(t/deftest link-later-overwrite-import-no-resolution +(t/deftest import-rejects-an-in-place-import + ;; The in-place import was removed: it never carried the previous storage + ;; pointer, so it could leave the replaced object unreclaimed. A caller that + ;; still sets `::bfc/file-id` fails instead of importing as new files. (let [{:keys [profile file-id team-id]} (import-sample-file) - output (tmp/tempfile :suffix ".zip")] - ;; Export file with link-later - (v3/export-files! - (-> th/*system* - (assoc ::bfc/file-ids #{file-id}) - (assoc ::bfc/export-type :link-later) - (assoc ::bfc/output (io/output-stream output)))) - - ;; Import with overwrite (file-id set) - (let [result (-> th/*system* + output (tmp/tempfile :suffix ".zip") + cause (try + (-> th/*system* (assoc ::bfc/project-id (:default-project-id profile)) (assoc ::bfc/profile-id (:id profile)) (assoc ::bfc/team-id team-id) (assoc ::bfc/file-id file-id) (assoc ::bfc/input output) - (v3/import-files!))] - ;; Overwrite should have empty resolution - (t/is (= {} (:resolution result)))))) + (v3/import-files!)) + nil + (catch clojure.lang.ExceptionInfo cause cause))] + (t/is (= :not-supported (:code (ex-data cause)))))) ;; ----------------------------------------------------------------------------- ;; Category 2: Cross-Team Migration @@ -3082,17 +3168,51 @@ (:object-id thumb))) (t/is (some? (:media-id thumb))))))) -(t/deftest get-file-without-data-rejects-locks - ;; Locking is incompatible with `load-data? false`: the guard fires - ;; before any query, so no DB row is needed. Regression test for the - ;; `lock-for-share?`/`lock-for-share?` typo that left `lock-for-update?` - ;; unchecked. NOTE: the guard is an unconditional `when`/`throw` in - ;; `get-file*` (not an `assert`), so this test is not flag-dependent: - ;; it behaves the same with or without `enable-backend-asserts`. - (doseq [opt [:lock-for-share? :lock-for-update? :skip-locked?]] +(t/deftest get-file-without-data-can-lock + ;; Locking is legal without loading the data: the row-only query selects + ;; directly from `file`, so `FOR UPDATE`/`FOR SHARE` apply to the row and + ;; the lock lives until the enclosing transaction ends. `SKIP LOCKED` only + ;; modifies a locking read, so on its own it is ignored. The SQL is pinned + ;; because a plain `SELECT` also returns the row: the lock clause is what + ;; serializes with the other writers. + (let [profile (th/create-profile* 1) + file (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile)})] + (doseq [[opt sql] {:lock-for-share? (str bfc/sql:get-file-without-data " FOR SHARE of f") + :lock-for-update? (str bfc/sql:get-file-without-data " FOR UPDATE of f") + :skip-locked? bfc/sql:get-file-without-data}] + (let [[row sqls] (th/read-recording-sql + #(bfc/get-file th/*system* (:id file) :load-data? false opt true))] + (t/is (= (:id file) (:id row))) + (t/is (nil? (:data row))) + (t/is (= [sql] sqls)))))) + +(t/deftest get-file-rejects-realize-without-data + ;; `:realize?` walks the decoded data; asking for it without loading or + ;; decoding the data is a caller bug. The guard fires before any query, so + ;; no file row is needed. + (let [id (uuid/random)] (t/is (thrown? IllegalArgumentException - (bfc/get-file th/*system* (uuid/random) - :load-data? false opt true))))) + (bfc/get-file th/*system* id :realize? true :decode? false))) + (t/is (thrown? IllegalArgumentException + (bfc/get-file th/*system* id :realize? true :load-data? false))))) + +(t/deftest get-file-rejects-a-shared-lock-with-data + ;; The probe inherits the caller's lock and the migration branch re-reads + ;; with FOR UPDATE, so a shared lock on a data read is a lock upgrade. The + ;; guard fires before any query, so no row is needed. + (t/is (thrown? IllegalArgumentException + (bfc/get-file th/*system* (uuid/random) :lock-for-share? true)))) + +(t/deftest get-file-rejects-a-bare-pool-or-connection + ;; The read opens a transaction (unless it is read-only) and resolves the + ;; cfg, so it takes a cfg map only: a bare pool or connection is a caller + ;; bug. The guard is an `IllegalArgumentException`, not an `assert`, which + ;; is elided when the backend runs without `:backend-asserts`. + (let [id (uuid/random)] + (t/is (thrown? IllegalArgumentException (bfc/get-file th/*pool* id))) + (with-open [conn (db/open th/*system*)] + (t/is (thrown? IllegalArgumentException (bfc/get-file conn id)))))) ;; --- Library relations security tests diff --git a/backend/test/backend_tests/fdata_upsert_test.clj b/backend/test/backend_tests/fdata_upsert_test.clj index 62fd2913c2..0bfa89b6f1 100644 --- a/backend/test/backend_tests/fdata_upsert_test.clj +++ b/backend/test/backend_tests/fdata_upsert_test.clj @@ -40,12 +40,13 @@ :kind fchanges/snapshot-kind :updated-at past}) - (t/is (true? (fdata/upsert! th/*system* - {:id id - :file-id (:id file) - :type "snapshot" - :backend "legacy-db" - :data payload}))) + (t/is (true? (:written (fdata/upsert! th/*system* + {:id id + :file-id (:id file) + :type "snapshot" + :backend "legacy-db" + :storage-object-id nil + :data payload})))) (let [row (th/db-get :file-change {:id id} {::db/remove-deleted false})] (t/testing "the payload lands in the legacy column" @@ -53,3 +54,73 @@ (t/testing "the watermark moves to the moment of the mutation" (t/is (pos? (compare (inst-ms (:updated-at row)) (inst-ms past)))))))) + +(t/deftest upsert-reports-the-pointer-it-created + ;; The write reports the object it created, so a caller that persists again + ;; touches the current pointer instead of the replaced one. + (let [profile (th/create-profile* 1) + file (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile)}) + data (byte-array [1 2 3])] + + (let [out (db/tx-run! th/*system* + (fn [cfg] + (fdata/upsert! cfg {:id (:id file) + :file-id (:id file) + :type "main" + :backend "storage" + :storage-object-id nil + :data data})))] + (t/testing "a storage write reports the object it created" + (t/is (true? (:written out))) + (t/is (uuid? (:storage-object-id out))) + (t/is (= (:storage-object-id out) + (:storage-object-id (th/db-get :file-data + {:file-id (:id file) + :id (:id file)})))))) + + (let [previous (:storage-object-id (th/db-get :file-data + {:file-id (:id file) + :id (:id file)})) + out (db/tx-run! th/*system* + (fn [cfg] + (fdata/upsert! cfg {:id (:id file) + :file-id (:id file) + :type "main" + :backend "db" + :storage-object-id previous + :data data})))] + (t/testing "a db write has no object to report" + (t/is (true? (:written out))) + (t/is (nil? (:storage-object-id out))) + (t/is (nil? (:storage-object-id (th/db-get :file-data + {:file-id (:id file) + :id (:id file)})))))))) + +(t/deftest upsert-rejects-a-stale-pointer + ;; The write only replaces the row whose pointer the caller read. A stale + ;; map must fail instead of touching the wrong object and leaking the blob. + (let [profile (th/create-profile* 1) + file (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile)}) + data (byte-array [1 2 3]) + before (th/db-get :file-data {:file-id (:id file) :id (:id file)}) + cause (try + (db/tx-run! th/*system* + (fn [cfg] + (fdata/upsert! cfg {:id (:id file) + :file-id (:id file) + :type "main" + :backend "db" + :storage-object-id (uuid/next) + :data data}))) + nil + (catch clojure.lang.ExceptionInfo cause cause))] + + (t/is (= :stale-file-data-pointer (:code (ex-data cause)))) + + (let [after (th/db-get :file-data {:file-id (:id file) :id (:id file)})] + (t/testing "the row is untouched" + (t/is (= (:backend before) (:backend after))) + (t/is (= (:storage-object-id before) (:storage-object-id after))) + (t/is (= (vec (:data before)) (vec (:data after)))))))) diff --git a/backend/test/backend_tests/file_data_test.clj b/backend/test/backend_tests/file_data_test.clj new file mode 100644 index 0000000000..ff37dda751 --- /dev/null +++ b/backend/test/backend_tests/file_data_test.clj @@ -0,0 +1,323 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS INC Sucursal en España SL + +(ns backend-tests.file-data-test + (:require + [app.binfile.common :as bfc] + [app.common.time :as ct] + [app.common.uuid :as uuid] + [app.config :as cf] + [app.db :as db] + [app.features.fdata :as fdata] + [app.storage :as sto] + [app.util.blob :as blob] + [backend-tests.helpers :as th] + [clojure.test :as t])) + +(t/use-fixtures :once th/state-init) +(t/use-fixtures :each th/database-reset) + +;; The storage pointer of a `file_data` row is a column with a foreign key and +;; it is the only source of truth: the release is a coordinated restart and +;; the metadata no longer carries `storage-ref-id`. + +(defn- find-object + [id] + (th/db-get :storage-object {:id id} {::db/remove-deleted false})) + +(defn- put-object! + ([bucket metadata content] + (sto/put-object! (::sto/storage th/*system*) + (merge {::sto/content content + :content-type "application/octet-stream" + :bucket bucket} + metadata))) + ([bucket metadata] + (put-object! bucket metadata (sto/content "sobject")))) + +(defn- create-file! + [] + (let [profile (th/create-profile* 1)] + (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile)}))) + +(defn- upsert! + [params] + (db/tx-run! th/*system* + (fn [cfg] + (fdata/upsert! cfg params)))) + +(defn- resolve! + [row] + (db/tx-run! th/*system* + (fn [cfg] + (fdata/resolve-file-data cfg row)))) + +(t/deftest upsert-to-storage-writes-the-column + (let [file (create-file!) + data (blob/encode {:pages [] :pages-index {}})] + + ;; `main` always carries `id = file_id`: update the row the file already + ;; has instead of inventing a second one. + (upsert! {:id (:id file) + :file-id (:id file) + :type "main" + :backend "storage" + :storage-object-id nil + :data data + :metadata {:generated-by "test"}}) + + (let [row (th/db-get :file-data {:file-id (:id file) :id (:id file)}) + metadata (fdata/decode-metadata (:metadata row)) + sobject (:storage-object-id row)] + + (t/testing "the row points at the object with the column" + (t/is (= "storage" (:backend row))) + (t/is (nil? (:data row))) + (t/is (uuid? sobject))) + + (t/testing "the metadata no longer carries the pointer" + (t/is (nil? (:storage-ref-id metadata))) + (t/is (= "test" (:generated-by metadata)))) + + (t/testing "the object is stored in the file-data bucket" + (t/is (= "file-data" (:bucket (meta (sto/get-object (::sto/storage th/*system*) + sobject)))))) + + (t/testing "the row resolves to the stored bytes" + (t/is (= (vec data) (vec (:data (resolve! row))))))))) + +(t/deftest upsert-back-to-db-clears-the-column-and-touches-the-object + (let [file (create-file!)] + + (upsert! {:id (:id file) + :file-id (:id file) + :type "main" + :backend "storage" + :storage-object-id nil + :data (blob/encode {:pages [] :pages-index {}})}) + + (let [sobject (:storage-object-id (th/db-get :file-data + {:file-id (:id file) :id (:id file)}))] + + ;; The caller passes the object the persisted row points at; `upsert!` + ;; touches it before replacing the row. The dead metadata pointer must + ;; not survive the write either. + (upsert! {:id (:id file) + :file-id (:id file) + :type "main" + :backend "db" + :storage-object-id sobject + :data (blob/encode {:pages [] :pages-index {}}) + :metadata {:generated-by "test" + :storage-ref-id (uuid/next)}}) + + (let [row (th/db-get :file-data {:file-id (:id file) :id (:id file)}) + metadata (fdata/decode-metadata (:metadata row))] + + (t/testing "the hot row no longer references the object" + (t/is (= "db" (:backend row))) + (t/is (nil? (:storage-object-id row))) + (t/is (nil? (:storage-ref-id metadata))) + (t/is (= "test" (:generated-by metadata)))) + + (t/testing "the replaced object is touched for the storage GC" + (t/is (some? (:touched-at (find-object sobject))))))))) + +(t/deftest resolve-file-data-uses-the-column-and-ignores-the-metadata + (let [file (create-file!) + data (blob/encode {:pages [] :pages-index {}}) + obj (put-object! "file-data" + {:file-id (:id file) :id (:id file)} + (sto/content data)) + ;; A different, well-formed pointer in the metadata: it must be + ;; ignored now that the column is the only source of truth. + other (put-object! "file-data" + {:file-id (uuid/next) :id (uuid/next)})] + + (th/db-update! :file-data + {:backend "storage" + :data nil + :storage-object-id (:id obj) + :metadata (db/json {"storage-ref-id" (str (:id other))})} + {:file-id (:id file) :id (:id file)}) + + (t/testing "the column wins over the metadata" + (let [row (th/db-get :file-data {:file-id (:id file) :id (:id file)})] + (t/is (= (:id obj) (:storage-object-id row))) + (t/is (= (vec data) (vec (:data (resolve! row))))))))) + +(t/deftest gc-touched-freezes-an-object-referenced-only-by-the-column + (let [file (create-file!) + ;; The object metadata points at other ids: the freeze must come from + ;; the column, not from the old metadata lookup. + sobject (put-object! "file-data" + {:file-id (uuid/next) :id (uuid/next)})] + + (th/db-update! :storage-object + {:touched-at (ct/in-past {:hours 3})} + {:id (:id sobject)}) + + (th/db-update! :file-data + {:backend "storage" + :data nil + :storage-object-id (:id sobject)} + {:file-id (:id file) :id (:id file)}) + + (let [result (th/run-task! :storage-gc-touched {})] + (t/is (= 1 (:freeze result))) + (t/is (= 0 (:delete result))) + (t/is (nil? (:touched-at (find-object (:id sobject)))))))) + +(t/deftest update-file!-ignores-the-pointer-in-the-params + ;; `:storage-object-id` is bookkeeping of the persisted row, not an editable + ;; field: a param must not be able to drop the pointer and skip the touch. + (let [profile (th/create-profile* 1) + file (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile)})] + + (th/run-task! :offload-file-data {:file-id (:id file)}) + (th/run-task! :storage-gc-touched {:skip-delay true}) + + (let [sobject (:storage-object-id (th/db-get :file-data {:file-id (:id file) + :id (:id file)}))] + (t/is (uuid? sobject)) + (t/is (nil? (:touched-at (find-object sobject)))) + + (db/tx-run! th/*system* + (fn [cfg] + (let [file (bfc/get-file cfg (:id file) + :migrate? false + :lock-for-update? true)] + (bfc/update-file! cfg file + :storage-object-id nil + :modified-at (ct/now))))) + + (t/testing "the previous object is still touched" + (t/is (some? (:touched-at (find-object sobject)))))))) + +(t/deftest resolve-file-data-reports-a-missing-storage-object + (let [file (create-file!)] + + (th/db-update! :file-data + {:backend "storage" + :data nil + :storage-object-id nil} + {:file-id (:id file) :id (:id file)}) + + (let [cause (try + (db/tx-run! th/*system* + (fn [cfg] + (->> (th/db-get :file-data {:file-id (:id file) + :id (:id file)}) + (fdata/resolve-file-data cfg)))) + nil + (catch clojure.lang.ExceptionInfo cause cause))] + (t/is (some? cause)) + (t/is (= :missing-storage-object (:code (ex-data cause)))) + (t/is (= (:id file) (:file-id (ex-data cause))))))) + +(t/deftest load-pointer-reports-the-file-when-the-object-is-gone + (let [file (create-file!) + fragment (first (th/db-query :file-data {:file-id (:id file) + :type "fragment"})) + sobject (put-object! "file-data" + {:file-id (:id file) :id (:id fragment)})] + + (t/is (uuid? (:id fragment))) + + ;; The foreign key accepts the row (the object exists); a soft-deleted + ;; object is simply unreadable, which is the case the error reports. + (th/db-update! :storage-object + {:deleted-at (ct/in-past "1h")} + {:id (:id sobject)}) + (th/db-update! :file-data + {:backend "storage" + :data nil + :storage-object-id (:id sobject)} + {:file-id (:id file) :id (:id fragment)}) + + (let [cause (try + (db/tx-run! th/*system* + (fn [cfg] + (fdata/load-pointer cfg (:id file) (:id fragment)))) + nil + (catch clojure.lang.ExceptionInfo cause cause)) + data (ex-data cause)] + (t/is (some? cause)) + (t/is (= :missing-storage-object (:code data))) + (t/is (= (:id file) (:file-id data))) + (t/is (= (:id fragment) (:id data)))))) + +(t/deftest update-file!-rejects-a-map-without-data + ;; A map read with `:load-data? false` has no `:data` and no storage pointer; + ;; persisting it would wipe the content and skip the touch. The guard is an + ;; explicit raise, not an assert, so it also exists in production. + (let [file (create-file!) + cause (try + (db/tx-run! th/*system* + (fn [cfg] + (bfc/update-file! cfg (bfc/get-file cfg (:id file) + :load-data? false)))) + nil + (catch clojure.lang.ExceptionInfo cause cause))] + (t/is (some? cause)) + (t/is (= :missing-file-data (:code (ex-data cause)))))) + +(t/deftest update-file!-ignores-the-identity-in-the-params + ;; The params change mutable fields only: `:id` cannot retarget the UPDATE. + (let [file (create-file!) + other (uuid/next) + when (ct/inst "2020-01-01T00:00:00Z")] + + (db/tx-run! th/*system* + (fn [cfg] + (let [file (bfc/get-file cfg (:id file) + :migrate? false + :lock-for-update? true)] + (bfc/update-file! cfg file :id other :modified-at when)))) + + (t/testing "the read file is updated" + (t/is (= when (:modified-at (th/db-get :file {:id (:id file)}))))) + + (t/testing "the id in the params is ignored" + (t/is (nil? (th/db-get :file {:id other})))))) + +(t/deftest update-file!-touches-the-previous-object-on-a-storage-write + ;; With `:file-data-backend` = "storage", every ordinary save replaces the + ;; object of the row: the previous one must be touched so the storage GC can + ;; reclaim it. The storage -> db and db -> storage directions are covered + ;; above; this pins the configuration where storage is the default. + (with-redefs [cf/get (th/config-get-mock {:file-data-backend "storage"})] + (let [profile (th/create-profile* 1) + file (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile)}) + previous (:storage-object-id (th/db-get :file-data + {:file-id (:id file) :id (:id file)}))] + + (t/is (uuid? previous)) + + ;; A new object is born touched; clear the mark so the assertion below + ;; proves the save is what touches it. + (th/db-update! :storage-object {:touched-at nil} {:id previous}) + (t/is (nil? (:touched-at (find-object previous)))) + + (db/tx-run! th/*system* + (fn [cfg] + (let [file (bfc/get-file cfg (:id file) + :migrate? false + :lock-for-update? true)] + (bfc/update-file! cfg file :modified-at (ct/now))))) + + (let [row (th/db-get :file-data {:file-id (:id file) :id (:id file)}) + current (:storage-object-id row)] + + (t/testing "the row points at a new object" + (t/is (uuid? current)) + (t/is (not= previous current))) + + (t/testing "the previous object is touched for the storage GC" + (t/is (some? (:touched-at (find-object previous))))))))) diff --git a/backend/test/backend_tests/file_migration_read_test.clj b/backend/test/backend_tests/file_migration_read_test.clj new file mode 100644 index 0000000000..da5821d21f --- /dev/null +++ b/backend/test/backend_tests/file_migration_read_test.clj @@ -0,0 +1,365 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS INC Sucursal en España SL + +(ns backend-tests.file-migration-read-test + "Persisting the migrations applied on read is an optimization of the read: + the read already returns migrated data, so the write must never overwrite a + row written after that read, and it must keep a single decoded copy of the + file in the heap." + (:require + [app.binfile.common :as bfc] + [app.common.features :as cfeat] + [app.common.files.migrations :as fmg] + [app.common.uuid :as uuid] + [app.config :as cf] + [app.db :as db] + [app.features.fdata :as fdata] + [app.features.file-migrations :as fmigr] + [app.rpc :as-alias rpc] + [app.util.pointer-map :as pmap] + [backend-tests.helpers :as th] + [clojure.test :as t])) + +(t/use-fixtures :once th/state-init) +(t/use-fixtures :each th/database-reset) + +(def ^:private migration-name + "0031-migrate-stroke-width-token-attr") + +(defn- create-file! + [] + (let [profile (th/create-profile* 1) + file (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile) + :is-shared false})] + {:profile-id (:id profile) + :file-id (:id file) + :page-id (first (:pages (:data (bfc/get-file th/*system* (:id file)))))})) + +(defn- force-migration! + "Make the next read see a pending migration, the way the snapshot tests do: + the version matches, so only a missing row triggers the migration." + [file-id] + (th/db-delete! :file-migration + {:file-id file-id + :name migration-name})) + +(defn- file-data-row + [file-id] + (th/db-get :file-data {:file-id file-id :id file-id})) + +(defn- storage-object + [id] + (th/db-get :storage-object {:id id} {::db/remove-deleted false})) + +(defn- save! + "Run a real save request so the test exercises the same write path a client + takes, not a hand-built row." + [profile-id file-id revn changes] + (th/command! {::th/type :update-file + ::rpc/profile-id profile-id + :id file-id + :session-id (uuid/random) + :revn revn + :vern 0 + :features (cfeat/get-enabled-features cf/flags) + :changes changes})) + +(t/deftest migrate-on-read-does-not-overwrite-a-concurrent-save + ;; A save commits after the read that saw the pending migration. The read + ;; must decide again on the map it takes under the `file` row lock, so it + ;; writes nothing: the save survives, `revn` never goes back (a client whose + ;; revn is higher than the stored one cannot save again) and the object the + ;; save left stays referenced instead of becoming a frozen orphan. + (with-redefs [cf/get (th/config-get-mock {:file-data-backend "storage"})] + (let [{:keys [profile-id file-id page-id]} (create-file!) + shape-id (uuid/random) + fired? (atom false) + replaced (atom nil) + saved-at (atom nil) + saved-revn (atom nil)] + + (force-migration! file-id) + + (with-redefs [fmigr/resolve-applied-migrations + (let [orig fmigr/resolve-applied-migrations] + (fn [cfg file] + (let [result (orig cfg file)] + (when (compare-and-set! fired? false true) + ;; The save commits once the read has already + ;; seen the pending migration. It applies the + ;; migration on its own and records it. + (let [out (save! profile-id file-id 0 + [(th/add-rect-change page-id shape-id)])] + (t/is (nil? (:error out)))) + + (let [row (file-data-row file-id)] + (reset! replaced (:storage-object-id row)) + (reset! saved-revn (:revn (th/db-get :file {:id file-id}))) + (reset! saved-at (:modified-at (th/db-get :file {:id file-id})))) + + ;; Freeze the object the save left, as the storage + ;; GC would if it saw it before a stale write + ;; replaced the row again: a write that does not + ;; touch it leaks the blob forever. + (th/db-update! :storage-object + {:touched-at nil} + {:id @replaced})) + result)))] + + (let [returned (db/tx-run! th/*system* + (fn [cfg] (bfc/get-file cfg file-id))) + row (th/db-get :file {:id file-id}) + persisted (db/tx-run! th/*system* + (fn [cfg] (bfc/get-file cfg file-id + :migrate? false + :realize? true)))] + + (t/testing "the save survives the read" + (t/is (= @saved-revn (:revn returned))) + (t/is (= @saved-revn (:revn row))) + (t/is (= @saved-at (:modified-at row))) + (t/is (some? (get-in persisted [:data :pages-index page-id :objects shape-id])))) + + (t/testing "the read writes nothing and the file ends migrated" + (t/is (not (fmg/need-migration? returned))) + (t/is (= @replaced (:storage-object-id (file-data-row file-id)))) + (t/is (seq (th/db-query :file-migration + {:file-id file-id + :name migration-name})))) + + (t/testing "the next save of the same client does not conflict" + (let [out (save! profile-id file-id @saved-revn + [(th/add-rect-change page-id (uuid/random))])] + (t/is (nil? (:error out)))))))))) + +(t/deftest migration-already-applied-by-another-reader-is-not-persisted + ;; Two readers see the migration pending; the first to get the lock migrates + ;; and persists. The second must re-check on its locked read and write + ;; nothing (a second write would create another object for the same data). + (with-redefs [cf/get (th/config-get-mock {:file-data-backend "storage"})] + (let [{:keys [file-id]} (create-file!) + fired? (atom false) + pointer (atom nil)] + + (force-migration! file-id) + + (with-redefs [fmg/need-migration? + (let [orig fmg/need-migration?] + (fn [file] + (let [result (orig file)] + (when (and result (compare-and-set! fired? false true)) + ;; The other reader wins the lock, migrates and + ;; persists before this one gets it. + (db/tx-run! th/*system* + (fn [cfg] (bfc/get-file cfg file-id))) + (reset! pointer (:storage-object-id (file-data-row file-id)))) + result)))] + + (db/tx-run! th/*system* + (fn [cfg] (bfc/get-file cfg file-id))) + + (t/testing "the reader that lost the race does not write" + (t/is (uuid? @pointer)) + (t/is (= @pointer (:storage-object-id (file-data-row file-id))))) + + (t/testing "the file ends migrated" + (t/is (seq (th/db-query :file-migration + {:file-id file-id + :name migration-name})))))))) + +(t/deftest migration-read-loads-the-file-data-once + ;; The probe decides with a row read without data, so the migration path + ;; keeps a single decoded copy of the file in the heap (a large file can be + ;; hundreds of MB and a second copy can exhaust it). + (let [{:keys [file-id]} (create-file!) + reads (atom 0)] + + (force-migration! file-id) + + (with-redefs [fdata/resolve-file-data + (let [orig fdata/resolve-file-data] + (fn [cfg file] + (when (= file-id (:id file)) + (swap! reads inc)) + (orig cfg file)))] + + (db/tx-run! th/*system* + (fn [cfg] (bfc/get-file cfg file-id))) + + (t/is (= 1 @reads))))) + +(t/deftest realize-runs-once-per-read-path + ;; `fdata/realize` is a full walk of the decoded data. The read owns it once + ;; per path; the migration path pays it twice on purpose, once inside + ;; `migrate-file` (the chain needs plain data) and once on its result. + (let [{:keys [file-id]} (create-file!) + calls (atom 0) + read! (fn [opts] + (reset! calls 0) + (with-redefs [fdata/realize + (let [orig fdata/realize] + (fn [cfg file] + (swap! calls inc) + (orig cfg file)))] + (db/tx-run! th/*system* + (fn [cfg] (bfc/get-file cfg file-id opts)))))] + (t/testing "a plain read realizes once" + (read! {:realize? true}) + (t/is (= 1 @calls))) + + (t/testing "a read-only read realizes once" + (read! {:realize? true :read-only? true}) + (t/is (= 1 @calls))) + + (t/testing "the migration path realizes twice" + (force-migration! file-id) + (read! {:realize? true}) + (t/is (= 2 @calls))))) + +(t/deftest row-only-read-resolves-the-applied-migrations-on-demand + ;; A row-only read resolves the applied migrations by default, because the + ;; row can be completed later and the caller may need them; the explicit + ;; opt-out leaves them out. + (let [{:keys [file-id]} (create-file!) + with-migrations (bfc/get-file th/*system* file-id :load-data? false) + without (bfc/get-file th/*system* file-id + :load-data? false + :load-migrations? false)] + (t/is (seq (:migrations with-migrations))) + (t/is (nil? (:migrations without))))) + +(t/deftest migrating-read-returns-the-pointer-it-persisted + ;; The map a migrating read returns must carry the pointer of the row it + ;; left: a caller that writes with it touches that object instead of the + ;; replaced one. The storage GC freezes a referenced object, so a write + ;; that misses it leaks the blob forever. + (with-redefs [cf/get (th/config-get-mock {:file-data-backend "storage"})] + (let [{:keys [file-id]} (create-file!)] + (force-migration! file-id) + (let [file (db/tx-run! th/*system* + (fn [cfg] (bfc/get-file cfg file-id))) + left (:storage-object-id (file-data-row file-id))] + + (t/testing "the read reports the pointer of the persisted row" + (t/is (uuid? left)) + (t/is (= left (:storage-object-id file)))) + + ;; Freeze the object while the row still references it, as the storage + ;; GC does; the later write must touch it (see the concurrent-save + ;; test for the same technique). + (th/db-update! :storage-object {:touched-at nil} {:id left}) + + (db/tx-run! th/*system* + (fn [cfg] (bfc/update-file! cfg file))) + + (let [row (file-data-row file-id)] + (t/testing "the write replaces the row and touches the object it left" + (t/is (not= left (:storage-object-id row))) + (t/is (some? (:touched-at (storage-object left)))))))))) + +(t/deftest read-completes-with-a-single-row-read + ;; The common read is one row read (with its data columns) and + ;; `attach-file-data` completes it. + (let [{:keys [file-id page-id]} (create-file!) + [file sqls] (th/read-recording-sql + #(db/tx-run! th/*system* + (fn [cfg] (bfc/get-file cfg file-id))))] + (t/is (= [bfc/sql:get-file] sqls)) + (t/is (some? (get-in file [:data :pages-index page-id]))))) + +(t/deftest read-resolves-the-applied-migrations-once + ;; The common read resolves the applied migrations once. The migration path + ;; resolves them again on the locked re-read (it must decide on the fresh + ;; row) and once more after persisting (to return the applied set); no path + ;; resolves them twice without a reason. + (let [{:keys [file-id]} (create-file!) + calls (atom 0) + read! (fn [] + (reset! calls 0) + (with-redefs [fmigr/resolve-applied-migrations + (let [orig fmigr/resolve-applied-migrations] + (fn [cfg file] + (swap! calls inc) + (orig cfg file)))] + (db/tx-run! th/*system* + (fn [cfg] (bfc/get-file cfg file-id)))))] + + (read!) + (t/is (= 1 @calls)) + + (force-migration! file-id) + (read!) + (t/is (= 3 @calls)))) + +(t/deftest migration-read-rereads-the-row-under-lock + ;; The first read decides; only when a migration is pending the row is read + ;; again, and that second read takes the `file` row lock. + (let [{:keys [file-id]} (create-file!)] + (force-migration! file-id) + (let [[_ sqls] (th/read-recording-sql + #(db/tx-run! th/*system* + (fn [cfg] (bfc/get-file cfg file-id))))] + (t/is (= [bfc/sql:get-file + (str bfc/sql:get-file " FOR UPDATE of f")] + sqls))))) + +(t/deftest read-only-migration-never-writes + ;; A read-only context migrates in memory: no lock, no row write and no + ;; storage write. The pending migration stays pending. + (with-redefs [cf/get (th/config-get-mock {:file-data-backend "storage"})] + (let [{:keys [file-id page-id]} (create-file!) + pointer (:storage-object-id (file-data-row file-id))] + + (force-migration! file-id) + + (let [returned (db/tx-run! th/*system* + (fn [cfg] (bfc/get-file cfg file-id :read-only? true)))] + + (t/testing "the read returns migrated, realized data" + (t/is (not (pmap/pointer-map? (get-in returned [:data :pages-index page-id]))))) + + (t/testing "nothing is persisted" + (t/is (= pointer (:storage-object-id (file-data-row file-id)))) + (t/is (empty? (th/db-query :file-migration + {:file-id file-id + :name migration-name})))))))) + +(t/deftest read-only-cfg-migration-never-writes + ;; A cfg marked `::db/read-only` migrates in memory too: `db/tx-run!` opens + ;; the transaction read-only, `get-file*` sees the connection read-only and + ;; takes the no-persist branch. The config flag that fills this key is + ;; legacy, but the key is set by hand (objects-gc, the REPL), so the read + ;; must keep working under it. + (with-redefs [cf/get (th/config-get-mock {:file-data-backend "storage"})] + (let [{:keys [file-id page-id]} (create-file!) + pointer (:storage-object-id (file-data-row file-id))] + + (force-migration! file-id) + + (let [returned (db/tx-run! (assoc th/*system* ::db/read-only true) + (fn [cfg] (bfc/get-file cfg file-id)))] + + (t/testing "the read returns migrated data" + (t/is (not (pmap/pointer-map? (get-in returned [:data :pages-index page-id]))))) + + (t/testing "nothing is persisted" + (t/is (= pointer (:storage-object-id (file-data-row file-id)))) + (t/is (empty? (th/db-query :file-migration + {:file-id file-id + :name migration-name})))))))) + +(t/deftest read-only-read-does-not-open-a-transaction + ;; A read-only read takes no lock and writes nothing, so `get-file` runs it + ;; on a plain connection instead of opening a transaction of its own. The + ;; redefinition proves both directions: the same call without the option + ;; does open one. + (let [{:keys [file-id]} (create-file!)] + (with-redefs [db/tx-run! (fn [& _] + (throw (ex-info "opened a transaction" {})))] + (t/is (some? (bfc/get-file th/*system* file-id :read-only? true))) + (t/is (thrown? clojure.lang.ExceptionInfo + (bfc/get-file th/*system* file-id)))))) diff --git a/backend/test/backend_tests/file_pointer_snapshot_test.clj b/backend/test/backend_tests/file_pointer_snapshot_test.clj index 8d1fcd34ab..359ab7b85a 100644 --- a/backend/test/backend_tests/file_pointer_snapshot_test.clj +++ b/backend/test/backend_tests/file_pointer_snapshot_test.clj @@ -837,7 +837,8 @@ (t/deftest migration-keeps-unchanged-page-fragments ;; Applying a pending migration on read keeps the fragment ids of the ;; pages the migration does not change; a page it changes gets a new - ;; id. The caller still receives realized data, not pointers. + ;; id. The read returns decoded data with pointers for the pages the + ;; migration does not change; `:realize?` is what returns it plain. (let [profile (th/create-profile* 1) file (th/create-file* 1 {:profile-id (:id profile) :project-id (:default-project-id profile) @@ -865,7 +866,9 @@ (let [returned (db/tx-run! th/*system* (fn [cfg] (bfc/get-file cfg file-id)))] - (t/is (not (pmap/pointer-map? (get-in returned [:data :pages-index page-a])))) + ;; The migration changed page-b, so it comes plain; page-a keeps + ;; its pointer (and its fragment id) until someone realizes it. + (t/is (pmap/pointer-map? (get-in returned [:data :pages-index page-a]))) (t/is (not (pmap/pointer-map? (get-in returned [:data :pages-index page-b])))) (t/is (not (fmg/need-migration? returned))) diff --git a/backend/test/backend_tests/helpers.clj b/backend/test/backend_tests/helpers.clj index 378450f056..1c7a76973d 100644 --- a/backend/test/backend_tests/helpers.clj +++ b/backend/test/backend_tests/helpers.clj @@ -368,8 +368,8 @@ :or {session-id (uuid/next) revn 0}}] (-> system (assoc ::files.update/timestamp (ct/now)) - (db/tx-run! (fn [{:keys [::db/conn] :as system}] - (let [file (files.update/get-file conn file-id)] + (db/tx-run! (fn [system] + (let [file (files.update/get-file system file-id)] (#'files.update/update-file* system {:id file-id :revn revn @@ -842,6 +842,19 @@ [& params] (apply db/get* *pool* params)) +(defn read-recording-sql + "Run `f` and return [result sqls], where `sqls` are the SQL statements of + every `db/get-with-sql` call. Those are exactly the file reads of + `binfile.common/get-file`: the fragment loads go through `db/get*`." + [f] + (let [sqls (atom [])] + (with-redefs [db/get-with-sql + (let [orig db/get-with-sql] + (fn [ds sql & opts] + (swap! sqls conj (first sql)) + (apply orig ds sql opts)))] + [(f) @sqls]))) + (defn sleep [ms-or-duration] (Thread/sleep (inst-ms (ct/duration ms-or-duration)))) diff --git a/backend/test/backend_tests/http_debug_test.clj b/backend/test/backend_tests/http_debug_test.clj index 9c874601f4..d2ac1ee20d 100644 --- a/backend/test/backend_tests/http_debug_test.clj +++ b/backend/test/backend_tests/http_debug_test.clj @@ -10,7 +10,9 @@ resolve the pointer-mapped pages with the file's own loader." (:require [app.http.debug :as debug] + [app.storage :as sto] [backend-tests.helpers :as th] + [clojure.string :as str] [clojure.test :as t] [yetti.response :as-alias yres])) @@ -27,3 +29,28 @@ (t/is (= 200 (::yres/status result))) (t/is (= "NO VALIDATION ERRORS FOUND" (::yres/body result))))) + +(t/deftest debug-download-does-not-expose-the-storage-object-id + (let [profile (th/create-profile* 1) + file (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile) + :is-shared false}) + sobject (sto/put-object! (::sto/storage th/*system*) + {::sto/content (sto/content "sobject") + :content-type "application/octet-stream" + :bucket "file-data" + :file-id (:id file) + :id (:id file)}) + _ (th/db-update! :file-data {:storage-object-id (:id sobject)} + {:file-id (:id file) + :id (:id file)}) + handler (deref #'debug/download-file-data) + result (handler th/*system* {:params {:file-id (str (:id file)) + :download true}}) + body (String. ^bytes (::yres/body result))] + + (t/is (= 200 (::yres/status result))) + (t/is (str/includes? (get-in result [::yres/headers "content-disposition"]) + "attachment")) + (t/is (not (str/includes? body "storage-object-id"))) + (t/is (not (str/includes? body "storage_object_id"))))) diff --git a/backend/test/backend_tests/migrations_test.clj b/backend/test/backend_tests/migrations_test.clj index 12597a4968..9a93eaecac 100644 --- a/backend/test/backend_tests/migrations_test.clj +++ b/backend/test/backend_tests/migrations_test.clj @@ -6,7 +6,9 @@ (ns backend-tests.migrations-test (:require + [app.common.uuid :as uuid] [app.db :as db] + [app.features.fdata :as fdata] [app.migrations :as migrations] [app.util.migrations :as mg] [backend-tests.helpers :as th] @@ -15,6 +17,7 @@ [clojure.test :as t])) (t/use-fixtures :once th/state-init) +(t/use-fixtures :each th/database-reset) (t/deftest migrations-registry-satisfies-steps-spec (t/testing "every entry is a {:name :fn} step (guards against stray forms)" @@ -51,6 +54,18 @@ (map :conname) set)) +(defn- foreign-key-flags + "The deferral and delete-action flags of one foreign key. `condeferred` is + true only for `INITIALLY DEFERRED`." + [table name] + (-> (th/db-exec! ["SELECT condeferrable, condeferred, confdeltype, convalidated + FROM pg_constraint + WHERE contype = 'f' + AND conname = ? + AND conrelid = ?::regclass" + name table]) + first)) + (t/deftest job-table-exists-with-expected-columns (t/is (= #{; dispatch/lifecycle columns "id" "name" "tenant" "queue" "label" "priority" "scheduled_at" @@ -216,3 +231,205 @@ (t/is (not (contains? triggers "job__modified_at__tgr")))) (t/testing "deletion protection is the only trigger on job" (t/is (= #{"deletion_protection__tgr"} triggers))))) + +;; --- File data storage reference (migration 0159) + +(def ^:private file-data-migration + (mg/resource "app/migrations/sql/0159-add-file-data-storage-object-id.sql")) + +(defn- apply-file-data-migration! + [] + (db/tx-run! (::db/pool th/*system*) + (fn [{:keys [::db/conn]}] + (file-data-migration conn)))) + +(t/deftest file-data-storage-object-reference-is-a-deferrable-no-action-fk + (t/testing "the pointer column exists" + (t/is (contains? (table-columns "file_data") "storage_object_id"))) + (t/testing "the index that serves the FK check and the GC scan exists" + (t/is (contains? (table-indexes "file_data") + "file_data__storage_object_id__idx"))) + (t/testing "the reference does not cascade" + (t/is (contains? (table-foreign-keys "file_data") + ["file_data_storage_object_id_fkey" "a"]))) + (t/testing "the reference is DEFERRABLE but checked immediately" + ;; The storage GC catches the violation inside the chunk transaction. An + ;; `INITIALLY DEFERRED` check fires at commit, outside the catch, and the + ;; sweep would never move past the blocked chunk. + (t/is (= {:condeferrable true + :condeferred false + :confdeltype "a" + :convalidated true} + (foreign-key-flags "file_data" + "file_data_storage_object_id_fkey"))))) + +(t/deftest deleting-a-referenced-storage-object-fails + (let [profile (th/create-profile* 1) + file (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile)}) + sobject (:id (th/db-insert! :storage-object {:backend "fs"})) + data-id (uuid/next)] + + (th/db-insert! :file-data + {:file-id (:id file) + :id data-id + :type "snapshot" + :backend "storage" + :storage-object-id sobject}) + + (t/testing "a live reference refuses the physical delete" + (let [cause (try + (th/db-exec! ["DELETE FROM storage_object WHERE id = ?" sobject]) + nil + (catch java.sql.SQLException cause cause))] + (t/is (some? cause)) + (t/is (= "23503" (.getSQLState cause))))) + + (th/db-delete! :file-data {:file-id (:id file) :id data-id}) + + (t/testing "without the reference the delete succeeds" + (th/db-exec! ["DELETE FROM storage_object WHERE id = ?" sobject]) + (t/is (nil? (th/db-get :storage-object {:id sobject})))))) + +(t/deftest file-data-migration-backfills-and-is-idempotent + (let [profile (th/create-profile* 1) + file (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile)}) + sobject (:id (th/db-insert! :storage-object {:backend "fs"})) + data-id (uuid/next)] + + (th/db-insert! :file-data + {:file-id (:id file) + :id data-id + :type "snapshot" + :backend "storage" + :metadata (db/json {"storage-ref-id" (str sobject)})}) + + (t/testing "the backfill copies the metadata pointer into the column" + (apply-file-data-migration!) + (t/is (= sobject (:storage-object-id (th/db-get :file-data + {:file-id (:id file) + :id data-id}))))) + + (t/testing "the backfill removes the dead metadata pointer" + (let [row (th/db-get :file-data {:file-id (:id file) :id data-id}) + metadata (fdata/decode-metadata (:metadata row))] + (t/is (nil? (:storage-ref-id metadata))))) + + (t/testing "a second application changes nothing" + (let [before (th/db-get :file-data {:file-id (:id file) :id data-id})] + (apply-file-data-migration!) + (t/is (= before (th/db-get :file-data {:file-id (:id file) + :id data-id}))))))) + +(t/deftest file-data-migration-guards-corrupt-and-orphan-pointers + (let [profile (th/create-profile* 1) + file (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile)}) + sobject (:id (th/db-insert! :storage-object {:backend "fs"})) + valid-id (uuid/next) + corrupt-id (uuid/next) + orphan-id (uuid/next) + other-id (uuid/next)] + + (th/db-insert! :file-data + {:file-id (:id file) :id valid-id :type "snapshot" :backend "storage" + :metadata (db/json {"storage-ref-id" (str sobject)})}) + (th/db-insert! :file-data + {:file-id (:id file) :id corrupt-id :type "snapshot" :backend "storage" + :metadata (db/json {"storage-ref-id" "not-a-uuid"})}) + (th/db-insert! :file-data + {:file-id (:id file) :id orphan-id :type "snapshot" :backend "storage" + :metadata (db/json {"storage-ref-id" (str (uuid/next))})}) + (th/db-insert! :file-data + {:file-id (:id file) :id other-id :type "snapshot" :backend "db" + :metadata (db/json {"storage-ref-id" (str sobject)})}) + + (t/testing "the migration applies without aborting" + (t/is (true? (apply-file-data-migration!)))) + + (t/testing "a valid pointer is backfilled" + (t/is (= sobject (:storage-object-id (th/db-get :file-data + {:file-id (:id file) + :id valid-id}))))) + + (t/testing "a non-uuid value stays NULL" + (t/is (nil? (:storage-object-id (th/db-get :file-data + {:file-id (:id file) + :id corrupt-id}))))) + + (t/testing "a pointer to a missing object stays NULL" + (t/is (nil? (:storage-object-id (th/db-get :file-data + {:file-id (:id file) + :id orphan-id}))))) + + (t/testing "rows of other backends are not touched" + (t/is (nil? (:storage-object-id (th/db-get :file-data + {:file-id (:id file) + :id other-id}))))))) + +(t/deftest file-data-migration-creates-a-missing-index + ;; In the test database the index already exists, so the creation branch is + ;; only reachable by dropping it first. It is the branch a fresh deployment + ;; runs. + (th/db-exec! ["DROP INDEX file_data__storage_object_id__idx"]) + (t/is (not (contains? (table-indexes "file_data") + "file_data__storage_object_id__idx"))) + + (t/is (true? (apply-file-data-migration!))) + + (t/is (contains? (table-indexes "file_data") + "file_data__storage_object_id__idx"))) + +(t/deftest file-data-migration-rebuilds-a-wrong-foreign-key + ;; The migration owns the definition. A constraint created by hand with + ;; CASCADE would delete the referencing rows instead of refusing the delete, + ;; so it is replaced with the NO ACTION one. + (th/db-exec! ["ALTER TABLE file_data + DROP CONSTRAINT file_data_storage_object_id_fkey"]) + (try + (th/db-exec! ["ALTER TABLE file_data + ADD CONSTRAINT file_data_storage_object_id_fkey + FOREIGN KEY (storage_object_id) REFERENCES storage_object(id) + ON DELETE CASCADE DEFERRABLE"]) + (t/is (true? (apply-file-data-migration!))) + (t/is (= "a" (:confdeltype + (foreign-key-flags "file_data" + "file_data_storage_object_id_fkey")))) + (finally + (apply-file-data-migration!)))) + +(t/deftest file-data-migration-rebuilds-a-non-deferrable-foreign-key + ;; Without DEFERRABLE the constraint breaks `SET CONSTRAINTS ALL DEFERRED` + ;; and `database-reset`; the migration recreates it deferrable. + (th/db-exec! ["ALTER TABLE file_data + DROP CONSTRAINT file_data_storage_object_id_fkey"]) + (try + (th/db-exec! ["ALTER TABLE file_data + ADD CONSTRAINT file_data_storage_object_id_fkey + FOREIGN KEY (storage_object_id) REFERENCES storage_object(id) + ON DELETE NO ACTION"]) + (t/is (true? (apply-file-data-migration!))) + (t/is (true? (:condeferrable + (foreign-key-flags "file_data" + "file_data_storage_object_id_fkey")))) + (finally + (apply-file-data-migration!)))) + +(t/deftest file-data-migration-rebuilds-a-deferred-foreign-key + ;; An `INITIALLY DEFERRED` check fires at commit, outside the + ;; `storage-gc-deleted` chunk transaction, so the sweep would never move past + ;; a blocked chunk. The migration must leave the check immediate. + (th/db-exec! ["ALTER TABLE file_data + DROP CONSTRAINT file_data_storage_object_id_fkey"]) + (try + (th/db-exec! ["ALTER TABLE file_data + ADD CONSTRAINT file_data_storage_object_id_fkey + FOREIGN KEY (storage_object_id) REFERENCES storage_object(id) + ON DELETE NO ACTION DEFERRABLE INITIALLY DEFERRED"]) + (t/is (true? (apply-file-data-migration!))) + (t/is (false? (:condeferred + (foreign-key-flags "file_data" + "file_data_storage_object_id_fkey")))) + (finally + (apply-file-data-migration!)))) diff --git a/backend/test/backend_tests/objects_gc_test.clj b/backend/test/backend_tests/objects_gc_test.clj index 646650a4ca..462cbc5f5a 100644 --- a/backend/test/backend_tests/objects_gc_test.clj +++ b/backend/test/backend_tests/objects_gc_test.clj @@ -287,6 +287,34 @@ (t/is (nil? (th/db-get :team {:id (:id team)} {::db/remove-deleted false}))) (t/is (empty? (:call-args-list @audit-mock)))))) +(t/deftest gc-deletes-file-data-and-touches-its-storage-object + (let [profile (th/create-profile* 1) + file (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile)}) + sobject (sto/put-object! (::sto/storage th/*system*) + {::sto/content (sto/content "file-data") + :content-type "application/octet-stream" + :bucket "file-data" + :file-id (:id file) + :id (:id file)})] + + ;; `main` always carries `id = file_id`: mark the real row as deleted + ;; instead of inventing a second main row. + (th/db-update! :file-data + {:backend "storage" + :data nil + :storage-object-id (:id sobject) + :deleted-at (ct/in-past "30m")} + {:file-id (:id file) :id (:id file)}) + + (th/run-task! :objects-gc {}) + + (t/testing "the row is purged" + (t/is (nil? (th/db-get :file-data {:file-id (:id file) :id (:id file)})))) + + (t/testing "the storage object is touched for the storage GC" + (t/is (some? (:touched-at (th/db-get :storage-object {:id (:id sobject)}))))))) + (t/deftest gc-emits-events-only-for-committed-chunks (with-mocks [audit-mock {:target 'app.loggers.audit/submit :return nil}] (with-redefs [cf/flags (conj cf/flags :audit-log)] diff --git a/backend/test/backend_tests/offload_file_data_test.clj b/backend/test/backend_tests/offload_file_data_test.clj index e7158f0567..8f4fad28a9 100644 --- a/backend/test/backend_tests/offload_file_data_test.clj +++ b/backend/test/backend_tests/offload_file_data_test.clj @@ -62,6 +62,42 @@ (t/is (every? #(= "storage" (:backend %)) after)) (t/is (every? #(nil? (:data %)) after))))) +(t/deftest offloads-a-file-data-row-with-metadata + ;; `SELECT fd.*` returns jsonb metadata as a `PGobject`; the offload has to + ;; decode it before `fdata/upsert!` validates the row. Imported files carry + ;; metadata, so without the normalization the job never offloads them. + (let [profile (th/create-profile* 1) + file (create-file 1 profile)] + + (th/db-update! :file-data + {:metadata (db/json {"generated-by" "offload-test"})} + {:file-id (:id file) :id (:id file)}) + + (let [result (th/run-task! :offload-file-data {:file-id (:id file)}) + row (th/db-get :file-data {:file-id (:id file) :id (:id file)})] + (t/is (pos? (:offloaded result))) + (t/is (= "storage" (:backend row))) + (t/is (uuid? (:storage-object-id row))) + (t/is (= {:generated-by "offload-test"} + (fdata/decode-metadata (:metadata row))))))) + +(t/deftest offloads-a-file-data-row-with-a-json-null-metadata + ;; A jsonb `null` decodes to nil; the key must disappear from the row instead + ;; of travelling as `:metadata nil`, which the update schema rejects. + (let [profile (th/create-profile* 1) + file (create-file 1 profile)] + + (th/db-exec! ["UPDATE file_data SET metadata = 'null'::jsonb + WHERE file_id = ? AND id = ?" + (:id file) (:id file)]) + + (let [result (th/run-task! :offload-file-data {:file-id (:id file)}) + row (th/db-get :file-data {:file-id (:id file) :id (:id file)})] + (t/is (pos? (:offloaded result))) + (t/is (= "storage" (:backend row))) + (t/is (uuid? (:storage-object-id row))) + (t/is (nil? (:metadata row)))))) + (t/deftest skips-file-locked-by-another-transaction (let [profile (th/create-profile* 1) file (create-file 1 profile) @@ -385,36 +421,65 @@ (let [result (th/run-task! :offload-file-data {:file-id (:id file)})] (t/is (pos? (:offloaded result)))) - (t/testing "live snapshot rows moved to cold storage" - (let [snapshots (th/db-query :file-data {:file-id (:id file) - :type "snapshot" - :deleted-at nil})] - (t/is (pos? (count snapshots))) - (t/is (every? #(= "storage" (:backend %)) snapshots)) - (t/is (every? #(nil? (:data %)) snapshots)))) + (th/run-task! :storage-gc-touched {:skip-delay true}) - (t/testing "snapshots are still listed" - (let [out (th/command! {::th/type :get-file-snapshots - ::rpc/profile-id (:id profile) - :file-id (:id file)})] - (t/is (nil? (:error out))) - (t/is (some #(= "before-offload" (:label %)) (:result out))))) + ;; The main row points at a cold object, and the storage GC froze it + ;; because the row still references it. The restore must touch it before + ;; replacing the row, or the blob is never reclaimed. + (let [sobject (:storage-object-id (th/db-get :file-data {:file-id (:id file) + :id (:id file)}))] + (t/is (uuid? sobject)) + (t/is (nil? (:touched-at (th/db-get :storage-object {:id sobject} + {::db/remove-deleted false})))) + + (t/testing "live snapshot rows moved to cold storage" + (let [snapshots (th/db-query :file-data {:file-id (:id file) + :type "snapshot" + :deleted-at nil})] + (t/is (pos? (count snapshots))) + (t/is (every? #(= "storage" (:backend %)) snapshots)) + (t/is (every? #(nil? (:data %)) snapshots)))) + + (t/testing "snapshots are still listed" + (let [out (th/command! {::th/type :get-file-snapshots + ::rpc/profile-id (:id profile) + :file-id (:id file)})] + (t/is (nil? (:error out))) + (t/is (some #(= "before-offload" (:label %)) (:result out))))) - (t/testing "snapshot restores from cold storage" (let [snap-id (->> (:result (th/command! {::th/type :get-file-snapshots ::rpc/profile-id (:id profile) :file-id (:id file)})) (filter #(= "before-offload" (:label %))) first - :id) - out (th/command! {::th/type :restore-file-snapshot + :id)] + (t/is (some? snap-id)) + + (t/testing "snapshot preview does not expose the storage pointer" + (let [out (th/command! {::th/type :get-file-snapshot ::rpc/profile-id (:id profile) :file-id (:id file) :id snap-id})] - (t/is (some? snap-id)) - (t/is (nil? (:error out))) - (t/is (some? (get-in (get-file-data profile (:id file)) - [:pages-index page-id :objects shape-id]))))))) + (t/is (nil? (:error out))) + (t/is (not (contains? (:result out) :storage-object-id))))) + + (t/testing "snapshot restores from cold storage" + (let [out (th/command! {::th/type :restore-file-snapshot + ::rpc/profile-id (:id profile) + :file-id (:id file) + :id snap-id})] + (t/is (nil? (:error out))) + (t/is (some? (get-in (get-file-data profile (:id file)) + [:pages-index page-id :objects shape-id])))))) + + (t/testing "the replaced main object is touched for the storage GC" + (t/is (some? (:touched-at (th/db-get :storage-object {:id sobject} + {::db/remove-deleted false}))))) + + (t/testing "the main row is back on hot storage without the old pointer" + (let [row (th/db-get :file-data {:file-id (:id file) :id (:id file)})] + (t/is (= "db" (:backend row))) + (t/is (nil? (:storage-object-id row)))))))) (t/deftest write-after-offload-returns-to-hot-storage (let [profile (th/create-profile* 1 {:is-active true}) @@ -483,11 +548,7 @@ (let [result (th/run-task! :offload-file-data {:file-id (:id file)})] (t/is (pos? (:offloaded result)))) - (let [ref-ids (into #{} - (comp (map :metadata) - (map fdata/decode-metadata) - (keep :storage-ref-id)) - (db-rows (:id file)))] + (let [ref-ids (into #{} (keep :storage-object-id) (db-rows (:id file)))] (t/is (pos? (count ref-ids))) (t/is (every? some? (map #(sto/get-object storage %) ref-ids))) diff --git a/backend/test/backend_tests/restore_storage_objects_test.clj b/backend/test/backend_tests/restore_storage_objects_test.clj index bb2201659d..ba19a17c67 100644 --- a/backend/test/backend_tests/restore_storage_objects_test.clj +++ b/backend/test/backend_tests/restore_storage_objects_test.clj @@ -91,8 +91,8 @@ file (th/create-file* 1 {:profile-id (:id profile) :project-id (:default-project-id profile)}) data-id (uuid/next) - ;; file-data objects carry the ids of their row: the GC resolves the - ;; reference with that pair. + ;; the row points at its object through `storage_object_id`; the + ;; metadata ids are informational for diagnostics sobject (:id (put-object! "file-data" {:file-id (:id file) :id data-id}))] @@ -103,7 +103,7 @@ :type "main" :backend "fs" :deleted-at (ct/in-past {:days 1}) - :metadata (db/json {"storage-ref-id" (str sobject)})}) + :storage-object-id sobject}) (mark-object! sobject) (restore-file! (:id file)) diff --git a/backend/test/backend_tests/rpc_comment_test.clj b/backend/test/backend_tests/rpc_comment_test.clj index 3447e5aad8..9b29853c75 100644 --- a/backend/test/backend_tests/rpc_comment_test.clj +++ b/backend/test/backend_tests/rpc_comment_test.clj @@ -15,6 +15,7 @@ [app.rpc.commands.comments :as comments] [app.rpc.cond :as cond] [app.rpc.quotes :as-alias quotes] + [app.storage :as sto] [backend-tests.helpers :as th] [clojure.test :as t] [datoteka.fs :as fs] @@ -1058,3 +1059,28 @@ :team-id (:id team)})] (t/is (th/success? out)) (t/is (= 0 (count (:result out)))))))) + +(t/deftest internal-file-map-for-comments-omits-the-storage-object-id + ;; The comments module builds its file map through `comments/get-file`; the + ;; storage pointer is internal bookkeeping and must not travel in it. The + ;; RPC responses never embed the file map, so this pins the internal choke + ;; point instead. + (let [profile (th/create-profile* 1 {:is-active true}) + team (th/create-team* 1 {:profile-id (:id profile)}) + project (th/create-project* 1 {:team-id (:id team) + :profile-id (:id profile)}) + file (th/create-file* 1 {:profile-id (:id profile) + :project-id (:id project)}) + page-id (get-in file [:data :pages 0]) + sobject (sto/put-object! (::sto/storage th/*system*) + {::sto/content (sto/content "sobject") + :content-type "application/octet-stream" + :bucket "file-data" + :file-id (:id file) + :id (:id file)}) + _ (th/db-update! :file-data {:storage-object-id (:id sobject)} + {:file-id (:id file) + :id (:id file)}) + result ((deref #'comments/get-file) th/*system* (:id file) page-id)] + + (t/is (not (contains? result :storage-object-id))))) diff --git a/backend/test/backend_tests/rpc_file_snapshot_test.clj b/backend/test/backend_tests/rpc_file_snapshot_test.clj index c90d4c94ac..91178ff14f 100644 --- a/backend/test/backend_tests/rpc_file_snapshot_test.clj +++ b/backend/test/backend_tests/rpc_file_snapshot_test.clj @@ -291,10 +291,11 @@ nil {:data (:data preview)}))] (t/is (= preview-data live))))) -(t/deftest user-snapshot-of-file-needing-migration-is-materialized - ;; A file with pending migrations cannot share fragments: the - ;; migration path realizes the data, so the stored snapshot holds - ;; plain maps. This pins the shape so it cannot drift silently. +(t/deftest user-snapshot-of-file-needing-migration-shares-fragments + ;; A file with pending migrations is migrated on read before the snapshot + ;; is taken. The read keeps the pointers of the pages the migration does + ;; not change, so the snapshot shares those fragments with the live file + ;; like any other snapshot instead of storing a materialized copy. (let [profile (th/create-profile* 1) file (th/create-file* 1 {:profile-id (:id profile) :project-id (:default-project-id profile) @@ -309,7 +310,11 @@ snap-id (:id (:result out)) snap (fsnap/get-snapshot th/*system* file-id snap-id)] (t/is (uuid? snap-id)) - (t/is (empty? (fdata/get-used-pointer-ids (:data snap)))) + (let [snap-pointers (fdata/get-used-pointer-ids (:data snap)) + live-pointers (fdata/get-used-pointer-ids + (:data (bfc/get-file th/*system* file-id)))] + (t/is (seq snap-pointers)) + (t/is (every? live-pointers snap-pointers))) ;; ...while preview keeps working on the migrated content. (let [preview (th/command! {::th/type :get-file-snapshot ::rpc/profile-id (:id profile) @@ -402,29 +407,69 @@ (t/is (nil? (:error out))) (t/is (true? (:result out))))))) +(t/deftest snapshot-creation-reads-the-file-row-for-update + ;; The read inherits the caller's lock, and the migration-on-read path may + ;; re-read the same row with FOR UPDATE. A shared lock would be an upgrade, + ;; and two concurrent readers can deadlock there. The read of the command + ;; takes the exclusive lock. + (let [profile (th/create-profile* 1) + file (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile) + :is-shared false}) + [out sqls] (th/read-recording-sql + #(th/command! {::th/type :create-file-snapshot + ::rpc/profile-id (:id profile) + :file-id (:id file) + :label "pinned"}))] + (t/is (nil? (:error out))) + (t/is (uuid? (get-in out [:result :id]))) + (t/is (= [(str bfc/sql:get-file " FOR UPDATE of f")] + sqls)))) + +(t/deftest restore-backup-reads-the-file-row-for-update + ;; The backup read is the same reader as the snapshot creation, so it also + ;; takes the exclusive lock: the migration-on-read re-read cannot upgrade a + ;; shared one and deadlock with a concurrent reader. + (let [profile (th/create-profile* 1) + file (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile) + :is-shared false}) + snap-id (:id (:result (th/command! {::th/type :create-file-snapshot + ::rpc/profile-id (:id profile) + :file-id (:id file) + :label "v1"}))) + [out sqls] (th/read-recording-sql + #(th/command! {::th/type :restore-file-snapshot + ::rpc/profile-id (:id profile) + :file-id (:id file) + :id snap-id}))] + (t/is (nil? (:error out))) + (t/is (some #(= (str bfc/sql:get-file " FOR UPDATE of f") %) sqls)) + (t/is (not-any? #(str/includes? % "FOR SHARE") sqls)))) + (t/deftest snapshot-creation-serializes-with-file-gc ;; A user snapshot shares fragment rows with the live file, so its ;; creation must serialize with a concurrent :file-gc pass (which ;; takes FOR UPDATE + SKIP LOCKED on the file row): creation holds - ;; FOR SHARE, the GC skips locked files instead of racing it. + ;; FOR UPDATE, the GC skips locked files instead of racing it. (let [profile (th/create-profile* 1) file (th/create-file* 1 {:profile-id (:id profile) :project-id (:default-project-id profile) :is-shared false}) file-id (:id file)] - (t/testing "gc skips a file locked for share" + (t/testing "gc skips a file locked for update" (db/tx-run! th/*system* (fn [cfg] - ;; Hold FOR SHARE to commit, like create-file-snapshot does. - (bfc/get-file cfg file-id :lock-for-share? true) + ;; Hold FOR UPDATE to commit, like create-file-snapshot does. + (bfc/get-file cfg file-id :lock-for-update? true) ;; The GC uses SKIP LOCKED on a different connection, so it ;; must skip instead of blocking or marking. (t/is (not (th/run-task! :file-gc {:file-id file-id})))))) (t/testing "creation cannot slip between gc scan and mark" ;; Deterministic interleaving: pause the GC right after its - ;; snapshot scan, then create a snapshot. With the FOR SHARE lock + ;; snapshot scan, then create a snapshot. With the FOR UPDATE lock ;; the creation waits at the read and sees post-GC fragment ids; ;; without it, it reads pre-GC ids and pins a fragment the pass ;; is about to mark. @@ -451,7 +496,7 @@ (t/is (= true (deref scan-started 10000 ::timeout))) (let [created (future (do-create))] ;; The creation must not reach the insert while the GC - ;; pass holds the file row: with FOR SHARE it waits at the + ;; pass holds the file row: with FOR UPDATE it waits at the ;; read instead of snapshotting pre-GC fragment ids. (t/is (= ::timeout (deref reached-insert 10000 ::timeout))) (deliver release-scan true) @@ -487,7 +532,7 @@ (t/deftest restore-backup-serializes-with-file-gc ;; The backup snapshot taken by restore shares fragment rows with ;; the live file, so its creation must serialize with a concurrent - ;; :file-gc pass, like ::create-file-snapshot does: with FOR SHARE + ;; :file-gc pass, like ::create-file-snapshot does: with FOR UPDATE ;; held by the GC pass, the restore waits at the locked read instead ;; of snapshotting pre-GC fragment ids that the pass is about to ;; mark as deleted. diff --git a/backend/test/backend_tests/rpc_file_test.clj b/backend/test/backend_tests/rpc_file_test.clj index 02eca2175e..e45794a739 100644 --- a/backend/test/backend_tests/rpc_file_test.clj +++ b/backend/test/backend_tests/rpc_file_test.clj @@ -23,6 +23,7 @@ [app.features.fdata :as fdata] [app.http :as http] [app.loggers.audit :as-alias audit] + [app.redis :as rds] [app.rpc :as-alias rpc] [app.rpc.commands.files :as files] [app.storage :as sto] @@ -68,7 +69,8 @@ (let [result (:result out)] (t/is (uuid? file-id)) (t/is (= (:name data) (:name result))) - (t/is (= proj-id (:project-id result))))) + (t/is (= proj-id (:project-id result))) + (t/is (not (contains? result :storage-object-id))))) (t/testing "rename file" (let [data {::th/type :rename-file @@ -923,6 +925,68 @@ (t/is (nil? (:error out))) (t/is (false? (-> out :result :is-shared)))))) +(t/deftest unsharing-a-library-touches-the-consumer-storage-object + ;; Regression: the absorption used to build a partial file map that dropped + ;; `:storage-object-id`, so the offloaded blob of the consumer was never + ;; touched and the storage GC could not reclaim it. + (let [profile (th/create-profile* 1 {:is-active true}) + library (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile) + :is-shared true}) + consumer (th/create-file* 2 {:profile-id (:id profile) + :project-id (:default-project-id profile)})] + + (th/link-file-to-library* {:file-id (:id consumer) + :library-id (:id library)}) + + (th/run-task! :offload-file-data {:file-id (:id consumer)}) + (th/run-task! :storage-gc-touched {:skip-delay true}) + + (let [sobject (:storage-object-id (th/db-get :file-data {:file-id (:id consumer) + :id (:id consumer)}))] + (t/is (uuid? sobject)) + (t/is (nil? (:touched-at (th/db-get :storage-object {:id sobject} + {::db/remove-deleted false})))) + + (let [out (th/command! {::th/type :set-file-shared + ::rpc/profile-id (:id profile) + :id (:id library) + :is-shared false})] + (t/is (nil? (:error out))) + + (t/testing "the absorbed file rewrites its data and touches the old object" + (let [row (th/db-get :file-data {:file-id (:id consumer) :id (:id consumer)})] + (t/is (= "db" (:backend row))) + (t/is (nil? (:storage-object-id row))) + (t/is (some? (:touched-at (th/db-get :storage-object {:id sobject} + {::db/remove-deleted false})))))))))) + +(t/deftest get-file-does-not-expose-the-storage-pointer + (let [profile (th/create-profile* 1) + file (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile)})] + + (th/run-task! :offload-file-data {:file-id (:id file)}) + (t/is (uuid? (:storage-object-id (th/db-get :file-data {:file-id (:id file) + :id (:id file)})))) + + ;; A row the migration does not backfill keeps the old pointer in the + ;; metadata. The metadata schema is open, so it would travel in the + ;; response unless `file->response` strips it. + (th/db-update! :file-data + {:metadata (db/json {"storage-ref-id" (str (uuid/next))})} + {:file-id (:id file) :id (:id file)}) + + (let [out (th/command! {::th/type :get-file + ::rpc/profile-id (:id profile) + :id (:id file)})] + (t/is (nil? (:error out))) + (t/is (= (:id file) (-> out :result :id))) + (t/testing "the internal storage object id never reaches the client" + (t/is (not (contains? (:result out) :storage-object-id)))) + (t/testing "a residual metadata pointer does not reach the client" + (t/is (not (contains? (:metadata (:result out)) :storage-ref-id))))))) + (t/deftest permissions-checks-link-to-library-1 (let [profile1 (th/create-profile* 1) profile2 (th/create-profile* 2) @@ -1606,14 +1670,13 @@ (t/is (every? #(nil? (:data %)) rows)) (t/is (every? #(= "storage" (:backend %)) rows))) - (let [file (-> (th/db-get :file-data {:id (:id file) :type "main"}) - (update :metadata fdata/decode-metadata)) + (let [file (th/db-get :file-data {:id (:id file) :type "main"}) storage (sto/resolve th/*system*)] ;; (pp/pprint file) (t/is (= "storage" (:backend file))) (t/is (nil? (:data file))) - (let [sobj (sto/get-object storage (-> file :metadata :storage-ref-id))] + (let [sobj (sto/get-object storage (:storage-object-id file))] (vreset! sobject sobj) ;; (pp/pprint (meta sobj)) (t/is (= "file-data" (:bucket (meta sobj)))) @@ -3049,6 +3112,36 @@ :id slink-id})] (t/is (nil? (:error out))))))) +(t/deftest get-file-fragment-returns-only-public-fields + (let [profile (th/create-profile* 1 {:is-active true}) + file (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile)}) + fragment (first (th/db-query :file-data {:file-id (:id file) + :type "fragment"}))] + + (t/is (uuid? (:id fragment))) + + (t/testing "a fragment on hot storage resolves" + (let [out (th/command! {::th/type :get-file-fragment + ::rpc/profile-id (:id profile) + :file-id (:id file) + :fragment-id (:id fragment)})] + (t/is (nil? (:error out))) + (let [result (:result out)] + (t/is (contains? result :data)) + (t/is (= #{:id :file-id :created-at :data} + (set (keys result))))))) + + (t/testing "a fragment moved to cold storage still resolves" + (th/run-task! :offload-file-data {:file-id (:id file)}) + (let [out (th/command! {::th/type :get-file-fragment + ::rpc/profile-id (:id profile) + :file-id (:id file) + :fragment-id (:id fragment)})] + (t/is (nil? (:error out))) + (t/is (contains? (:result out) :data)) + (t/is (not (contains? (:result out) :storage-object-id))))))) + (t/deftest share-link-fragment-access-denied (let [owner (th/create-profile* 1 {:is-active true}) viewer (th/create-profile* 2 {:is-active true}) @@ -3272,3 +3365,71 @@ :team-id (:default-team-id profile)}))] (t/is (th/success? out)) (t/is (= #{(:id file2)} (into #{} (map :id) (:result out)))))) + +(t/deftest get-team-shared-files-does-not-expose-the-storage-pointer + (let [profile (th/create-profile* 1 {:is-active true}) + file (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile) + :is-shared true})] + + (th/run-task! :offload-file-data {:file-id (:id file)}) + + (let [out (th/command! {::th/type :get-team-shared-files + ::rpc/profile-id (:id profile) + :team-id (:default-team-id profile)}) + result (:result out)] + (t/is (nil? (:error out))) + (t/is (= #{(:id file)} (into #{} (map :id) result))) + (t/is (every? #(not (contains? % :storage-object-id)) result))))) + +(t/deftest save-reads-the-file-row-for-update + ;; The save persists the map it read, so the read holds the `file` row from + ;; the read to the write, with the exclusive lock: the migration-on-read + ;; re-read cannot upgrade a shared one and deadlock with another reader. + (let [profile (th/create-profile* 1) + file (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile) + :is-shared false}) + page-id (first (get-in file [:data :pages])) + [_ sqls] (th/read-recording-sql + #(update-file! :profile-id (:id profile) + :file-id (:id file) + :revn 0 + :changes [(th/add-rect-change page-id (uuid/random))]))] + (t/is (some #(= (str bfc/sql:get-file " FOR UPDATE of f") %) sqls)) + (t/is (not-any? #(str/includes? % "FOR SHARE") sqls)))) + +(t/deftest get-team-shared-files-does-not-return-the-applied-migrations + ;; The applied migrations are read bookkeeping for the write path: this + ;; summary is not where a client completes a read, so both cache paths strip + ;; the key. The recorded SQL proves which path ran: a miss reads the file, a + ;; hit reads the row only. + (let [profile (th/create-profile* 1 {:is-active true}) + file (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile) + :is-shared true}) + shared (fn [] + (th/command! {::th/type :get-team-shared-files + ::rpc/profile-id (:id profile) + :team-id (:default-team-id profile)}))] + ;; Redis is not reset between tests and the helper builds deterministic + ;; file ids: drop the entry so the first call is a real cache miss. + (rds/run! th/*system* + (fn [{:keys [::rds/conn]}] + (rds/del conn (files/file-summary-cache-key (:id file))))) + (with-redefs [cf/flags (conj cf/flags :redis-cache)] + (let [[miss sqls-miss] (th/read-recording-sql shared) + [hit sqls-hit] (th/read-recording-sql shared)] + (t/is (= [bfc/sql:get-file] sqls-miss)) + (t/is (= [bfc/sql:get-file-without-data] sqls-hit)) + + (t/testing "the cache miss does not carry the applied migrations" + (t/is (nil? (:error miss))) + (t/is (= #{(:id file)} (into #{} (map :id) (:result miss)))) + (t/is (every? #(not (contains? % :migrations)) (:result miss)))) + + (t/testing "the cache hit does not carry the applied migrations" + (t/is (nil? (:error hit))) + (t/is (= (into #{} (map :id) (:result miss)) + (into #{} (map :id) (:result hit)))) + (t/is (every? #(not (contains? % :migrations)) (:result hit)))))))) diff --git a/backend/test/backend_tests/rpc_management_test.clj b/backend/test/backend_tests/rpc_management_test.clj index eec3aaabb9..9eefb5ef28 100644 --- a/backend/test/backend_tests/rpc_management_test.clj +++ b/backend/test/backend_tests/rpc_management_test.clj @@ -177,6 +177,39 @@ (let [rows (th/db-query :file {:project-id (:id project)})] (t/is (= 3 (count rows)))))))) +(t/deftest duplicate-file-does-not-touch-the-source-storage-object + (let [profile (th/create-profile* 1 {:is-active true}) + project (th/create-project* 1 {:team-id (:default-team-id profile) + :profile-id (:id profile)}) + file1 (th/create-file* 1 {:profile-id (:id profile) + :project-id (:id project)})] + + (th/run-task! :offload-file-data {:file-id (:id file1)}) + (th/run-task! :storage-gc-touched {:skip-delay true}) + + (let [sobject (:storage-object-id (th/db-get :file-data {:file-id (:id file1) + :id (:id file1)}))] + (t/is (uuid? sobject)) + (t/is (nil? (:touched-at (th/db-get :storage-object {:id sobject} + {::db/remove-deleted false})))) + + (let [out (th/command! {::th/type :duplicate-file + ::rpc/profile-id (:id profile) + :file-id (:id file1)})] + (t/is (nil? (:error out))) + (t/is (not (contains? (:result out) :storage-object-id))) + + (t/testing "duplication does not write to the source object" + (t/is (nil? (:touched-at (th/db-get :storage-object {:id sobject} + {::db/remove-deleted false}))))) + + (let [file-id (:id (:result out)) + row (th/db-get :file-data {:file-id file-id :id file-id})] + (t/testing "the new file does not inherit the source pointer" + (t/is (= "db" (:backend row))) + (t/is (nil? (:storage-object-id row))) + (t/is (bytes? (:data row))))))))) + (t/deftest duplicate-file-with-deleted-relations (let [storage (-> (:app.storage/storage th/*system*) (configure-storage-backend)) diff --git a/backend/test/backend_tests/rpc_team_test.clj b/backend/test/backend_tests/rpc_team_test.clj index 51a1e746e1..f562fc5980 100644 --- a/backend/test/backend_tests/rpc_team_test.clj +++ b/backend/test/backend_tests/rpc_team_test.clj @@ -1371,6 +1371,10 @@ file (th/create-file* 1 {:profile-id (:id owner) :project-id (:id proj)}) + ;; Cold storage makes the file map carry the internal pointer, so the + ;; response assertion below is meaningful. + _ (th/run-task! :offload-file-data {:file-id (:id file)}) + data {::th/type :create-team-access-request ::rpc/profile-id (:id requester) :file-id (:id file)}] @@ -1383,7 +1387,8 @@ :requester-id (:id requester)})] (t/is (th/success? out)) (t/is (= 1 (:call-count @mock))) - (t/is (= 1 (count requests)))) + (t/is (= 1 (count requests))) + (t/is (not (contains? (:result out) :storage-object-id)))) ;; request again fails (th/reset-mock! mock) diff --git a/backend/test/backend_tests/rpc_viewer_test.clj b/backend/test/backend_tests/rpc_viewer_test.clj index ac09c1e67f..58e568ef02 100644 --- a/backend/test/backend_tests/rpc_viewer_test.clj +++ b/backend/test/backend_tests/rpc_viewer_test.clj @@ -910,3 +910,38 @@ (t/is (= #{comp-b} (set (keys (get-in by-id [(:id lib2) :data :components]))))) (t/is (= [] (get-in by-id [(:id lib2) :data :pages]))))))) + +(t/deftest view-only-bundle-does-not-expose-the-storage-pointer + ;; Both the file and its libraries are moved to cold storage, so their file + ;; maps carry the internal pointer unless the response strips it. + (let [owner (th/create-profile* 1 {:is-active true}) + proj-id (:default-project-id owner) + file (th/create-file* 1 {:profile-id (:id owner) + :project-id proj-id + :is-shared false}) + lib (th/create-file* 2 {:profile-id (:id owner) + :project-id proj-id + :is-shared true}) + page-id (get-in file [:data :pages 0]) + _ (th/link-file-to-library* {:file-id (:id file) + :library-id (:id lib)}) + slink (th/command! {::th/type :create-share-link + ::rpc/profile-id (:id owner) + :file-id (:id file) + :pages #{page-id} + :who-comment "team" + :who-inspect "all"}) + slink-id (get-in slink [:result :id])] + + (th/run-task! :offload-file-data {:file-id (:id file)}) + (th/run-task! :offload-file-data {:file-id (:id lib)}) + + (let [out (th/command! {::th/type :get-view-only-bundle + :share-id slink-id + :file-id (:id file)}) + result (:result out)] + (t/is (nil? (:error out))) + (t/is (seq (:libraries result))) + (t/is (not (contains? (:file result) :storage-object-id))) + (t/is (every? #(not (contains? % :storage-object-id)) + (:libraries result)))))) diff --git a/backend/test/backend_tests/storage_gc_deleted_test.clj b/backend/test/backend_tests/storage_gc_deleted_test.clj index ecd805af73..f2924d681c 100644 --- a/backend/test/backend_tests/storage_gc_deleted_test.clj +++ b/backend/test/backend_tests/storage_gc_deleted_test.clj @@ -6,6 +6,7 @@ (ns backend-tests.storage-gc-deleted-test (:require + [app.common.logging :as l] [app.common.time :as ct] [app.common.uuid :as uuid] [app.db :as db] @@ -155,3 +156,75 @@ (t/testing "the objects after the failed chunk are gone" (t/is (every? #(nil? (find-object (:id %))) recent))))) + +(defn- blocked-by-file-data! + "Puts an object referenced by a `file_data` row, so the `NO ACTION` + foreign key of `storage_object_id` refuses the physical delete." + [file-id] + (let [blocked (put-object! (ct/in-past "30m"))] + ;; `main` always carries `id = file_id`: point the real row at the object + ;; instead of inventing a second main row. + (th/db-update! :file-data + {:backend "storage" + :data nil + :storage-object-id (:id blocked)} + {:file-id file-id :id file-id}) + blocked)) + +(def ^:private blocked-chunk-hint + "chunk deletion blocked by a live reference; fix the reference and let the next run reclaim the chunk") + +(defn- blocked-chunk-log + "The first captured error record with the blocked-chunk hint, waiting for it + because logging is asynchronous." + [records] + (let [deadline (+ (System/currentTimeMillis) 30000)] + (loop [] + (let [record (first (filter (fn [record] + (and (= :error (::l/level record)) + (= blocked-chunk-hint + (get (::l/props record) :hint)))) + @records))] + (cond + record record + (> (System/currentTimeMillis) deadline) nil + :else (do (Thread/sleep 10) (recur))))))) + +(t/deftest file-data-reference-blocks-its-chunk + (let [profile (th/create-profile* 1) + file (th/create-file* 1 {:profile-id (:id profile) + :project-id (:default-project-id profile)}) + + ;; The blocked object goes first in `deleted_at` order, so it lands in + ;; the first chunk with the objects that fill it up. The last object + ;; lands in the next chunk: it proves the run kept going. + blocked (blocked-by-file-data! (:id file)) + others (mapv (fn [_] (put-object! (ct/in-past "20m"))) (range (dec chunk-size))) + last-id (:id (put-object! (ct/in-past "5m"))) + + records (atom []) + _ (add-watch l/log-record ::capture + (fn [_ _ _ record] (swap! records conj record))) + result (try + (th/run-task! :storage-gc-deleted {}) + (finally + (remove-watch l/log-record ::capture)))] + + (t/testing "the run finishes and only the object of the second chunk is deleted" + (t/is (= 1 (:deleted result)))) + + (t/testing "the object referenced by the file data row is still alive" + (t/is (some? (find-object (:id blocked))))) + + (t/testing "the other objects of the failed chunk are still alive" + (t/is (every? #(some? (find-object (:id %))) others))) + + (t/testing "the blocked chunk names the constraint that blocks it" + (let [record (blocked-chunk-log records)] + (t/is (some? record)) + (t/is (= "file_data_storage_object_id_fkey" + (get (::l/props record) :constraint))) + (t/is (= "file_data" (get (::l/props record) :table))))) + + (t/testing "the object of the next chunk is gone" + (t/is (nil? (find-object last-id)))))) diff --git a/common/src/app/common/types/file.cljc b/common/src/app/common/types/file.cljc index 444484b68f..78fa9cf3b4 100644 --- a/common/src/app/common/types/file.cljc +++ b/common/src/app/common/types/file.cljc @@ -95,7 +95,6 @@ (def schema:file-metadata [:map {:title "Metadata"} - [:storage-ref-id {:optional true} ::sm/uuid] [:generated-by {:optional true} :string] [:referer {:optional true} :string]]) diff --git a/docs/technical-guide/developer/backend.md b/docs/technical-guide/developer/backend.md index f2e7bc4d15..d1a40b50fa 100644 --- a/docs/technical-guide/developer/backend.md +++ b/docs/technical-guide/developer/backend.md @@ -95,6 +95,52 @@ Examples: **NOTE**: if table name has more than one word, we still use - as a separator. +### Large tables + +A migration runs inside a single transaction, so it cannot use `CONCURRENTLY`, +and a `CREATE INDEX` in a migration locks the table against writes until the +whole run commits. + +Write the migration as plain, idempotent SQL: `ADD COLUMN IF NOT EXISTS`, +`CREATE INDEX IF NOT EXISTS`, and `DROP CONSTRAINT IF EXISTS` followed by +`ADD CONSTRAINT`. Do not encode shape checks as `pg_catalog` introspection. +They are easy to get wrong (forgetting `NOT condeferred` accepts an +`INITIALLY DEFERRED` foreign key, which moves the integrity check to commit) and +they duplicate what the runbook can verify by hand. + +On a large table, build the column and the index out of band before deploying, +with `CREATE INDEX CONCURRENTLY` on its own (never inside a transaction block). +A partitioned table cannot take `CREATE INDEX CONCURRENTLY` on the parent: build +the index on each partition, create the parent index with +`CREATE INDEX ... ON ONLY`, and attach every partition with +`ALTER INDEX ... ATTACH PARTITION`. An interrupted concurrent build leaves the +index `INVALID`: drop it and retry. `CREATE INDEX IF NOT EXISTS` matches the +name, not the definition, so the runbook verifies `pg_index.indisvalid`, that +the index has no predicate or expression, and that every partition is attached. + +A validated foreign key is the other expensive DDL: `ADD FOREIGN KEY` runs as an +`ALTER TABLE`, so it takes `ACCESS EXCLUSIVE` on the referencing table and +`SHARE ROW EXCLUSIVE` on the referenced one, and it scans the whole referencing +table. `NOT VALID` on a partitioned table is only available since PostgreSQL 18. +A reference that a GC task resolves must be `DEFERRABLE INITIALLY IMMEDIATE`, +never `INITIALLY DEFERRED`: the immediate check fires inside the transaction +that catches the violation and retries; the deferred one fires at commit, +outside the catch. + +The runbook ends by verifying the end state (column, pending rows, index and +constraint shape) and marking the migration step as applied +(`INSERT INTO migrations (module, step) ...`), so the body does not run at +startup. The marker is the only thing that skips the migration. + +A backfill that removes a key an old reader uses (for example a JSONB pointer +that the previous GC scans) must run with the old code stopped. Otherwise the +old reader loses the reference and can delete a live object, and the rows it +writes in the meantime miss the backfill. + +`ADD COLUMN` takes a brief `ACCESS EXCLUSIVE` even when it changes nothing. A +migration accepts that lock; a deployment that cannot take it runs the runbook +and marks the step as applied. + If you need to have a global overview of the all schema of the database you can extract it using postgresql: diff --git a/operations/2.20.0-file-data-storage-object-id.md b/operations/2.20.0-file-data-storage-object-id.md new file mode 100644 index 0000000000..5355c3a909 --- /dev/null +++ b/operations/2.20.0-file-data-storage-object-id.md @@ -0,0 +1,285 @@ +# 2.20.0 — File data storage object foreign key + +Runbook for the release that moves the file data storage pointer out of +`file_data.metadata->>'storage-ref-id'` and into a column, +`file_data.storage_object_id`, with a foreign key to `storage_object(id)`. + +## When to run this + +The migration `0159-add-file-data-storage-object-id` does the whole job on a +fresh database and on small instances: it adds the column, backfills the +pointer, creates the index and the foreign key. + +A large instance builds everything online and keeps the stopped window as +short as possible: + +- the column, the index and the foreign key are built online, without stopping + the service; +- the backfill runs in the stopped window **only if** the metadata pointer + exists; when the storage backend was never enabled there is nothing to + backfill and the new binaries roll out like any other deploy; +- the step is then marked as applied in the `migrations` table, so the + migration body does not run at startup. + +The coordinated restart is required only when the instance uses the storage +backend (`PENPOT_FILE_DATA_BACKEND=storage`) or still has rows that carry the +metadata pointer. The previous code writes and reads `metadata.storage-ref-id`, +the new code reads the column, and a rolling deploy would mix both. With the +default `db` backend and no row carrying the pointer (see the pre-flight +query), the previous and the new code read the same `file_data.data` column, so +a rolling deploy is safe. + +**Do not run the backfill while the previous version is serving** when +`PENPOT_FILE_DATA_BACKEND` is configured to `storage`, or when the table still +has rows with the metadata pointer. The previous code writes the pointer only to +`metadata.storage-ref-id`, and its storage GC reads the reference from there. +Removing the key while that code runs leaves the rows it writes without a +pointer (unreadable after the deploy), and it can make the GC delete an object +that a live row still references. When the pre-flight query returns 0 there is +nothing to backfill and this risk does not apply. + +## 0. Pre-flight + +```sql +SELECT version(); -- 16.x: `NOT VALID` foreign keys on a partitioned table need 18 + +SELECT count(*) FROM pg_inherits + WHERE inhparent = 'file_data'::regclass; -- 16 + +-- Whether the storage backend was ever enabled. When this is 0 there is no +-- pointer in the metadata and step 6 is skipped. +SELECT count(*) FROM file_data + WHERE backend = 'storage' AND jsonb_exists(metadata, 'storage-ref-id'); + +-- Every online ALTER uses a short lock timeout and is retried, so a long +-- transaction cannot make it queue and block the table behind it. +SELECT pid, state, now() - xact_start AS age, left(query, 60) + FROM pg_stat_activity + WHERE xact_start IS NOT NULL + AND now() - xact_start > interval '1 minute'; +``` + +## 1. Column + +```sql +BEGIN; +SET LOCAL lock_timeout = '3s'; +ALTER TABLE file_data + ADD COLUMN storage_object_id uuid NULL; +COMMIT; +``` + +Skip this step if the column already exists. The change is catalog-only (a +nullable column without a default), so it takes `ACCESS EXCLUSIVE` for an +instant. If it fails with `lock timeout`, wait for a quieter moment and retry: +it did not block anything. + +## 2. One index per partition, concurrently + +```sql +CREATE INDEX CONCURRENTLY file_data_00__storage_object_id__idx + ON file_data_00 (storage_object_id); +-- ... up to file_data_15 +``` + +Run each statement in autocommit: `CREATE INDEX CONCURRENTLY` cannot run inside +a transaction block. An interrupted build leaves the index `INVALID`: drop it +and retry. + +```sql +SELECT c.relname, i.indisvalid + FROM pg_index AS i + JOIN pg_class AS c ON c.oid = i.indexrelid + WHERE c.relname LIKE 'file_data_%storage_object_id%' + ORDER BY c.relname; +``` + +## 3. Parent index and partition attachment + +```sql +CREATE INDEX file_data__storage_object_id__idx + ON ONLY file_data (storage_object_id); + +ALTER INDEX file_data__storage_object_id__idx + ATTACH PARTITION file_data_00__storage_object_id__idx; +-- ... up to file_data_15 +``` + +The step creates the index unconditionally: if an index with that name already +exists, drop it and rebuild it. Verify the result before continuing: + +```sql +SELECT i.indisvalid, i.indpred IS NULL AS plain, i.indexprs IS NULL AS no_expression + FROM pg_index AS i + WHERE i.indexrelid = 'file_data__storage_object_id__idx'::regclass; +-- t | t | t + +SELECT count(*) FROM pg_inherits + WHERE inhparent = 'file_data__storage_object_id__idx'::regclass; +-- 16 +``` + +## 4. Foreign key, online + +```sql +BEGIN; +SET LOCAL lock_timeout = '3s'; +ALTER TABLE file_data + ADD CONSTRAINT file_data_storage_object_id_fkey + FOREIGN KEY (storage_object_id) REFERENCES storage_object(id) + ON DELETE NO ACTION DEFERRABLE INITIALLY IMMEDIATE; +COMMIT; +``` + +The column is NULL until step 6, so the validation finds nothing to check and +the `SHARE ROW EXCLUSIVE` lock (it blocks writes, not reads) is brief. Retry on +`lock timeout`. If the constraint already exists, do not add it again: verify +its shape with the query in step 7. Drop it first only if the shape is wrong. + +Creating it here, before the backfill, keeps the full validation scan out of +the stopped window: the backfill validates each row it writes. + +`INITIALLY IMMEDIATE` is required. The storage GC catches the violation inside +the chunk transaction and retries the chunk; an `INITIALLY DEFERRED` check +fires at commit, outside that transaction, and the GC would never move past a +blocked chunk. + +## 5. Stop every instance + +Coordinated restart when the pre-flight count is not 0: stop all of them before +the next step. No rolling deploy. When the count is 0 there is nothing to +backfill: run steps 1-4 online, run the verification (step 7) and mark the step +as applied (step 8) without stopping the service. + +## 6. Backfill, only if the metadata pointer exists + +Skip this step when the pre-flight count is 0: the storage backend was never +enabled, no row carries the pointer, and the new code reads all file data from +the `data` column. + +```sql +WITH candidates AS MATERIALIZED ( + SELECT file_id, + id, + (metadata->>'storage-ref-id')::uuid AS soid + FROM file_data + WHERE backend = 'storage' + AND storage_object_id IS NULL + AND (metadata->>'storage-ref-id') ~* + '^[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}$' +) +UPDATE file_data AS fd + SET storage_object_id = c.soid, + metadata = fd.metadata - 'storage-ref-id' + FROM candidates AS c + WHERE fd.file_id = c.file_id + AND fd.id = c.id + AND EXISTS (SELECT 1 FROM storage_object AS so WHERE so.id = c.soid); +``` + +The `MATERIALIZED` CTE keeps the uuid cast away from values that did not pass +the regex: without it PostgreSQL can inline the subquery and cast a corrupt +value, aborting the statement. The foreign key already exists, so every written +row is validated here instead of in a separate full scan. + +## 7. Verify + +Do not mark the migration as applied until every check passes. + +```sql +-- Column. +SELECT data_type, is_nullable + FROM information_schema.columns + WHERE table_schema = 'public' + AND table_name = 'file_data' + AND column_name = 'storage_object_id'; +-- uuid | YES + +-- Rows that will be unreadable. Must be 0. The backfill only fills pointers it +-- can resolve, so a non-zero count means pointers that were already corrupt or +-- orphan: inspect them and accept them before continuing. With the backfill +-- skipped this must still be 0, or the storage backend was enabled after all. +-- A row pointing at a storage_object with deleted_at set is legitimate and +-- restorable, not unreadable: the backfill fills it on purpose and the +-- restore flow revives the object, so it is not counted here. +SELECT count(*) FROM file_data + WHERE backend = 'storage' AND storage_object_id IS NULL; + +-- Index: valid, plain, with every partition attached. +SELECT i.indisvalid, i.indpred IS NULL AS plain, i.indexprs IS NULL AS no_expression + FROM pg_index AS i + WHERE i.indexrelid = 'file_data__storage_object_id__idx'::regclass; +SELECT count(*) FROM pg_inherits + WHERE inhparent = 'file_data__storage_object_id__idx'::regclass; -- 16 + +-- Foreign key: deferrable, checked immediately, no action, validated. +SELECT condeferrable, condeferred, confdeltype, convalidated + FROM pg_constraint + WHERE conname = 'file_data_storage_object_id_fkey' + AND conrelid = 'file_data'::regclass; +-- t | f | a | t +``` + +## 8. Mark the migration as applied + +Only after step 7 passes: + +```sql +INSERT INTO migrations (module, step) +VALUES ('main', '0159-add-file-data-storage-object-id') +ON CONFLICT (module, step) DO NOTHING; +``` + +Check it: + +```sql +SELECT module, step, created_at FROM migrations + WHERE module = 'main' AND step = '0159-add-file-data-storage-object-id'; +``` + +The row is the only thing that skips the migration at startup. Without it, the +migration runs at the next start and repeats the backfill and the foreign key. + +## 9. Start the new instances + +Start every instance you stopped in step 5. When the pre-flight count was 0 +nothing was stopped, and the rolling deploy simply finishes. + +## Post-deploy checks + +```sql +SELECT conname, condeferrable, condeferred, confdeltype, convalidated + FROM pg_constraint + WHERE conname = 'file_data_storage_object_id_fkey'; + +SELECT count(*) FROM file_data + WHERE backend = 'storage' AND storage_object_id IS NULL; + +-- The delete check must use the index, not a sequential scan. +BEGIN; +EXPLAIN (ANALYZE, BUFFERS) + DELETE FROM storage_object WHERE id = ''; +ROLLBACK; +``` + +## Rollback + +Before step 8: + +```sql +ALTER TABLE file_data + DROP CONSTRAINT IF EXISTS file_data_storage_object_id_fkey; + +-- A partitioned index cannot be dropped with CONCURRENTLY; the plain drop +-- takes a brief ACCESS EXCLUSIVE. +DROP INDEX file_data__storage_object_id__idx; + +ALTER TABLE file_data DROP COLUMN storage_object_id; +``` + +After step 8 the backfill has already removed the metadata key that the +previous code reads, so a rollback to the previous binaries is **not +supported** when the instance uses the storage backend: the files offloaded by +the new code would be unreadable, and the recovery is a data restore, not a +binary switch. With the default `db` backend and no storage rows, the previous +binaries read the same `data` column and the rollback is a plain binary switch. From 861bb5bb371dc42146c019b41bfd0666db9f5cea Mon Sep 17 00:00:00 2001 From: Usama Sadiq Date: Sat, 14 Nov 2026 22:12:22 +0500 Subject: [PATCH 09/15] :bug: Fix database pool min-size falling back to max-size (#13052) * :bug: Fix database pool min-size falling back to max-size Since 2.16 the pool minimum defaults to the maximum (60), so an idle backend opens and holds 60 connections at startup. Restore the pre-2.16 default of 0 so the pool grows with load and idle connections expire. Closes #12622 Signed-off-by: Usama Sadiq * :books: Document database connection pool settings Describe PENPOT_DATABASE_MIN_POOL_SIZE and PENPOT_DATABASE_MAX_POOL_SIZE, their defaults, and sizing on a PostgreSQL shared with other apps. Signed-off-by: Usama Sadiq --------- Signed-off-by: Usama Sadiq --- backend/src/app/db.clj | 12 +++++------- docs/technical-guide/configuration.md | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/backend/src/app/db.clj b/backend/src/app/db.clj index d34911ff32..e3fde46fb0 100644 --- a/backend/src/app/db.clj +++ b/backend/src/app/db.clj @@ -70,6 +70,7 @@ (def defaults {::name :main + ::min-size 0 ::max-size 60 ::connection-timeout 30000 ::validation-timeout 10000 @@ -92,8 +93,7 @@ :read-only read-only :credentials (and (contains? cfg ::username) (contains? cfg ::password)) - :min-size (or (::min-size cfg) - (::max-size cfg)) + :min-size (::min-size cfg) :max-size (::max-size cfg)) (create-pool cfg)))) @@ -114,9 +114,7 @@ [{:keys [::uri] :as cfg}] ;; (app.common.pprint/pprint cfg) - (let [config (HikariConfig.) - max-size (::max-size cfg) - min-size (or (::min-size cfg) max-size)] + (let [config (HikariConfig.)] (doto config (.setJdbcUrl (str "jdbc:" uri)) (.setPoolName (d/name (::name cfg))) @@ -126,8 +124,8 @@ (.setValidationTimeout (::validation-timeout cfg)) (.setIdleTimeout (::idle-timeout cfg)) (.setMaxLifetime (::max-lifetime cfg)) - (.setMinimumIdle min-size) - (.setMaximumPoolSize max-size) + (.setMinimumIdle (::min-size cfg)) + (.setMaximumPoolSize (::max-size cfg)) (.setConnectionInitSql initsql) (.setInitializationFailTimeout -1)) diff --git a/docs/technical-guide/configuration.md b/docs/technical-guide/configuration.md index 9715bc2ce3..37ea377bec 100644 --- a/docs/technical-guide/configuration.md +++ b/docs/technical-guide/configuration.md @@ -723,6 +723,25 @@ POSTGRES_USER: penpot POSTGRES_PASSWORD: penpot ``` +#### Connection pool + +The backend keeps a pool of database connections. Its size can be tuned with: + +```bash +# Backend +PENPOT_DATABASE_MIN_POOL_SIZE: 0 +PENPOT_DATABASE_MAX_POOL_SIZE: 60 +``` + +The values above are the defaults. The pool keeps at least `MIN_POOL_SIZE` idle +connections open and never opens more than `MAX_POOL_SIZE`, growing only under load +and closing idle connections after two minutes. A minimum larger than the maximum is +capped at the maximum. + +If the PostgreSQL server is shared with other applications, make sure +`PENPOT_DATABASE_MAX_POOL_SIZE` plus the connections used by everything else stays below +the server's `max_connections` (100 by default). + ### Storage Storage refers to storing the user uploaded different objects in Penpot (assets, file data,...). From 557774e4f451b20e3fb7812168b480628fa46be1 Mon Sep 17 00:00:00 2001 From: Filip Sajdak Date: Tue, 10 Nov 2026 11:32:58 +0100 Subject: [PATCH 10/15] :bug: Call the team hook on every render of the team container (#12920) team-container* called mf/deref only after returning early for a team id that is not a uuid. When the same component rendered once without a team id and then with one, the number of hooks changed between the two renders and React threw "Rendered more hooks than during the previous render" (error #310), which sent the user to the internal error page. The reverse order throws "Rendered fewer hooks than expected". Call the hook on every render and keep the uuid check in the condition that builds the providers. Two tests mount the component in jsdom and render it with a team id that arrives and with one that leaves; both fail without this change. Closes #12728 AI-assisted-by: claude-opus-5-5 AI-assisted-by: claude-sonnet-5-5 Signed-off-by: Filip Sajdak --- frontend/src/app/main/ui.cljs | 18 ++--- frontend/test/frontend_tests/runner.cljs | 2 + .../ui/team_container_test.cljs | 80 +++++++++++++++++++ 3 files changed, 90 insertions(+), 10 deletions(-) create mode 100644 frontend/test/frontend_tests/ui/team_container_test.cljs diff --git a/frontend/src/app/main/ui.cljs b/frontend/src/app/main/ui.cljs index 5e2075082c..13c077c99c 100644 --- a/frontend/src/app/main/ui.cljs +++ b/frontend/src/app/main/ui.cljs @@ -64,16 +64,14 @@ :code :invalid-team-id :hint "provided team-id is invalid or nil"))) - (if-not (uuid? team-id) - nil - (let [{:keys [permissions] :as team} (mf/deref refs/team)] - (when (= team-id (:id team)) - [:> (mf/provider ctx/current-team-id) {:value team-id} - [:> (mf/provider ctx/permissions) {:value permissions} - [:> (mf/provider ctx/can-edit?) {:value (:can-edit permissions)} - ;; The `:key` is mandatory here because we want to reinitialize - ;; all dom tree instead of simple rerender. - [:* {:key (str team-id)} children]]]])))) + (let [{:keys [permissions] :as team} (mf/deref refs/team)] + (when (and (uuid? team-id) (= team-id (:id team))) + [:> (mf/provider ctx/current-team-id) {:value team-id} + [:> (mf/provider ctx/permissions) {:value permissions} + [:> (mf/provider ctx/can-edit?) {:value (:can-edit permissions)} + ;; The `:key` is mandatory here because we want to reinitialize + ;; all dom tree instead of simple rerender. + [:* {:key (str team-id)} children]]]]))) (mf/defc page* {::mf/props :obj diff --git a/frontend/test/frontend_tests/runner.cljs b/frontend/test/frontend_tests/runner.cljs index 1be1f38f0a..e7f9173a2f 100644 --- a/frontend/test/frontend_tests/runner.cljs +++ b/frontend/test/frontend_tests/runner.cljs @@ -171,6 +171,7 @@ [frontend-tests.ui.static-test] [frontend-tests.ui.stroke-menu-test] [frontend-tests.ui.subscription-test] + [frontend-tests.ui.team-container-test] [frontend-tests.ui.text-attrs-multiple-test] [frontend-tests.ui.viewport-top-bar-test] [frontend-tests.ui.workspace-libraries-test] @@ -374,6 +375,7 @@ 'frontend-tests.ui.sortable-auto-scroll-test 'frontend-tests.ui.static-test 'frontend-tests.ui.stroke-menu-test + 'frontend-tests.ui.team-container-test 'frontend-tests.ui.text-attrs-multiple-test 'frontend-tests.ui.viewport-top-bar-test 'frontend-tests.ui.workspace-libraries-test diff --git a/frontend/test/frontend_tests/ui/team_container_test.cljs b/frontend/test/frontend_tests/ui/team_container_test.cljs new file mode 100644 index 0000000000..b16d0b089d --- /dev/null +++ b/frontend/test/frontend_tests/ui/team_container_test.cljs @@ -0,0 +1,80 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns frontend-tests.ui.team-container-test + "`team-container*` renders the same hooks with and without a team id. + React throws when the hook count changes between two renders of the + same component, so these tests mount it and render it again; the + documents come from jsdom since the runner has no DOM." + (:require + ["jsdom" :refer [JSDOM]] + ["react-dom" :as rd] + ["react-dom/client" :as rdc] + [app.common.uuid :as uuid] + [app.main.refs :as refs] + [app.main.ui :as ui] + [cljs.test :as t :include-macros true] + [cuerdas.core :as str] + [okulary.core :as l] + [rumext.v2 :as mf])) + +(def ^:private team-id (uuid/next)) + +(defn- next-immediate + "Resolves after the tasks React's scheduler queued with `setImmediate` + so far have run." + [] + (js/Promise. (fn [resolve] (js/setImmediate resolve)))) + +(defn- ^:async render-with + "Mounts `team-container*` once and renders it again with each of + `team-ids`, in order. Resolves to the errors React reported and the + markup left in the document. + + react-dom reads the global `window` while it renders, and its scheduler + does so again on a later turn, so the jsdom one stays installed until + the root is unmounted and the scheduler has run. + + The effect is replaced by a hook that does nothing: the real one raises + for an invalid id, and what is under test is the render. It must still + take a hook slot, because React only compares the hooks of two renders + when the first one had any." + [team-ids] + (let [errors (volatile! []) + on-error (fn [error _info] (vswap! errors conj error)) + window (.-window (JSDOM. "
    ")) + container (.getElementById (.-document window) "root") + root (rdc/createRoot container #js {:onUncaughtError on-error + :onCaughtError on-error + :onRecoverableError on-error}) + child (mf/html [:span "inside"]) + previous (.-window js/globalThis)] + (set! (.-window js/globalThis) window) + (try + (with-redefs [refs/team (l/atom {:id team-id :permissions {:can-edit true}}) + mf/useEffect (fn [_f _deps] (mf/use-ref nil))] + (doseq [id team-ids] + (rd/flushSync + (fn [] + (.render root (mf/element (deref #'ui/team-container*) + #js {:teamId id :children child})))))) + (let [markup (.-innerHTML container)] + (rd/flushSync #(.unmount root)) + (await (next-immediate)) + {:errors @errors + :markup markup}) + (finally + (set! (.-window js/globalThis) previous))))) + +(t/deftest ^:async team-id-arriving-after-the-first-render-keeps-the-hooks + (let [{:keys [errors markup]} (await (render-with [nil team-id]))] + (t/is (empty? (map #(.-message %) errors))) + (t/is (str/includes? markup "inside")))) + +(t/deftest ^:async team-id-leaving-after-the-first-render-keeps-the-hooks + (let [{:keys [errors markup]} (await (render-with [team-id nil]))] + (t/is (empty? (map #(.-message %) errors))) + (t/is (= "" markup)))) From fa6af23d752dc861760f9efc06b5cf9e3dd13abd Mon Sep 17 00:00:00 2001 From: Eva Marco Date: Mon, 16 Nov 2026 09:02:31 +0100 Subject: [PATCH 11/15] :bug: Disconnect a library that fails to load instead of breaking the file (#13048) Opening a file connected to a broken shared library showed the "Internal Error" page: every connected library was loaded in a single stream, so one failing get-file broke the whole workspace. Catch the error per library. When the backend fails to load it (a server error, never a transport or transient error), a directly connected library is disconnected and a toast tells the user; an indirect one is only dropped from the state. Closes #12429 AI-assisted-by: claude-opus-5-5 --- frontend/src/app/main/data/workspace.cljs | 13 ++-- .../app/main/data/workspace/libraries.cljs | 28 +++++++++ .../data/workspace_libraries_load_test.cljs | 60 +++++++++++++++++++ frontend/test/frontend_tests/runner.cljs | 2 + frontend/translations/en.po | 4 ++ frontend/translations/es.po | 4 ++ 6 files changed, 107 insertions(+), 4 deletions(-) create mode 100644 frontend/test/frontend_tests/data/workspace_libraries_load_test.cljs diff --git a/frontend/src/app/main/data/workspace.cljs b/frontend/src/app/main/data/workspace.cljs index cc9c9c7954..f5f2ce201d 100644 --- a/frontend/src/app/main/data/workspace.cljs +++ b/frontend/src/app/main/data/workspace.cljs @@ -190,15 +190,20 @@ (rx/merge (->> (rx/from libraries) (rx/merge-map - (fn [{:keys [id synced-at is-indirect]}] + (fn [{:keys [id synced-at is-indirect] :as library}] (->> (rp/cmd! :get-file {:id id :features features}) (rx/map (fn [file] (-> file (assoc :synced-at synced-at) (assoc :library-of file-id) - (assoc :is-indirect is-indirect))))))) - (rx/mapcat resolve-file) - (rx/map library-resolved)) + (assoc :is-indirect is-indirect)))) + (rx/mapcat resolve-file) + (rx/map library-resolved) + ;; A broken library must not break the whole file + (rx/catch (fn [cause] + (if (dwl/broken-library-error? cause) + (rx/of (dwl/library-load-failed file-id library cause)) + (rx/throw cause)))))))) (->> (rx/from libraries) (rx/map :id) (rx/mapcat (fn [file-id] diff --git a/frontend/src/app/main/data/workspace/libraries.cljs b/frontend/src/app/main/data/workspace/libraries.cljs index 8d55ad89cc..7964e2d1c8 100644 --- a/frontend/src/app/main/data/workspace/libraries.cljs +++ b/frontend/src/app/main/data/workspace/libraries.cljs @@ -1933,3 +1933,31 @@ (rx/mapcat (fn [_] (rp/cmd! :get-file-libraries {:file-id file-id}))) (rx/map (partial cleanup-unlinked-libraries file-id)))))) + +(defn broken-library-error? + "True when `cause` comes from the backend failing to load a library + file. Transport and transient errors (network, gateway, offline, + rate limit) do not count, so they never disconnect a library." + [cause] + (= :server-error (:type (ex-data cause)))) + +(defn library-load-failed + "Handles a library of `file-id` that fails to load, so a single + broken library does not break the whole file: a directly connected + library is disconnected and the user is told; an indirect one is only + dropped from the state." + [file-id {:keys [id name is-indirect]} cause] + (ptk/reify ::library-load-failed + ptk/WatchEvent + (watch [_ _ _] + (log/error :hint "unable to load library" + :file-id (str file-id) + :library-id (str id) + :cause cause) + (if is-indirect + (rx/of (ptk/reify ::drop-broken-library + ptk/UpdateEvent + (update [_ state] + (update state :files dissoc id)))) + (rx/of (unlink-file-from-library file-id id) + (ntf/warn (tr "workspace.libraries.broken-library-disconnected" name))))))) diff --git a/frontend/test/frontend_tests/data/workspace_libraries_load_test.cljs b/frontend/test/frontend_tests/data/workspace_libraries_load_test.cljs new file mode 100644 index 0000000000..11f140f151 --- /dev/null +++ b/frontend/test/frontend_tests/data/workspace_libraries_load_test.cljs @@ -0,0 +1,60 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns frontend-tests.data.workspace-libraries-load-test + "Tests for a library of the open file that fails to load." + (:require + [app.common.uuid :as uuid] + [app.main.data.workspace.libraries :as dwl] + [beicon.v2.core :as rx] + [cljs.test :as t :include-macros true] + [potok.v2.core :as ptk])) + +(defn- error-of-type + [type] + (ex-info "http error" {:type type})) + +(t/deftest broken-library-error-only-for-backend-errors + ;; the backend failed to load the library file + (t/is (true? (dwl/broken-library-error? (error-of-type :server-error)))) + ;; transient or transport errors never disconnect a library + (doseq [type [:network :offline :bad-gateway :service-unavailable + :gateway-error :rate-limit nil]] + (t/is (false? (dwl/broken-library-error? (error-of-type type))) + (str "type " type))) + (t/is (false? (dwl/broken-library-error? (js/Error. "boom"))))) + +(defn- watch-events + "Synchronously collects the events the watch of `event` emits." + [event state] + (let [events (atom [])] + (->> (ptk/watch event state (rx/empty)) + (rx/subs! #(swap! events conj %))) + @events)) + +(t/deftest library-load-failed-disconnects-a-direct-library + (let [file-id (uuid/next) + library-id (uuid/next) + library {:id library-id :name "Broken" :is-indirect false} + events (watch-events (dwl/library-load-failed file-id library (error-of-type :server-error)) + {})] + ;; the library is unlinked from the file and the user is told + (t/is (= [::dwl/detach-library :app.main.data.notifications/show] + (mapv ptk/type events))))) + +(t/deftest library-load-failed-drops-an-indirect-library + (let [file-id (uuid/next) + library-id (uuid/next) + other-id (uuid/next) + library {:id library-id :name "Broken" :is-indirect true} + state {:files {library-id {:id library-id} + other-id {:id other-id}}} + events (watch-events (dwl/library-load-failed file-id library (error-of-type :server-error)) + state)] + ;; an indirect library is not linked to the file: only drop it from the state + (t/is (= 1 (count events))) + (t/is (= {other-id {:id other-id}} + (:files (ptk/update (first events) state)))))) diff --git a/frontend/test/frontend_tests/runner.cljs b/frontend/test/frontend_tests/runner.cljs index e7f9173a2f..465a0dc7f8 100644 --- a/frontend/test/frontend_tests/runner.cljs +++ b/frontend/test/frontend_tests/runner.cljs @@ -40,6 +40,7 @@ [frontend-tests.data.workspace-context-menu-test] [frontend-tests.data.workspace-focus-test] [frontend-tests.data.workspace-interactions-test] + [frontend-tests.data.workspace-libraries-load-test] [frontend-tests.data.workspace-libraries-sync-test] [frontend-tests.data.workspace-mcp-test] [frontend-tests.data.workspace-media-test] @@ -244,6 +245,7 @@ 'frontend-tests.data.workspace-context-menu-test 'frontend-tests.data.workspace-focus-test 'frontend-tests.data.workspace-interactions-test + 'frontend-tests.data.workspace-libraries-load-test 'frontend-tests.data.workspace-libraries-sync-test 'frontend-tests.data.workspace-mcp-test 'frontend-tests.data.workspace-media-test diff --git a/frontend/translations/en.po b/frontend/translations/en.po index 5f4fc3b975..51557f48a5 100644 --- a/frontend/translations/en.po +++ b/frontend/translations/en.po @@ -7101,6 +7101,10 @@ msgstr "Add a shared library" msgid "workspace.libraries.assets-in-this-file" msgstr "ASSETS IN THIS FILE" +#: src/app/main/data/workspace/libraries.cljs:1963 +msgid "workspace.libraries.broken-library-disconnected" +msgstr "The library \"%s\" couldn't be loaded and has been disconnected from this file. Components and colors already used in the file aren't affected but will be orphan until the library is fixed and connected again." + #: src/app/main/ui/workspace/libraries.cljs:178, src/app/main/ui/workspace/libraries.cljs:207, src/app/main/ui/workspace/libraries.cljs:251 msgid "workspace.libraries.colors" msgid_plural "workspace.libraries.colors" diff --git a/frontend/translations/es.po b/frontend/translations/es.po index 068dee9cf8..85357435e3 100644 --- a/frontend/translations/es.po +++ b/frontend/translations/es.po @@ -7173,6 +7173,10 @@ msgstr "Añadir una biblioteca compartida" msgid "workspace.libraries.assets-in-this-file" msgstr "RECURSOS EN ESTE ARCHIVO" +#: src/app/main/data/workspace/libraries.cljs:1963 +msgid "workspace.libraries.broken-library-disconnected" +msgstr "No se ha podido cargar la biblioteca \"%s\" y se ha desconectado de este archivo. Los componentes y colores que ya usa el archivo no se ven afectados, pero quedarán huérfanos hasta que la biblioteca se arregle y se vuelva a conectar." + #: src/app/main/ui/workspace/libraries.cljs:178, src/app/main/ui/workspace/libraries.cljs:207, src/app/main/ui/workspace/libraries.cljs:251 msgid "workspace.libraries.colors" msgid_plural "workspace.libraries.colors" From 77c87d16f1a9f2bb268059337943cc31f9d852fa Mon Sep 17 00:00:00 2001 From: Pablo Alba Date: Mon, 16 Nov 2026 09:30:29 +0100 Subject: [PATCH 12/15] :bug: Keep variant property order past eight properties (#13014) --- common/src/app/common/files/variant.cljc | 2 +- common/src/app/common/types/variant.cljc | 12 +++++- .../common_tests/logic/variants_test.cljc | 14 +++++++ common/test/common_tests/variant_test.cljc | 6 +++ .../src/app/main/data/workspace/variants.cljs | 10 ++--- .../data/workspace_variants_test.cljs | 38 +++++++++++++++++++ 6 files changed, 73 insertions(+), 9 deletions(-) diff --git a/common/src/app/common/files/variant.cljc b/common/src/app/common/files/variant.cljc index 802c90fd1c..80dde18d72 100644 --- a/common/src/app/common/files/variant.cljc +++ b/common/src/app/common/files/variant.cljc @@ -59,7 +59,7 @@ [data objects variant-id] (->> (find-variant-components data objects variant-id) (mapcat :variant-properties) - (group-by :name) + (ctv/group-properties-by-name) (mapv (fn [[k v]] (let [mdata (reduce merge {} (map meta v))] (with-meta {:name k diff --git a/common/src/app/common/types/variant.cljc b/common/src/app/common/types/variant.cljc index f91075a170..0469c3c674 100644 --- a/common/src/app/common/types/variant.cljc +++ b/common/src/app/common/types/variant.cljc @@ -315,6 +315,16 @@ props2) :props))) +(defn group-properties-by-name + "Group properties by :name, preserving first-seen name order. + clojure.core/group-by loses order once maps grow past ArrayMap's + 8-key threshold." + [props] + (reduce (fn [m p] + (update m (:name p) (fnil conj []) p)) + (d/ordered-map) + props)) + (defn compare-properties "Compares vectors of properties keeping the value if it is the same for all or setting a custom value where their values do not coincide" @@ -322,7 +332,7 @@ (compare-properties props-list nil)) ([props-list distinct-mark] - (let [grouped (group-by :name (apply concat props-list)) + (let [grouped (group-properties-by-name (apply concat props-list)) check-values (fn [values] (let [vals (map :value values)] (if (apply = vals) diff --git a/common/test/common_tests/logic/variants_test.cljc b/common/test/common_tests/logic/variants_test.cljc index 8162cf5598..a2ba965283 100644 --- a/common/test/common_tests/logic/variants_test.cljc +++ b/common/test/common_tests/logic/variants_test.cljc @@ -7,6 +7,7 @@ (ns common-tests.logic.variants-test (:require [app.common.files.changes-builder :as pcb] + [app.common.files.variant :as cfv] [app.common.geom.point :as gpt] [app.common.logic.libraries :as cll] [app.common.logic.shapes :as cls] @@ -722,3 +723,16 @@ (cpn/merge-path-item (:path comp2') (:name comp2')))) (t/is (some? (:variant-id comp1'))) (t/is (some? (:variant-id comp2'))))) + +(t/deftest test-extract-properties-values-preserves-order-past-eight + (let [names (mapv #(str "Property " %) (range 1 10)) + props1 (mapv (fn [n] {:name n :value (str n "-a")}) names) + props2 (mapv (fn [n] {:name n :value (str n "-b")}) names) + file (-> (thf/sample-file :file1) + (thv/add-variant :v01 :c01 :m01 :c02 :m02) + (thc/update-component :c01 {:variant-properties props1}) + (thc/update-component :c02 {:variant-properties props2})) + page (thf/current-page file) + v-id (:id (ths/get-shape file :v01)) + result (cfv/extract-properties-values (:data file) (:objects page) v-id)] + (t/is (= names (mapv :name result))))) diff --git a/common/test/common_tests/variant_test.cljc b/common/test/common_tests/variant_test.cljc index 0dea2aa947..bcccea9f1c 100644 --- a/common/test/common_tests/variant_test.cljc +++ b/common/test/common_tests/variant_test.cljc @@ -119,6 +119,12 @@ (t/is (= (ctv/compare-properties [props-1 props-2 props-3] "&") [{:name "border" :value "&"} {:name "color" :value "&"}]))))) +(t/deftest compare-properties-preserves-order-past-eight + (let [names (mapv #(str "Property " %) (range 1 10)) + props1 (mapv (fn [n] {:name n :value "a"}) names) + props2 (mapv (fn [n] {:name n :value "b"}) names)] + (t/is (= names (mapv :name (ctv/compare-properties [props1 props2])))))) + (t/deftest check-belong-same-variant (let [components-1 [{:variant-id "a variant"} {:variant-id "a variant"}] diff --git a/frontend/src/app/main/data/workspace/variants.cljs b/frontend/src/app/main/data/workspace/variants.cljs index 49eb435f34..8c7cdbe34c 100644 --- a/frontend/src/app/main/data/workspace/variants.cljs +++ b/frontend/src/app/main/data/workspace/variants.cljs @@ -235,14 +235,10 @@ (let [data (dsh/lookup-file-data state)] (with-main-page data {:variant-id variant-id} (fn [page-id objects] - (let [variant-components (cfv/find-variant-components data objects variant-id) - properties-empty-pos (->> variant-components - (mapcat :variant-properties) - (group-by :name) + (let [properties-empty-pos (->> (cfv/extract-properties-values data objects variant-id) (map-indexed - (fn [i [_ v]] - [i (->> v - (map :value) + (fn [i {:keys [value]}] + [i (->> value (remove empty?) empty?)])) (reverse)) diff --git a/frontend/test/frontend_tests/data/workspace_variants_test.cljs b/frontend/test/frontend_tests/data/workspace_variants_test.cljs index d596add6c2..b293ba2560 100644 --- a/frontend/test/frontend_tests/data/workspace_variants_test.cljs +++ b/frontend/test/frontend_tests/data/workspace_variants_test.cljs @@ -35,6 +35,26 @@ (thv/add-variant-two-properties :v01 :c01 :m01 :c02 :m02) (cthf/add-sample-page :page2))) +(defn- nine-property-file-with-empty-middle + "Nine properties; Property 5 is empty on every variant." + [] + (let [names (mapv #(str "Property " %) (range 1 10)) + props1 (mapv (fn [n i] + {:name n :value (if (= i 4) "" (str "a" (inc i)))}) + names + (range)) + props2 (mapv (fn [n i] + {:name n :value (if (= i 4) "" (str "b" (inc i)))}) + names + (range))] + (-> (cthf/sample-file :file1 :page-label :page1) + (thv/add-variant :v01 :c01 :m01 :c02 :m02) + (cthc/update-component :c01 {:variant-properties props1}) + (cthc/update-component :c02 {:variant-properties props2}) + (cths/update-shape :m01 :variant-name (ctv/properties-to-name props1)) + (cths/update-shape :m02 :variant-name (ctv/properties-to-name props2)) + (cthf/add-sample-page :page2)))) + (defn- run [file events done assert-fn] (ths/run-store @@ -124,6 +144,24 @@ (t/is (= "crítica" (-> props second :value))) (t/is (= (ctv/properties-to-name props) (variant-name file :m01))))))))) +(t/deftest remove-empty-properties-preserves-order-past-eight + (t/async + done + (let [file (nine-property-file-with-empty-middle) + variant-id (cthi/id :v01) + expected ["Property 1" "Property 2" "Property 3" "Property 4" + "Property 6" "Property 7" "Property 8" "Property 9"]] + (run file + [(dwv/remove-empty-properties variant-id)] + done + (fn [state] + (let [file (ths/get-file-from-state state) + names1 (mapv :name (:variant-properties (cthc/get-component file :c01))) + names2 (mapv :name (:variant-properties (cthc/get-component file :c02)))] + (t/is (= expected names1)) + (t/is (= expected names2)) + (t/is (not-any? #(= "Property 5" %) names1)))))))) + (t/deftest update-property-name-from-another-page (t/async done From 635f7fc9146663f291ee783ed4448b22c0bc01b8 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Mon, 16 Nov 2026 09:54:23 +0100 Subject: [PATCH 13/15] :sparkles: Disable the admin-console by default in the devenv frontend (#13061) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * :sparkles: Disable the admin-console by default in the devenv frontend The devenv does not run the admin-console service, but the frontend showed its UI by default. The devenv containers now build the frontend with PENPOT_DEVENV=true, which prepends disable-admin-console ahead of the developer's config.js flags. Flags parse in order, so enable-admin-console in config.js re-enables it; builds outside devenv are unchanged. Closes #13060 AI-assisted-by: inkling-2-small-free * :books: Require English in all code-facing text Add a hard rule to AGENTS.md: everything in code must be in English — identifiers, comments, docstrings, test descriptions, log hints — with tests not excepted. The only allowed place for another language is product copy rendered through the i18n system. Also restated in mem:clojure/design-rules and mem:testing. AI-assisted-by: inkling-2-small-free --- .serena/memories/clojure/design-rules.md | 1 + .serena/memories/devenv/core.md | 1 + .serena/memories/testing.md | 2 ++ AGENTS.md | 6 +++++ docker/devenv/docker-compose.main.yml | 6 +++++ docs/technical-guide/developer/devenv.md | 8 ++++++ frontend/shadow-cljs.edn | 3 ++- frontend/src/app/config.cljs | 18 ++++++++++++- frontend/test/frontend_tests/config_test.cljs | 27 +++++++++++++++++++ frontend/test/frontend_tests/runner.cljs | 2 ++ 10 files changed, 72 insertions(+), 2 deletions(-) create mode 100644 frontend/test/frontend_tests/config_test.cljs diff --git a/.serena/memories/clojure/design-rules.md b/.serena/memories/clojure/design-rules.md index 2b439ee038..e39facedc3 100644 --- a/.serena/memories/clojure/design-rules.md +++ b/.serena/memories/clojure/design-rules.md @@ -8,6 +8,7 @@ How to shape a function in this codebase. Each rule came from a review round; do - **Functions that obtain data start with `get-` or `fetch-`.** A reader must see at a glance that the function reads (`get-file`, `get-snapshot`, `get-file-json`); a bare noun hides it (`file-json` says nothing about reading). New helpers that fetch by id follow this, never a `current-` prefix: the id parameter already says which one it is. - **`?` marks a predicate, not a data prop.** A var or function that answers a question takes `?` (`export-error?`, `(defn editable? …)`); a map key that names a property does not (`:export-error`, never `:export-error?`), even when its value is a boolean. Existing `?` keys are legacy to remove gradually; do not add a new one. - **Do not shape production code for tests.** No return values added just to assert them; assert on the observable effect. When preconditions move, move the tests with them. +- **All code-facing text MUST be in English** — identifiers, comments, docstrings, test descriptions. No exceptions; see the hard rule in `AGENTS.md`. Reference implementation: the http metrics samplers in `app.http` (`sample-worker-metrics` / `sample-connector-metrics` are guard-free; `sample-http-metrics` guards at the boundary). diff --git a/.serena/memories/devenv/core.md b/.serena/memories/devenv/core.md index 3507a5d6e0..a6645ed04f 100644 --- a/.serena/memories/devenv/core.md +++ b/.serena/memories/devenv/core.md @@ -15,6 +15,7 @@ Compose-based dev environment under `docker/devenv/`, driven by `manage.sh`. Par - For ws1+, `instance-env-overrides` computes the per-instance overrides (container/volume names, host ports offset `10000·N`, `PENPOT_PUBLIC_URI`, `PENPOT_REDIS_URI`, `PENPOT_TEST_DATABASE_URI`, `PENPOT_TEST_REDIS_URI`) and `instance-compose` injects them as env vars at compose time — never written to disk, recomputed each call so they can't drift. ws0 uses `defaults.env` as-is. - Per-instance test databases are created by `ensure-instance-test-database` (called from `start-instance`): `CREATE DATABASE` guarded by a `pg_database` existence check, so it is idempotent. Needed because `postgresql_init.sql` only runs on first volume creation. The backend applies migrations itself on first use, so an empty database is enough. - `backend/scripts/_env`: backend-internal only — secret keys, `PENPOT_FLAGS` (with `enable-backend-worker` gated on `PENPOT_BACKEND_WORKER`), `JAVA_OPTS`, `setup_s3_bucket()`. Never duplicates `defaults.env`. +- Frontend admin-console is off by default in devenv: `docker-compose.main.yml` sets `PENPOT_DEVENV=true`, which makes `app.config` (`goog-define devenv-mode`) prepend `disable-admin-console` ahead of the user `config.js` flags; flags parse in order and user flags win, so `enable-admin-console` in `frontend/resources/public/js/config.js` re-enables it. Outside devenv (prod builds) the default is unset and the admin-console stays on, matching `flags/default`. - Compose files use pure `${VAR}` substitution; missing var = compose fails. ## Invariants diff --git a/.serena/memories/testing.md b/.serena/memories/testing.md index ce301ff5bb..75b775d0ec 100644 --- a/.serena/memories/testing.md +++ b/.serena/memories/testing.md @@ -73,6 +73,8 @@ a test that reproduces it: - **One Assertion Per Concept** — each test verifies one behavior; split compound assertions - **Descriptive Test Names** — names read like specifications +- **English Only** — test names, `testing` strings, comments, and every + code-facing text MUST be in English; see the hard rule in `AGENTS.md`. ## Prefer Real Implementations Over Mocks diff --git a/AGENTS.md b/AGENTS.md index 4e4bf1fec1..261d099e68 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -28,6 +28,12 @@ - Before `gh pr create` / `gh pr edit` → `mem:workflow/creating-prs` (title format, body structure, AI note) Don't infer format from the title of a previous commit/issue/PR — the memory is the source of truth. +- **Everything in code MUST be in English.** Identifiers (functions, + variables, types, namespaces), code comments, docstrings, test + descriptions, log hints — always English, regardless of the language the + author works in. Tests are not an exception. The only place another + language may appear in the codebase is product copy rendered through the + i18n system. ## CRITICAL: Read module memories BEFORE writing any code diff --git a/docker/devenv/docker-compose.main.yml b/docker/devenv/docker-compose.main.yml index c55330a80a..6836861354 100644 --- a/docker/devenv/docker-compose.main.yml +++ b/docker/devenv/docker-compose.main.yml @@ -101,5 +101,11 @@ services: - SERENA_UPDATE_VERSION=2.1.0 - SHADOW_SERVER_URL=${SHADOW_SERVER_URL} + # Frontend dev devenv switch: makes the frontend compile with + # PENPOT_DEVENV=true so the admin-console is disabled by default + # (app.config/devenv-mode). `enable-admin-console` in config.js + # re-enables it. + - PENPOT_DEVENV=true + networks: - default diff --git a/docs/technical-guide/developer/devenv.md b/docs/technical-guide/developer/devenv.md index 697d119b1c..20cd2d8a3c 100644 --- a/docs/technical-guide/developer/devenv.md +++ b/docs/technical-guide/developer/devenv.md @@ -508,6 +508,14 @@ Some examples of commonly used flags: The full list of available flags can be found in `common/src/app/common/flags.cljc`. +One flag is off by default in the devenv: the frontend compiles with +`PENPOT_DEVENV=true` in the development containers, which hides all +admin-console UI (links, routes, team menu entries) whether or not you created +a `config.js`. To test the admin-console against a local Nitrate instance, +reactivate it from your `config.js`: `var penpotFlags = "enable-admin-console";`. +Your flags are parsed after the injected default, so your entry wins. Production +builds are unaffected: `PENPOT_DEVENV` only exists in the devenv containers. + After creating or modifying this file, **reload the browser** (no need to restart anything). ### Backend flags via PENPOT_FLAGS diff --git a/frontend/shadow-cljs.edn b/frontend/shadow-cljs.edn index 9956823eeb..53ffb91aab 100644 --- a/frontend/shadow-cljs.edn +++ b/frontend/shadow-cljs.edn @@ -70,7 +70,8 @@ :compiler-options {:output-feature-set :es2020 :output-wrapper false - :warnings {:fn-deprecated false}} + :warnings {:fn-deprecated false} + :closure-defines {app.config/devenv-mode #shadow/env ["PENPOT_DEVENV" :as :bool :default false]}} :release {:closure-defines {goog.DEBUG false diff --git a/frontend/src/app/config.cljs b/frontend/src/app/config.cljs index 6bdfa17381..370159ca82 100644 --- a/frontend/src/app/config.cljs +++ b/frontend/src/app/config.cljs @@ -73,10 +73,26 @@ :browser :webworker)) +;; --- Compile-time devenv switch +;; +;; Set to true only by the devenv containers (PENPOT_DEVENV=true in +;; docker-compose.main.yml); production builds leave the default. When true, +;; the frontend disables the admin-console by default; `enable-admin-console` +;; in the user's `config.js` re-enables it (user flags parse after the +;; injected default and the last flag wins). +(goog-define devenv-mode false) + +(defn- apply-devenv-defaults + "Prepends the devenv default `disable-admin-console` before the user flags. + User flags are parsed after it, so `enable-admin-console` wins." + [devenv? flags] + (if devenv? (concat [:disable-admin-console] flags) flags)) + (defn- parse-flags [global] (let [flags (obj/get global "penpotFlags" "") - flags (sequence (map keyword) (str/words flags))] + flags (sequence (map keyword) (str/words flags)) + flags (apply-devenv-defaults devenv-mode flags)] (flags/parse flags/default flags))) (defn- parse-version diff --git a/frontend/test/frontend_tests/config_test.cljs b/frontend/test/frontend_tests/config_test.cljs new file mode 100644 index 0000000000..25c9b7d465 --- /dev/null +++ b/frontend/test/frontend_tests/config_test.cljs @@ -0,0 +1,27 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns frontend-tests.config-test + (:require + [app.common.flags :as flags] + [app.config :as cf] + [cljs.test :as t :include-macros true])) + +(def ^:private apply-devenv-defaults @#'cf/apply-devenv-defaults) + +(t/deftest devenv-default-disables-admin-console + (t/testing "in devenv, without user flags, the admin-console is disabled by default" + (t/is (not (contains? (flags/parse flags/default (apply-devenv-defaults true [])) + :admin-console)))) + (t/testing "in devenv, the user can re-enable it with enable-admin-console" + (t/is (contains? (flags/parse flags/default (apply-devenv-defaults true [:enable-admin-console])) + :admin-console))) + (t/testing "in devenv, the user's disable-admin-console is idempotent" + (t/is (not (contains? (flags/parse flags/default (apply-devenv-defaults true [:disable-admin-console])) + :admin-console)))) + (t/testing "outside devenv, no default is injected and the flags defaults apply" + (t/is (contains? (flags/parse flags/default (apply-devenv-defaults false [])) + :admin-console)))) diff --git a/frontend/test/frontend_tests/runner.cljs b/frontend/test/frontend_tests/runner.cljs index 465a0dc7f8..a0b6bce8a7 100644 --- a/frontend/test/frontend_tests/runner.cljs +++ b/frontend/test/frontend_tests/runner.cljs @@ -8,6 +8,7 @@ [frontend-tests.browser-history-test] [frontend-tests.code-gen-style-test] [frontend-tests.composable-tests.comp.sync-test] + [frontend-tests.config-test] [frontend-tests.copy-as-svg-test] [frontend-tests.data.auth-test] [frontend-tests.data.comments-batches-test] @@ -210,6 +211,7 @@ ['frontend-tests.basic-shapes-test 'frontend-tests.code-gen-style-test 'frontend-tests.composable-tests.comp.sync-test + 'frontend-tests.config-test 'frontend-tests.copy-as-svg-test 'frontend-tests.data.auth-test 'frontend-tests.data.comments-batches-test From 328e41e03789e5de4b2794519abc30728d853f9e Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Mon, 16 Nov 2026 09:17:27 +0000 Subject: [PATCH 14/15] :recycle: Stop excluding release blocker from the changelog flow The `release blocker` label marks work that blocks the release; it is not a signal about whether users will notice a change. Drop it from the changelog exclusions and keep only `no changelog`. Closed bugs carrying `release blocker` are analyzed by `dev-only-bugs` like any other bug: a bug found while testing an unreleased feature never reached production, so the tool is the right filter for them. Closed non-bug issues with the label are surfaced by a manual review step in the flow: `gh.py issues --label "release blocker"` lists them and the user decides between `no changelog` and keeping the entry. Update the memories, the update-changelog skill and the tests to match. AI-assisted-by: inkling-2-small-free --- .agents/skills/update-changelog/SKILL.md | 3 ++- .serena/memories/scripts/changelog.md | 4 ++-- .serena/memories/scripts/gh.md | 2 +- .serena/memories/workflow/updating-changelog.md | 9 +++++---- scripts/changelog.py | 7 ++++--- scripts/gh.py | 4 ++-- scripts/test_changelog.py | 14 +++++++++++--- 7 files changed, 27 insertions(+), 16 deletions(-) diff --git a/.agents/skills/update-changelog/SKILL.md b/.agents/skills/update-changelog/SKILL.md index 87c493e7c4..0c5ca79f23 100644 --- a/.agents/skills/update-changelog/SKILL.md +++ b/.agents/skills/update-changelog/SKILL.md @@ -37,7 +37,8 @@ memory; it does not repeat it. ## Commands ```bash -python3 scripts/gh.py issues "" --exclude "release blocker,no changelog" +python3 scripts/gh.py issues "" --exclude "no changelog" +python3 scripts/gh.py issues "" --label "release blocker" --exclude "no changelog" # closed non-bug residue to review by hand python3 scripts/gh.py advisories --patched-version "" python3 scripts/changelog.py dev-only-bugs "" # list bugs that never reached production python3 scripts/changelog.py dev-only-bugs "" --apply # label the sure ones `no changelog` diff --git a/.serena/memories/scripts/changelog.md b/.serena/memories/scripts/changelog.md index 6785692d82..b6cf2115a8 100644 --- a/.serena/memories/scripts/changelog.md +++ b/.serena/memories/scripts/changelog.md @@ -8,7 +8,7 @@ - `boom [--changes CHANGES.md --write]` — generate `### :boom: Breaking changes & Deprecations` from the closed, non-excluded milestone issues labelled `breaking change`, sorted by issue number. Each issue becomes a `#### ` block: content = body of the issue's last `## Breaking change note` comment (whole markdown, headings demoted to `#####` outside code fences; fallback = heading plus refs with a stderr warning), refs = issue link plus merged closing PRs on their own line, with `(by @author)` for `community contribution`. `--write` replaces only that subsection of the version (inserts it after the version header when missing, keeps the heading when there are no entries; idempotent). Print-only by default. Uses `gh.py comments`. - `security [--changes CHANGES.md --write]` — generate `### :lock: Security fixes` from the advisories whose `patched_versions` include the version (via `gh.py advisories --patched-version`), sorted by GHSA id. Entry: `- Fix () (PR: [#N](...))`; description = `Fix` + advisory summary, first word decapitalized unless it starts with an acronym; PRs = merged milestone PRs labelled `pr pt` whose body contains the GHSA id, comma-separated, and no PR is legal. Warns on stderr when a `pr pt` PR references an advisory with no matching `patched_versions`. `--write` replaces only that subsection, placed after `:rocket:` (or `:boom:`, or the version header when neither exists); empty entries remove the subsection; idempotent. Print-only by default. - `cross-ref [--changes CHANGES.md]` — fetch milestone PRs with `--state all` once, report merged PRs missing from the version section plus CLOSED (unmerged) warnings. PR links inside `:lock:` entries count as referenced; advisory URLs never appear here by design. -- `dev-only-bugs [--apply]` — list closed milestone bugs that never reached production, without `no changelog`. Sure: parent issue is a `Feature`/`Enhancement` of the same milestone; `--apply` adds `no changelog` to these only. Candidates (never labelled automatically): parent is an epic of the milestone, `regression` label, or the "Affected versions" / "Environment" body field names only hourly, develop, staging, PRE or this milestone (`only_unreleased_envs`; any production hint or other version keeps the bug in). Reads `body` and `parent` from one batched `gh.py issue` lookup. +- `dev-only-bugs [--apply]` — list closed milestone bugs that never reached production, without `no changelog` (bugs carrying `release blocker` are analyzed too). Sure: parent issue is a `Feature`/`Enhancement` of the same milestone; `--apply` adds `no changelog` to these only. Candidates (never labelled automatically): parent is an epic of the milestone, `regression` label, or the "Affected versions" / "Environment" body field names only hourly, develop, staging, PRE or this milestone (`only_unreleased_envs`; any production hint or other version keeps the bug in). Reads `body` and `parent` from one batched `gh.py issue` lookup. - `report [--changes CHANGES.md --output CHANGES-ISSUES.md]` — overwrite the anomaly report: A/B milestone mismatches, E `:boom:` entry without the label, F `:rocket:` sync, G `:boom:` sync, plus the C/D `:rocket:` gaps and the `dev-only-bugs` lists (warnings). `:boom:`/`:rocket:` rule violations are fixed by regenerating or relabelling, not by editing entries in place. ## Conventions @@ -19,4 +19,4 @@ - PR/issue reference parsing covers standard `[#N](.../pull/N)` / `[#N](.../issues/N)` links plus legacy `PR:[N]` / `[Github #N]` forms. - Shared parsing helpers (`extract_version_section`, `extract_subsection`, `collect_changelog_prs/issues`) live at module top so future subcommands reuse them instead of duplicating regexes. - Report anomaly types: A/B milestone mismatches, E `:boom:` entry whose issue lacks the `breaking change` label (counted, entries preserved — human labels the issue or moves the entry), G `breaking change` issue missing from `:boom:` (counted, same contract as F); C/D are `:rocket:` gaps (warnings, never counted). -- `issue_excluded` (closed, non-Task, non-Rejected, no `release blocker`/`no changelog` label) is module-level and shared by `report` and `boom` so the exclusions cannot drift. +- `issue_excluded` (closed, non-Task, non-Rejected, no `no changelog` label) is module-level and shared by `report` and `boom` so the exclusions cannot drift. diff --git a/.serena/memories/scripts/gh.md b/.serena/memories/scripts/gh.md index 6b546e578c..d9775f3d82 100644 --- a/.serena/memories/scripts/gh.md +++ b/.serena/memories/scripts/gh.md @@ -40,7 +40,7 @@ python3 scripts/gh.py issues "2.16.0" --label "bug" python3 scripts/gh.py issues "2.16.0" --label "bug,regression" # Exclude by label -python3 scripts/gh.py issues "2.16.0" --exclude "release blocker,no changelog" +python3 scripts/gh.py issues "2.16.0" --exclude "no changelog" # Show only issues NOT yet in CHANGES.md python3 scripts/gh.py issues "2.16.0" --compare CHANGES.md diff --git a/.serena/memories/workflow/updating-changelog.md b/.serena/memories/workflow/updating-changelog.md index 6cf4134d1a..909b821006 100644 --- a/.serena/memories/workflow/updating-changelog.md +++ b/.serena/memories/workflow/updating-changelog.md @@ -28,7 +28,7 @@ Editing `CHANGES.md` happens only on explicit request or as part of the release - **Security fixes come from advisories**: an advisory whose `patched_versions` include the version becomes a `:lock:` entry (section 7). `triage`, `draft` and `published` are all valid states; the advisory may still be unpublished when the changelog is written. A security fix may also carry a regular bug issue; both entries are legal and independent (the issue lives in `:bug:` as any other bug). - **Two labels override the Issue Type**: `breaking change` → an entry only under `:boom:`, never in its regular section; `release highlight` → an extra entry under `:rocket:` on top of its regular entry. - **Only closed issues**. An open issue in the milestone is not generated, even if someone added a provisional entry by hand before the merge. -- **Excluded issues**: `no changelog`, `release blocker`, `Task` issue type, `Rejected` project status. +- **Excluded issues**: `no changelog`, `Task` issue type, `Rejected` project status. `release blocker` is a workflow label (the issue blocks the release), not a changelog exclusion: bugs carrying it are analyzed by `dev-only-bugs` (section 8, step 3), and closed non-bug issues carrying it are reviewed by hand (section 8, step 3). - **Bugs that never reached production** stay out: a bug found and fixed while building a feature of the same, unreleased version never affected users. The team rarely labels them, so the flow finds them with `dev-only-bugs` and labels them `no changelog` (section 8, step 3). - **Excluded PRs**: a PR with `release blocker` or `no issue required` does not bring an entry of its own. - **An issue with no milestone** belongs to another (probably private) project: it is neither changelog candidate nor anomaly. @@ -159,10 +159,11 @@ Generated from the advisories that declare the version in `patched_versions`; ne ## 8. Flow 1. **Confirm the version** (a semver string such as `2.19.0`) with the user when it is not obvious. -2. **Fetch the milestone issues**: `python3 scripts/gh.py issues "" --exclude "release blocker,no changelog"` (closed by default; add `--state all` only to inspect open issues). `gh.py` already drops `Task` and `Rejected`; pass `--include-tasks` / `--include-rejected` only to inspect them. To find what is missing from an existing section, add `--compare CHANGES.md`. -3. **Keep out bugs that never reached production**: `python3 scripts/changelog.py dev-only-bugs ""` lists them; add `--apply` to label the sure ones `no changelog` on GitHub, then re-fetch step 2. The label puts the decision on the issue, so it survives a regeneration. +2. **Fetch the milestone issues**: `python3 scripts/gh.py issues "" --exclude "no changelog"` (closed by default; add `--state all` only to inspect open issues). `gh.py` already drops `Task` and `Rejected`; pass `--include-tasks` / `--include-rejected` only to inspect them. To find what is missing from an existing section, add `--compare CHANGES.md`. +3. **Keep out bugs that never reached production**: `python3 scripts/changelog.py dev-only-bugs ""` lists them; add `--apply` to label the sure ones `no changelog` on GitHub, then re-fetch step 2. The label puts the decision on the issue, so it survives a regeneration. Bugs carrying `release blocker` are analyzed like any other bug. - **Sure**: a bug whose parent issue is a `Feature` or `Enhancement` of the same milestone. `--apply` labels these. - **Candidates**: a bug whose parent is an epic of the same milestone, that carries the `regression` label, or whose "Affected versions" / "Environment" field names only hourly, develop, staging, PRE or a version of this milestone. `regression` also marks regressions users hit in production, and the field says where the bug was seen, not since when it exists, so never label these yourself: show them to the user, who labels the ones they confirm. The anomaly report lists them too (section 10). + - **Closed non-bug issues carrying `release blocker`** have no automatic check: `python3 scripts/gh.py issues "" --label "release blocker" --exclude "no changelog"` lists them (closed, non-Task). Show them to the user: the internal ones get the `no changelog` label, the rest keep their entry. 4. **Fetch PR details when needed**: `python3 scripts/gh.py prs ` (also `--milestone`). Use it for the PR author, labels and merge state. 5. **Categorize** by Issue Type and the two labels (section 2), and **build the descriptions** (section 4). 6. **Verify PR merge status** before writing: `python3 scripts/changelog.py check-merged `. Only merged PRs go in the changelog; if a closing PR was closed unmerged, look for the PR that superseded it. @@ -179,7 +180,7 @@ Re-fetch the milestone (do not trust cached data) and fix these in `CHANGES.md` 1. **Duplicate across versions** → remove it from the current section; the earlier version wins for backports. The same holds after merging a patch branch into `staging`: the fix keeps its released patch entry and leaves the newer, unreleased section (the merge conflict in `CHANGES.md` is resolved by keeping both sections, newest first, minus the duplicate). 2. **Stale milestone assignment** (the issue moved out of this milestone) → remove the entry. -3. **Newly applied exclusion labels** (`no changelog`, `release blocker`) → remove the entry. +3. **Newly applied exclusion labels** (`no changelog`) → remove the entry. (`release blocker` is not an exclusion label; it never removes an entry by itself.) 4. **Issue no longer closed, deleted, or now `Rejected`** → remove the entry. 5. **Unmerged or moved PR reference** → fix the reference or remove the entry. A PR merged in a different milestone is an anomaly (section 10); do not remove it silently. 6. **Issue Type changed** → to `Task`: remove the entry; to another type: move the entry to that type's section. diff --git a/scripts/changelog.py b/scripts/changelog.py index e09cfa1218..9937f003f6 100644 --- a/scripts/changelog.py +++ b/scripts/changelog.py @@ -45,7 +45,7 @@ GH_PY = Path(__file__).resolve().parent / "gh.py" BOOM_HEADING = "### :boom: Breaking changes & Deprecations" SECURITY_HEADING = "### :lock: Security fixes" BREAKING_LABEL = "breaking change" -EXCLUDED_LABELS = {"release blocker", "no changelog"} +EXCLUDED_LABELS = {"no changelog"} # PR label carried by the security fixes that reference their advisory in the # PR body, and the GHSA id pattern used to find that reference. @@ -658,14 +658,15 @@ def find_dev_only_bugs(milestone: str, issues: list[dict]) -> tuple[list[dict], """Split the milestone's changelog bugs that never reached production. ``issues`` come from ``gh.py issue`` (they carry ``body`` and - ``parent``). Returns ``(auto, suggested)``: ``auto`` bugs are + ``parent``). Bugs labelled ``release blocker`` are analyzed like any + other bug. Returns ``(auto, suggested)``: ``auto`` bugs are sub-issues of a Feature or Enhancement of this milestone and get the `no changelog` label; ``suggested`` ones need a human to confirm.""" auto, suggested = [], [] for issue in issues: if issue.get("issue_type") != "Bug" or issue.get("state") != "CLOSED": continue - if {"no changelog", "release blocker"} & set(issue.get("labels", [])): + if "no changelog" in set(issue.get("labels", [])): continue parent = issue.get("parent") or {} entry = {"issue": issue["number"], "issue_title": issue.get("title", "")} diff --git a/scripts/gh.py b/scripts/gh.py index 30c12e58f2..a94c5be8fb 100755 --- a/scripts/gh.py +++ b/scripts/gh.py @@ -15,7 +15,7 @@ Subcommands: Usage: python3 scripts/gh.py issues (default: state=closed) python3 scripts/gh.py issues "2.16.0" --state all - python3 scripts/gh.py issues "2.16.0" --exclude "release blocker,no changelog" + python3 scripts/gh.py issues "2.16.0" --exclude "no changelog" python3 scripts/gh.py issues "2.16.0" --label "bug" (include only issues with label) python3 scripts/gh.py issues "2.16.0" --label "bug,regression" --exclude "no changelog" python3 scripts/gh.py issues "2.16.0" --compare CHANGES.md @@ -1092,7 +1092,7 @@ def main() -> None: ) p_issues.add_argument( "--exclude", "--exclude-labels", - help="Comma-separated labels to exclude, e.g. 'release blocker,no changelog'" + help="Comma-separated labels to exclude, e.g. 'no changelog'" ) p_issues.add_argument( "--label", "--labels", diff --git a/scripts/test_changelog.py b/scripts/test_changelog.py index 8eb0311c9f..0ad4181eca 100644 --- a/scripts/test_changelog.py +++ b/scripts/test_changelog.py @@ -131,9 +131,14 @@ class IssueExcludedTests(unittest.TestCase): self.assertTrue(ch.issue_excluded(self.issue(state="OPEN"))) self.assertTrue(ch.issue_excluded(self.issue(issue_type="Task"))) self.assertTrue(ch.issue_excluded(self.issue(project_status="Rejected"))) - self.assertTrue(ch.issue_excluded(self.issue(labels=["release blocker"]))) self.assertTrue(ch.issue_excluded(self.issue(labels=["no changelog"]))) + def test_keeps_a_release_blocker_issue(self): + # `release blocker` marks work that blocks the release; it is not a + # changelog exclusion. Bugs go through dev-only-bugs, anything else is + # a human call in the pre-flight. + self.assertFalse(ch.issue_excluded(self.issue(labels=["release blocker"]))) + class RenderBoomEntryTests(unittest.TestCase): def test_entry_is_a_heading_plus_note_plus_references(self): @@ -707,10 +712,13 @@ class FindDevOnlyBugsTests(unittest.TestCase): {**bug(8, parent={"number": 10, "issue_type": "Feature", "milestone": "2.19.0"}), "issue_type": "Enhancement"}, bug(9, body="### Affected versions\n\n2.18.2\n", labels=["regression"]), + bug(10, parent={"number": 10, "issue_type": "Feature", "milestone": "2.19.0"}, + labels=["release blocker"]), + bug(11, body="### Affected versions\n\nhourly\n", labels=["release blocker"]), ] auto, suggested = ch.find_dev_only_bugs("2.19.0", issues) - self.assertEqual([e["issue"] for e in auto], [1, 2]) - self.assertEqual([e["issue"] for e in suggested], [3, 5, 9]) + self.assertEqual([e["issue"] for e in auto], [1, 2, 10]) + self.assertEqual([e["issue"] for e in suggested], [3, 5, 9, 11]) if __name__ == "__main__": From 4a9f553c19b389d89da9457f78039dbca109bc58 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Mon, 16 Nov 2026 10:03:54 +0000 Subject: [PATCH 15/15] :books: Update the 2.20.0 changelog Regenerate the 2.20.0 section for the new layout: `:boom:` and `:lock:` are generated from the breaking change notes and the advisory (patched_versions includes 2.20.0); features and enhancements move to their own `:tada:` and `:sparkles:` sections instead of one mixed section; `:rocket:` lists the two milestone issues labelled `release highlight` (#11759 and #11761, no closing PR in the milestone). Add the issues closed since the section was written: #10436, #12052, #12135, #12615, #12728, #12738, #12740, #12948 and #13028 as bugs, and #10465, #12265, #12796, #12902, #13029 and #13030 as enhancements. Bugs that never reached production stay out per `dev-only-bugs` (#12860, #12138) and manual review (#12901). #12977 leaves the section: its fix (PR #12978, merged 2026-11-13 in develop) carries milestone 2.21.0, so the issue moves there and will be picked up by the 2.21.0 changelog. AI-assisted-by: inkling-2-small-free --- CHANGES.md | 124 ++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 89 insertions(+), 35 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index e1aa8dc7b9..22d44df2fe 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,10 +2,91 @@ ## 2.20.0 -### :bug: Bugs fixed +### :boom: Breaking changes & Deprecations +#### Add table partitioning for file-change + +The file history table (`file_change`) is now partitioned by file, with the same 16 partitions as the file payload table (`file_data`). Queries that filter by file, and the join between the two tables, resolve against a single partition, so they stay fast as the table grows. The rewrite also removes the dead columns and stores the table in its final layout; the space the old table held comes back when it is dropped. A save no longer writes the file features on every revision, and the team snapshot quota is a single lookup. + +The four snapshot RPC methods (`update-file-snapshot`, `delete-file-snapshot`, `lock-file-snapshot` and `unlock-file-snapshot`) now require `file-id`. The frontend already sends it; API clients that call them must add it. + +On small and medium instances the table is rewritten automatically on boot; there is nothing to do. On large instances (hundreds of GB in `file_change`) it does not fit in the startup transaction: run it by hand **before upgrading to 2.20.0** following the [large-instance runbook](https://github.com/penpot/penpot/blob/staging/operations/2.20.0-file-change-partition.md). The copy runs while the instance is online, followed by a cutover of a few minutes with the backend and the workers stopped. The old table is kept for rollback; check the disk headroom before starting, and drop the old table days later to get the space back. + +[#12416](https://github.com/penpot/penpot/issues/12416) (PR: [#12417](https://github.com/penpot/penpot/pull/12417)) + +#### Protect the file data storage objects with a foreign key in the backend + +This release moves the file data storage pointer out of `file_data.metadata->>'storage-ref-id'` and into a new `file_data.storage_object_id` column with a foreign key to `storage_object(id)`. The metadata key is gone: nothing writes or reads it. + +What it means for self-hosted instances: + +- The release is a coordinated restart. The new code reads the column; the migration drops `storage-ref-id` from the rows it backfills. Rolling back to the previous binary is not supported after this release runs. +- A fresh database or a small instance needs nothing special: the startup migration does the whole job. +- A large instance should not let the startup migration run. Build the column, the index and the foreign key online beforehand, stop the instances, run the backfill only if a row still carries `metadata.storage-ref-id`, verify the end state, and mark the migration as applied. The full procedure is documented in [`operations/2.20.0-file-data-storage-object-id.md`](https://github.com/penpot/penpot/blob/main/operations/2.20.0-file-data-storage-object-id.md). +- If the storage backend was never enabled, there is nothing to backfill and the stopped window is just the restart. + +If you upgrade blindly: on fresh databases and small instances the startup migration completes the change. On a large instance, the foreign key validation scan can block writes for a long time; follow the runbook instead. + +[#12768](https://github.com/penpot/penpot/issues/12768) (PR: [#12769](https://github.com/penpot/penpot/pull/12769)) + +### :rocket: Epics and highlights + +- Allow MCP to work with multiple open Penpot files [#11759](https://github.com/penpot/penpot/issues/11759) +- Allow MCP to access and manage Penpot Dashboard resources [#11761](https://github.com/penpot/penpot/issues/11761) + +### :lock: Security fixes + +- Fix real-time collaboration WebSocket never re-validates file access after `:subscribe-file`, letting revoked users keep receiving live edit content (https://github.com/penpot/penpot/security/advisories/GHSA-m53j-2766-6jqw) + +### :tada: Features + +- Add a feedback channel to the MCP server [#12490](https://github.com/penpot/penpot/issues/12490) (PR: [#12551](https://github.com/penpot/penpot/pull/12551)) +- Allow MCP to work with multiple open Penpot files [#11759](https://github.com/penpot/penpot/issues/11759) +- Implement full text format paste within Penpot [#12047](https://github.com/penpot/penpot/issues/12047) (PR: [#12137](https://github.com/penpot/penpot/pull/12137)) +- Add team, project and file management to the Plugin API for global plugins [#12170](https://github.com/penpot/penpot/issues/12170) (PR: [#12169](https://github.com/penpot/penpot/pull/12169)) +- Serve Penpot under a subpath [#3194](https://github.com/penpot/penpot/issues/3194) (PR: [#12358](https://github.com/penpot/penpot/pull/12358)) +- Allow MCP to access and manage Penpot Dashboard resources [#11761](https://github.com/penpot/penpot/issues/11761) +- Add OAuth support for MCP client connections [#12034](https://github.com/penpot/penpot/issues/12034) (PR: [#12480](https://github.com/penpot/penpot/pull/12480)) + +### :sparkles: Enhancements + +- Auto-scroll sortable lists when dragging an item near their top or bottom edge (by @ShreyashAgare26) [#10478](https://github.com/penpot/penpot/issues/10478) (PR: [#11595](https://github.com/penpot/penpot/pull/11595)) +- Make WebGL the default renderer for new accounts [#11855](https://github.com/penpot/penpot/issues/11855) (PR: [#12880](https://github.com/penpot/penpot/pull/12880)) +- Prevent the hosted MCP server from running execute_code twice when the plugin is open in two tabs [#12008](https://github.com/penpot/penpot/issues/12008) (PR: [#11927](https://github.com/penpot/penpot/pull/11927)) +- Allow detaching typography tokens [#11887](https://github.com/penpot/penpot/issues/11887) (PR: [#11969](https://github.com/penpot/penpot/pull/11969)) +- Complete the Russian translations [#11955](https://github.com/penpot/penpot/issues/11955) (PR: [#11956](https://github.com/penpot/penpot/pull/11956)) +- Run .penpot file import and export as durable background jobs [#12194](https://github.com/penpot/penpot/issues/12194) (PR: [#12056](https://github.com/penpot/penpot/pull/12056), [#12192](https://github.com/penpot/penpot/pull/12192)) +- Open files and projects in a new tab with Ctrl/Cmd+click in the dashboard [#12228](https://github.com/penpot/penpot/issues/12228) (PR: [#12230](https://github.com/penpot/penpot/pull/12230)) +- Add an RPC method to read file-level plugin data [#12258](https://github.com/penpot/penpot/issues/12258) (PR: [#10171](https://github.com/penpot/penpot/pull/10171)) +- Add a get-file-tokens RPC method to read a file's design tokens [#12260](https://github.com/penpot/penpot/issues/12260) (PR: [#12172](https://github.com/penpot/penpot/pull/12172)) +- Show the progress and the size of each file in the import and export dialogs [#12340](https://github.com/penpot/penpot/issues/12340) (PR: [#12192](https://github.com/penpot/penpot/pull/12192)) +- Focus the search field when opening the color tokens tab of the color picker [#12344](https://github.com/penpot/penpot/issues/12344) (PR: [#12345](https://github.com/penpot/penpot/pull/12345)) +- Harden the file data offload job with a row lock, per-file transactions, throttle and metrics [#12398](https://github.com/penpot/penpot/issues/12398) (PR: [#12403](https://github.com/penpot/penpot/pull/12403)) +- Render email templates inside the mail job in the backend [#12445](https://github.com/penpot/penpot/issues/12445) (PR: [#12448](https://github.com/penpot/penpot/pull/12448)) +- Run local WASM exports for shapes and boards without freezing the UI [#12512](https://github.com/penpot/penpot/issues/12512) (PR: [#12515](https://github.com/penpot/penpot/pull/12515)) +- Use WASM to generate SVG markup in the code inspector when render-wasm is active [#12567](https://github.com/penpot/penpot/issues/12567) (PR: [#12608](https://github.com/penpot/penpot/pull/12608)) +- Close the workspace search with the Escape key (by @filipsajdak) [#9540](https://github.com/penpot/penpot/issues/9540) (PR: [#12374](https://github.com/penpot/penpot/pull/12374)) +- Restrict management RPC methods to their declared shared keys [#12730](https://github.com/penpot/penpot/issues/12730) (PR: [#12736](https://github.com/penpot/penpot/pull/12736)) +- Reduce memory used by duplicate shape ids in the workspace [#12791](https://github.com/penpot/penpot/issues/12791) (PR: [#12792](https://github.com/penpot/penpot/pull/12792)) +- Reduce memory consumption on shape decoding [#12789](https://github.com/penpot/penpot/issues/12789) (PR: [#12790](https://github.com/penpot/penpot/pull/12790)) +- Notify other users when a snapshot is restored [#12425](https://github.com/penpot/penpot/issues/12425) (PR: [#12555](https://github.com/penpot/penpot/pull/12555)) +- Normalize the configured tenant to lowercase [#12842](https://github.com/penpot/penpot/issues/12842) (PR: [#12843](https://github.com/penpot/penpot/pull/12843)) +- Speed up saves on large files by checking only the pages a save changes [#12602](https://github.com/penpot/penpot/issues/12602) (PR: [#12555](https://github.com/penpot/penpot/pull/12555)) +- Retry saves after short server failures and keep the workspace open when a save fails [#12601](https://github.com/penpot/penpot/issues/12601) (PR: [#12555](https://github.com/penpot/penpot/pull/12555)) +- Switch the workspace to read-only mode when the file can't be saved [#12507](https://github.com/penpot/penpot/issues/12507) (PR: [#12555](https://github.com/penpot/penpot/pull/12555)) +- Validate upload content against real file bytes in media uploads [#12796](https://github.com/penpot/penpot/issues/12796) (PR: [#12801](https://github.com/penpot/penpot/pull/12801)) +- Add the file and MCP session to audit events sent from plugins and the MCP [#12902](https://github.com/penpot/penpot/issues/12902) (PR: [#12972](https://github.com/penpot/penpot/pull/12972)) +- Add skip-if-active option to job submission in the backend jobs substrate [#12265](https://github.com/penpot/penpot/issues/12265) (PR: [#12266](https://github.com/penpot/penpot/pull/12266)) +- Add HTML paste from clipboard in text editor v3 [#10465](https://github.com/penpot/penpot/issues/10465) +- Skip loading the thumbnail rasterizer in WebGL renderer sessions to cut workspace memory [#13029](https://github.com/penpot/penpot/issues/13029) (PR: [#12835](https://github.com/penpot/penpot/pull/12835)) +- Speed up uploading page shapes to WASM [#13030](https://github.com/penpot/penpot/issues/13030) (PR: [#12854](https://github.com/penpot/penpot/pull/12854)) + +### :bug: Bugs + +- Fix font upload failing with a regex error [#10436](https://github.com/penpot/penpot/issues/10436) (PR: [#11951](https://github.com/penpot/penpot/pull/11951)) - Fix dashboard file thumbnails loading slow for large multi-page files [#12053](https://github.com/penpot/penpot/issues/12053) (PR: [#12054](https://github.com/penpot/penpot/pull/12054)) - Fix Plugin API resolving a font family token to its separate words when the family name has several words [#11748](https://github.com/penpot/penpot/issues/11748) (PR: [#11750](https://github.com/penpot/penpot/pull/11750)) +- Fix file import recreating circular library references when the exported libraries use each other [#12052](https://github.com/penpot/penpot/issues/12052) (PR: [#12062](https://github.com/penpot/penpot/pull/12062)) - Fix asset redirect responses advertising a cache lifetime of about 1000 days instead of 24 hours (by @ners) [#12761](https://github.com/penpot/penpot/issues/12761) (PR: [#12758](https://github.com/penpot/penpot/pull/12758)) - Fix an internal error on typography tokens due to circular dependencies on other token types [#9580](https://github.com/penpot/penpot/issues/9580) - Fix the MCP server rejecting request bodies over 100kb (express.json default) with PayloadTooLargeError (by @GitNimay) [#12080](https://github.com/penpot/penpot/issues/12080) (PR: [#12450](https://github.com/penpot/penpot/pull/12450)) @@ -22,40 +103,13 @@ - Fix token not showing when selecting two text layers (by @ShreyashAgare26) [#11924](https://github.com/penpot/penpot/issues/11924) (PR: [#11944](https://github.com/penpot/penpot/pull/11944)) - Fix inner shadow fill not working in the SVG render while it works in WebGL [#11967](https://github.com/penpot/penpot/issues/11967) (PR: [#12141](https://github.com/penpot/penpot/pull/12141)) - Fix @penpot/plugin-styles loading Work Sans from a third-party CDN (by @ump45nose) [#12023](https://github.com/penpot/penpot/issues/12023) (PR: [#12087](https://github.com/penpot/penpot/pull/12087)) - -### :sparkles: New features & Enhancements - -- Add a feedback channel to the MCP server [#12490](https://github.com/penpot/penpot/issues/12490) (PR: [#12551](https://github.com/penpot/penpot/pull/12551)) -- Auto-scroll sortable lists when dragging an item near their top or bottom edge (by @ShreyashAgare26) [#10478](https://github.com/penpot/penpot/issues/10478) (PR: [#11595](https://github.com/penpot/penpot/pull/11595)) -- Make WebGL the default renderer for new accounts [#11855](https://github.com/penpot/penpot/issues/11855) (PR: [#12880](https://github.com/penpot/penpot/pull/12880)) -- Allow MCP to work with multiple open Penpot files [#11759](https://github.com/penpot/penpot/issues/11759) -- Complete the Russian translations [#11955](https://github.com/penpot/penpot/issues/11955) (PR: [#11956](https://github.com/penpot/penpot/pull/11956)) -- Implement full text format paste within Penpot [#12047](https://github.com/penpot/penpot/issues/12047) (PR: [#12137](https://github.com/penpot/penpot/pull/12137)) -- Prevent the hosted MCP server from running execute_code twice when the plugin is open in two tabs [#12008](https://github.com/penpot/penpot/issues/12008) (PR: [#11927](https://github.com/penpot/penpot/pull/11927)) -- Allow detaching typography tokens [#11887](https://github.com/penpot/penpot/issues/11887) (PR: [#11969](https://github.com/penpot/penpot/pull/11969)) -- Add team, project and file management to the Plugin API for global plugins [#12170](https://github.com/penpot/penpot/issues/12170) (PR: [#12169](https://github.com/penpot/penpot/pull/12169)) -- Run .penpot file import and export as durable background jobs [#12194](https://github.com/penpot/penpot/issues/12194) (PR: [#12056](https://github.com/penpot/penpot/pull/12056), [#12192](https://github.com/penpot/penpot/pull/12192)) -- Open files and projects in a new tab with Ctrl/Cmd+click in the dashboard [#12228](https://github.com/penpot/penpot/issues/12228) (PR: [#12230](https://github.com/penpot/penpot/pull/12230)) -- Add an RPC method to read file-level plugin data [#12258](https://github.com/penpot/penpot/issues/12258) (PR: [#10171](https://github.com/penpot/penpot/pull/10171)) -- Add a get-file-tokens RPC method to read a file's design tokens [#12260](https://github.com/penpot/penpot/issues/12260) (PR: [#12172](https://github.com/penpot/penpot/pull/12172)) -- Show the progress and the size of each file in the import and export dialogs [#12340](https://github.com/penpot/penpot/issues/12340) (PR: [#12192](https://github.com/penpot/penpot/pull/12192)) -- Focus the search field when opening the color tokens tab of the color picker [#12344](https://github.com/penpot/penpot/issues/12344) (PR: [#12345](https://github.com/penpot/penpot/pull/12345)) -- Harden the file data offload job with a row lock, per-file transactions, throttle and metrics [#12398](https://github.com/penpot/penpot/issues/12398) (PR: [#12403](https://github.com/penpot/penpot/pull/12403)) -- Render email templates inside the mail job in the backend [#12445](https://github.com/penpot/penpot/issues/12445) (PR: [#12448](https://github.com/penpot/penpot/pull/12448)) -- Serve Penpot under a subpath [#3194](https://github.com/penpot/penpot/issues/3194) (PR: [#12358](https://github.com/penpot/penpot/pull/12358)) -- Run local WASM exports for shapes and boards without freezing the UI [#12512](https://github.com/penpot/penpot/issues/12512) (PR: [#12515](https://github.com/penpot/penpot/pull/12515)) -- Use WASM to generate SVG markup in the code inspector when render-wasm is active [#12567](https://github.com/penpot/penpot/issues/12567) (PR: [#12608](https://github.com/penpot/penpot/pull/12608)) -- Close the workspace search with the Escape key (by @filipsajdak) [#9540](https://github.com/penpot/penpot/issues/9540) (PR: [#12374](https://github.com/penpot/penpot/pull/12374)) -- Allow MCP to access and manage Penpot Dashboard resources [#11761](https://github.com/penpot/penpot/issues/11761) -- Add OAuth support for MCP client connections [#12034](https://github.com/penpot/penpot/issues/12034) (PR: [#12480](https://github.com/penpot/penpot/pull/12480)) -- Restrict management RPC methods to their declared shared keys [#12730](https://github.com/penpot/penpot/issues/12730) (PR: [#12736](https://github.com/penpot/penpot/pull/12736)) -- Reduce memory used by duplicate shape ids in the workspace [#12791](https://github.com/penpot/penpot/issues/12791) (PR: [#12792](https://github.com/penpot/penpot/pull/12792)) -- Reduce memory consumption on shape decoding [#12789](https://github.com/penpot/penpot/issues/12789) (PR: [#12790](https://github.com/penpot/penpot/pull/12790)) -- Notify other users when a snapshot is restored [#12425](https://github.com/penpot/penpot/issues/12425) (PR: [#12555](https://github.com/penpot/penpot/pull/12555)) -- Normalize the configured tenant to lowercase [#12842](https://github.com/penpot/penpot/issues/12842) (PR: [#12843](https://github.com/penpot/penpot/pull/12843)) -- Speed up saves on large files by checking only the pages a save changes [#12602](https://github.com/penpot/penpot/issues/12602) (PR: [#12555](https://github.com/penpot/penpot/pull/12555)) -- Retry saves after short server failures and keep the workspace open when a save fails [#12601](https://github.com/penpot/penpot/issues/12601) (PR: [#12555](https://github.com/penpot/penpot/pull/12555)) -- Switch the workspace to read-only mode when the file can't be saved [#12507](https://github.com/penpot/penpot/issues/12507) (PR: [#12555](https://github.com/penpot/penpot/pull/12555)) +- Fix schema error on swatch component for image fills [#12135](https://github.com/penpot/penpot/issues/12135) (PR: [#12136](https://github.com/penpot/penpot/pull/12136)) +- Fix worker storing more shape data than needed [#12740](https://github.com/penpot/penpot/issues/12740) (PR: [#12840](https://github.com/penpot/penpot/pull/12840)) +- Fix worker not freeing pages [#12738](https://github.com/penpot/penpot/issues/12738) (PR: [#12751](https://github.com/penpot/penpot/pull/12751)) +- Fix editing the L value in HSL mode resetting H and S to 0 (by @makesomethingshit) [#12615](https://github.com/penpot/penpot/issues/12615) (PR: [#12629](https://github.com/penpot/penpot/pull/12629)) +- Restrict dashboard plugins to the MCP plugin and localhost plugins [#12948](https://github.com/penpot/penpot/issues/12948) (PR: [#12949](https://github.com/penpot/penpot/pull/12949)) +- Fix workspace memory growing on every page switch with the WebGL renderer [#13028](https://github.com/penpot/penpot/issues/13028) (PR: [#12877](https://github.com/penpot/penpot/pull/12877)) +- Fix workspace showing React error #310 when the team container renders without a team id (by @filipsajdak) [#12728](https://github.com/penpot/penpot/issues/12728) (PR: [#12920](https://github.com/penpot/penpot/pull/12920)) ## 2.19.1