mirror of
https://github.com/penpot/penpot.git
synced 2026-09-23 04:16:13 +00:00
0e388442a1
* ♻️ Simplify storage GC delays and add skip-delay task params The touched GC no longer applies an extra deletion-delay when marking storage objects as deleted. By the time a storage object is touched, its referencing domain row has already passed its own deletion delay, and the reference scan is the only safety check needed. Touched objects are now marked with deleted_at = now, so the deleted GC removes them on the next run. For the tempfile bucket, upload chunks now set touched-at in the future (1h, aligned with the upload-session-gc TTL) instead of relying on a special-case deletion delay. Task handlers now read their task props: - storage-gc-touched accepts :skip-delay to process all touched objects immediately, bypassing the min-age threshold. - objects-gc accepts :chunk-size and :skip-delay to process recently deleted rows without waiting for the deletion delay. This allows running the deletion cascade immediately from the REPL via run-task! with the skip-delay option. AI-assisted-by: deepseek-v4-flash * ✨ Add storage object status lifecycle, verified dedup, and deletion retry tracking Storage object lifecycle hardening: - Add status column ('valid' | 'pending') as write-ahead marker for object creation. put-object! inserts in 'pending' state, writes blob, then promotes to 'valid'. Failed writes remove the pending row. - Add :storage-pending-gc task to reclaim orphaned pending rows (e.g. after crash between blob write and promotion). - Verify blob existence on every dedup hit via exists-object? (fs stat / s3 headObject). Missing blobs mark the row as deleted and create fresh object. - Add deletion_attempts column (migration 0154) to track physical blob deletion attempts. Restructure gc_deleted to use chunked processing with per-chunk transactions (short lock duration). Failed deletions are deferred to tomorrow (deleted_at = NOW() + 1 day) to prevent infinite loops. After 7 attempts, give up and accept orphan. - Change del-objects-in-bulk contract to return #{fail-ids} for precise per-id tracking (fs and s3 backends updated). - Use tmp/tempfile for fs atomic writes with cleanup queue registration (crashed-JVM temp files swept ~60min later). Document ATOMIC_MOVE POSIX-only assumption. - Add linear backoff to s3 exists-object? retries (100ms/200ms/300ms). - Wrap compensating delete in put-object! catch block to prevent masking original error when connection is aborted. - Fix assert messages in pending_gc.clj and gc_deleted.clj (pool assertion said 'expected valid storage' instead of 'db pool'). - Add pending-objects-excluded-from-gc-deleted test. Use unique path in put-object-write-failure-leaves-no-row test to avoid collisions. AI-assisted-by: qwen3.7-plus * 🐛 Fix review comments on gc-deleted and storage - Fix process-chunk! returning nil causing (+ acc nil) crash - Add FOR UPDATE SKIP LOCKED to sql:get-deleted-chunk to prevent infinite loop when another worker holds locks - Pass :cause to log messages in gc_deleted.clj and s3.clj - Fix extra space in log hint string - Remove unused ::blob-missing? reference from storage memory - Rename test to match actual behavior (leaves pending row) - Add test for gc-deleted giving up after max attempts AI-assisted-by: qwen3.7-plus
446 lines
18 KiB
Clojure
446 lines
18 KiB
Clojure
;; This Source Code Form is subject to the terms of the Mozilla Public
|
|
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
;;
|
|
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
|
|
|
(ns app.rpc.commands.files-thumbnails
|
|
(:require
|
|
[app.binfile.common :as bfc]
|
|
[app.common.data :as d]
|
|
[app.common.data.macros :as dm]
|
|
[app.common.features :as cfeat]
|
|
[app.common.files.helpers :as cfh]
|
|
[app.common.geom.shapes :as gsh]
|
|
[app.common.schema :as sm]
|
|
[app.common.thumbnails :as thc]
|
|
[app.common.time :as ct]
|
|
[app.common.types.shape-tree :as ctt]
|
|
[app.config :as cf]
|
|
[app.db :as db]
|
|
[app.db.sql :as-alias sql]
|
|
[app.loggers.audit :as-alias audit]
|
|
[app.loggers.webhooks :as-alias webhooks]
|
|
[app.media.validation :as media.v]
|
|
[app.rpc :as-alias rpc]
|
|
[app.rpc.climit :as-alias climit]
|
|
[app.rpc.commands.files :as files]
|
|
[app.rpc.commands.teams :as teams]
|
|
[app.rpc.cond :as-alias cond]
|
|
[app.rpc.doc :as-alias doc]
|
|
[app.rpc.retry :as rtry]
|
|
[app.storage :as sto]
|
|
[app.util.pointer-map :as pmap]
|
|
[app.util.services :as sv]
|
|
[cuerdas.core :as str]))
|
|
|
|
;; --- FEATURES
|
|
|
|
(def long-cache-duration
|
|
(ct/duration {:days 7}))
|
|
|
|
;; --- COMMAND QUERY: get-file-object-thumbnails
|
|
|
|
(defn- get-object-thumbnails-by-tag
|
|
[conn file-id tag]
|
|
(let [sql (str/concat
|
|
"select object_id, media_id, tag "
|
|
" from file_tagged_object_thumbnail"
|
|
" where file_id=? and tag=? and deleted_at is null")
|
|
res (db/exec! conn [sql file-id tag])]
|
|
(->> res
|
|
(d/index-by :object-id :media-id)
|
|
(d/without-nils))))
|
|
|
|
(defn- get-object-thumbnails
|
|
([conn file-id]
|
|
(let [sql (str/concat
|
|
"select object_id, media_id, tag "
|
|
" from file_tagged_object_thumbnail"
|
|
" where file_id=? and deleted_at is null")
|
|
res (db/exec! conn [sql file-id])]
|
|
(->> res
|
|
(d/index-by :object-id :media-id)
|
|
(d/without-nils))))
|
|
|
|
([conn file-id object-ids]
|
|
(let [sql (str/concat
|
|
"select object_id, media_id, tag "
|
|
" from file_tagged_object_thumbnail"
|
|
" where file_id=? and object_id = ANY(?) and deleted_at is null")
|
|
ids (db/create-array conn "text" (seq object-ids))
|
|
res (db/exec! conn [sql file-id ids])]
|
|
|
|
(->> res
|
|
(d/index-by :object-id :media-id)
|
|
(d/without-nils)))))
|
|
|
|
(sv/defmethod ::get-file-object-thumbnails
|
|
"Retrieve a file object thumbnails."
|
|
{::doc/added "1.17"
|
|
::doc/module :files
|
|
::sm/params [:map {:title "get-file-object-thumbnails"}
|
|
[:file-id ::sm/uuid]
|
|
[:tag {:optional true} [:string {:max 50}]]]
|
|
::sm/result [:map-of [:string {:max 250}] [:string {:max 250}]]}
|
|
[{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id tag] :as params}]
|
|
(dm/with-open [conn (db/open pool)]
|
|
(files/check-read-permissions! cfg profile-id file-id)
|
|
(if tag
|
|
(get-object-thumbnails-by-tag conn file-id tag)
|
|
(get-object-thumbnails conn file-id))))
|
|
|
|
;; --- COMMAND QUERY: get-file-data-for-thumbnail
|
|
|
|
;; We need to improve how we set frame for thumbnail in order to avoid
|
|
;; loading all pages into memory for find the frame set for thumbnail.
|
|
|
|
(defn get-file-data-for-thumbnail
|
|
[{:keys [::db/conn] :as cfg} {:keys [data id] :as file} strip-frames-with-thumbnails]
|
|
(letfn [;; function responsible on finding the frame marked to be
|
|
;; used as thumbnail; the returned frame always have
|
|
;; the :page-id set to the page that it belongs.
|
|
(get-thumbnail-frame [{:keys [data]}]
|
|
(d/seek #(or (:use-for-thumbnail %)
|
|
(:use-for-thumbnail? %)) ; NOTE: backward comp (remove on v1.21)
|
|
(for [page (-> data :pages-index vals)
|
|
frame (-> page :objects ctt/get-frames)]
|
|
(assoc frame :page-id (:id page)))))
|
|
|
|
;; function responsible to filter objects data structure of
|
|
;; all unneeded shapes if a concrete frame is provided. If no
|
|
;; frame, the objects is returned untouched.
|
|
(filter-objects [objects frame-id]
|
|
(d/index-by :id (cfh/get-children-with-self objects frame-id)))
|
|
|
|
;; function responsible of assoc available thumbnails
|
|
;; to frames and remove all children shapes from objects if
|
|
;; thumbnails is available
|
|
(assoc-thumbnails [objects page-id thumbnails]
|
|
(loop [objects objects
|
|
frames (filter cfh/frame-shape? (vals objects))]
|
|
|
|
(if-let [frame (-> frames first)]
|
|
(let [frame-id (:id frame)
|
|
object-id (thc/fmt-object-id (:id file) page-id frame-id "frame")
|
|
|
|
frame (if-let [media-id (get thumbnails object-id)]
|
|
(-> frame
|
|
(assoc :thumbnail-id media-id)
|
|
(assoc :shapes []))
|
|
(dissoc frame :thumbnail))
|
|
|
|
children-ids
|
|
(cfh/get-children-ids objects frame-id)
|
|
|
|
bounds
|
|
(when (:show-content frame)
|
|
(gsh/shapes->rect (cons frame (map (d/getf objects) children-ids))))
|
|
|
|
frame
|
|
(cond-> frame
|
|
(some? bounds)
|
|
(assoc :children-bounds bounds))]
|
|
|
|
(if (:thumbnail frame)
|
|
(recur (-> objects
|
|
(assoc frame-id frame)
|
|
(d/without-keys children-ids))
|
|
(rest frames))
|
|
(recur (assoc objects frame-id frame)
|
|
(rest frames))))
|
|
|
|
objects)))]
|
|
|
|
(let [frame (get-thumbnail-frame file)
|
|
frame-id (:id frame)
|
|
page-id (or (:page-id frame)
|
|
(-> data :pages first))
|
|
|
|
page (dm/get-in data [:pages-index page-id])
|
|
page (cond-> page (pmap/pointer-map? page) deref)
|
|
frame-ids (if (some? frame) (list frame-id) (map :id (ctt/get-frames (:objects page))))
|
|
|
|
obj-ids (map #(thc/fmt-object-id (:id file) page-id % "frame") frame-ids)
|
|
thumbs (get-object-thumbnails conn id obj-ids)]
|
|
|
|
(cond-> page
|
|
;; If we have frame, we need to specify it on the page level
|
|
;; and remove the all other unrelated objects.
|
|
(some? frame-id)
|
|
(-> (assoc :thumbnail-frame-id frame-id)
|
|
(update :objects filter-objects frame-id))
|
|
|
|
;; Assoc the available thumbnails and prune not visible shapes
|
|
;; for avoid transfer unnecessary data.
|
|
strip-frames-with-thumbnails
|
|
(update :objects assoc-thumbnails page-id thumbs)))))
|
|
|
|
(def ^:private
|
|
schema:get-file-data-for-thumbnail
|
|
[:map {:title "get-file-data-for-thumbnail"}
|
|
[:file-id ::sm/uuid]])
|
|
|
|
(def ^:private
|
|
schema:partial-file
|
|
[:map {:title "PartialFile"}
|
|
[:id ::sm/uuid]
|
|
[:revn {:min 0} ::sm/int]
|
|
[:page [:map-of :keyword ::sm/any]]
|
|
[:strip-frames-with-thumbnails {:optional true} ::sm/boolean]])
|
|
|
|
(sv/defmethod ::get-file-data-for-thumbnail
|
|
"Retrieves the data for generate the thumbnail of the file. Used
|
|
mainly for render thumbnails on dashboard."
|
|
{::doc/added "1.17"
|
|
::doc/module :files
|
|
::sm/params schema:get-file-data-for-thumbnail
|
|
::sm/result schema:partial-file}
|
|
[cfg {:keys [::rpc/profile-id file-id strip-frames-with-thumbnails] :as params}]
|
|
(db/run! cfg (fn [cfg]
|
|
(files/check-read-permissions! cfg profile-id file-id)
|
|
(let [team (teams/get-team cfg
|
|
:profile-id profile-id
|
|
:file-id file-id)
|
|
file (bfc/get-file cfg file-id
|
|
:include-deleted? true
|
|
:realize? true
|
|
:read-only? true)
|
|
strip-frames-with-thumbnails
|
|
(or (nil? strip-frames-with-thumbnails) ;; if not present, default to true
|
|
(true? strip-frames-with-thumbnails))]
|
|
|
|
(-> (cfeat/get-team-enabled-features cf/flags team)
|
|
(cfeat/check-file-features! (:features file)))
|
|
|
|
{:file-id file-id
|
|
:revn (:revn file)
|
|
:page (get-file-data-for-thumbnail cfg file strip-frames-with-thumbnails)}))))
|
|
|
|
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
;; MUTATION COMMANDS
|
|
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
|
|
(def sql:get-file-object-thumbnail
|
|
"SELECT * FROM file_tagged_object_thumbnail
|
|
WHERE file_id = ? AND object_id = ? AND tag = ?
|
|
FOR UPDATE")
|
|
|
|
(def sql:create-file-object-thumbnail
|
|
"INSERT INTO file_tagged_object_thumbnail (file_id, object_id, tag, media_id)
|
|
VALUES (?, ?, ?, ?)
|
|
ON CONFLICT (file_id, object_id, tag)
|
|
DO UPDATE SET updated_at=?, media_id=?, deleted_at=?
|
|
RETURNING *")
|
|
|
|
(defn- persist-thumbnail!
|
|
[storage media created-at]
|
|
(let [path (:path media)
|
|
mtype (:mtype media)
|
|
hash (sto/calculate-hash path)
|
|
data (-> (sto/content path)
|
|
(sto/wrap-with-hash hash))]
|
|
|
|
(sto/put-object! storage
|
|
{::sto/content data
|
|
::sto/deduplicate? true
|
|
::sto/touched-at created-at
|
|
:content-type mtype
|
|
:bucket "file-object-thumbnail"})))
|
|
|
|
(defn- create-file-object-thumbnail!
|
|
[{:keys [::sto/storage] :as cfg} file object-id media tag]
|
|
(let [file-id (:id file)
|
|
timestamp (ct/now)
|
|
media (persist-thumbnail! storage media timestamp)
|
|
[th1 th2] (db/tx-run! cfg (fn [{:keys [::db/conn]}]
|
|
(let [th1 (db/exec-one! conn [sql:get-file-object-thumbnail file-id object-id tag])
|
|
th2 (db/exec-one! conn [sql:create-file-object-thumbnail
|
|
file-id object-id tag
|
|
(:id media)
|
|
timestamp
|
|
(:id media)
|
|
(:deleted-at file)])]
|
|
[th1 th2])))]
|
|
|
|
(when (and (some? th1)
|
|
(not= (:media-id th1)
|
|
(:media-id th2)))
|
|
(sto/touch-object! storage (:media-id th1)))
|
|
|
|
th2))
|
|
|
|
(def ^:private
|
|
schema:create-file-object-thumbnail
|
|
[:map {:title "create-file-object-thumbnail"}
|
|
[:file-id ::sm/uuid]
|
|
[:object-id [:string {:max 250}]]
|
|
[:media media.v/schema:upload]
|
|
[:tag {:optional true} [:string {:max 50}]]])
|
|
|
|
(sv/defmethod ::create-file-object-thumbnail
|
|
{::doc/added "1.19"
|
|
::doc/module :files
|
|
::climit/id [[:file-thumbnail-ops/by-profile ::rpc/profile-id]
|
|
[:file-thumbnail-ops/global]]
|
|
::rtry/enabled true
|
|
::rtry/when rtry/conflict-exception?
|
|
::audit/skip true
|
|
::sm/params schema:create-file-object-thumbnail}
|
|
|
|
[cfg {:keys [::rpc/profile-id file-id object-id media tag]}]
|
|
(media.v/validate-media-type! media)
|
|
(media.v/validate-media-size! media)
|
|
|
|
(db/run! cfg files/check-edition-permissions! profile-id file-id)
|
|
(when-let [file (files/get-minimal-file cfg file-id {::db/check-deleted false})]
|
|
(create-file-object-thumbnail! cfg file object-id media (or tag "frame"))))
|
|
|
|
;; --- MUTATION COMMAND: delete-file-object-thumbnail
|
|
|
|
(defn- delete-file-object-thumbnail!
|
|
[{:keys [::db/conn] :as cfg} file-id object-id]
|
|
(when-let [{:keys [media-id tag]} (db/get* conn :file-tagged-object-thumbnail
|
|
{:file-id file-id
|
|
:object-id object-id}
|
|
{::sql/for-update true})]
|
|
(let [storage (sto/resolve cfg ::db/reuse-conn true)]
|
|
(sto/touch-object! storage media-id)
|
|
(db/update! conn :file-tagged-object-thumbnail
|
|
{:deleted-at (ct/now)}
|
|
{:file-id file-id
|
|
:object-id object-id
|
|
:tag tag}))))
|
|
|
|
(defn- delete-file-object-thumbnails!
|
|
"Soft-deletes multiple object thumbnails in a single UPDATE statement
|
|
with RETURNING, then touches all returned media objects."
|
|
[{:keys [::db/conn] :as cfg} object-ids]
|
|
(let [storage (sto/resolve cfg ::db/reuse-conn true)
|
|
ids (db/create-array conn "text" (seq object-ids))
|
|
sql (str/concat
|
|
"UPDATE file_tagged_object_thumbnail"
|
|
" SET deleted_at = now()"
|
|
" WHERE object_id = ANY(?)"
|
|
" AND deleted_at IS NULL"
|
|
" RETURNING media_id")
|
|
rows (db/exec! conn [sql ids])]
|
|
(doseq [{:keys [media-id]} rows]
|
|
(sto/touch-object! storage media-id))))
|
|
|
|
(def ^:private schema:delete-file-object-thumbnail
|
|
[:map {:title "delete-file-object-thumbnail"}
|
|
[:file-id ::sm/uuid]
|
|
[:object-id [:string {:max 250}]]])
|
|
|
|
(def ^:private schema:delete-file-object-thumbnails
|
|
[:map {:title "delete-file-object-thumbnails"}
|
|
[:object-ids [:vector {:max 200} [:string {:max 250}]]]])
|
|
|
|
(sv/defmethod ::delete-file-object-thumbnail
|
|
{::doc/added "1.19"
|
|
::doc/module :files
|
|
::sm/params schema:delete-file-object-thumbnail
|
|
::audit/skip true}
|
|
[cfg {:keys [::rpc/profile-id file-id object-id]}]
|
|
(files/check-edition-permissions! cfg profile-id file-id)
|
|
(db/tx-run! cfg (fn [cfg]
|
|
(delete-file-object-thumbnail! cfg file-id object-id)
|
|
nil)))
|
|
|
|
(sv/defmethod ::delete-file-object-thumbnails
|
|
{::doc/added "1.19"
|
|
::doc/module :files
|
|
::climit/id [[:file-thumbnail-ops/by-profile ::rpc/profile-id]
|
|
[:file-thumbnail-ops/global]]
|
|
::sm/params schema:delete-file-object-thumbnails
|
|
::audit/skip true}
|
|
[cfg {:keys [::rpc/profile-id object-ids]}]
|
|
(when (seq object-ids)
|
|
;; Extract unique file-ids from object-ids for permission checks
|
|
(let [file-ids (->> object-ids
|
|
(map thc/get-file-id)
|
|
(into #{}))]
|
|
;; Check permissions for each unique file using a single connection
|
|
(db/run! cfg (fn [{:keys [::db/conn]}]
|
|
(doseq [file-id file-ids]
|
|
(files/check-edition-permissions! conn profile-id file-id))))
|
|
;; Delete all matching thumbnails in one transaction
|
|
(db/tx-run! cfg delete-file-object-thumbnails! object-ids))))
|
|
|
|
;; --- MUTATION COMMAND: create-file-thumbnail
|
|
|
|
(def ^:private
|
|
schema:create-file-thumbnail
|
|
[:map {:title "create-file-thumbnail"}
|
|
[:file-id ::sm/uuid]
|
|
[:revn ::sm/int]
|
|
[:media media.v/schema:upload]])
|
|
|
|
(sv/defmethod ::create-file-thumbnail
|
|
"Creates or updates the file thumbnail. Mainly used for paint the
|
|
grid thumbnails."
|
|
{::doc/added "1.19"
|
|
::doc/module :files
|
|
::audit/skip true
|
|
::climit/id [[:file-thumbnail-ops/by-profile ::rpc/profile-id]
|
|
[:file-thumbnail-ops/global]]
|
|
::rtry/enabled true
|
|
::rtry/when rtry/conflict-exception?
|
|
::sm/params schema:create-file-thumbnail}
|
|
|
|
[cfg {:keys [::rpc/profile-id file-id] :as params}]
|
|
(media.v/validate-media-type! (:media params))
|
|
(media.v/validate-media-size! (:media params))
|
|
|
|
(db/run! cfg files/check-edition-permissions! profile-id file-id)
|
|
|
|
(when-not (db/read-only? (::db/pool cfg))
|
|
(let [storage (::sto/storage cfg)
|
|
file (bfc/get-file cfg file-id :include-deleted? true :load-data? false)
|
|
props (db/tjson (or (:props params) {}))
|
|
{:keys [path mtype]} (:media params)
|
|
hash (sto/calculate-hash path)
|
|
data (-> (sto/content path)
|
|
(sto/wrap-with-hash hash))
|
|
tnow (ct/now)
|
|
|
|
media (sto/put-object! storage
|
|
{::sto/content data
|
|
::sto/deduplicate? true
|
|
::sto/touched-at tnow
|
|
:content-type mtype
|
|
:bucket "file-thumbnail"})
|
|
|
|
revn (:revn params)
|
|
|
|
result (db/tx-run! cfg
|
|
(fn [{:keys [::db/conn]}]
|
|
(let [thumb (db/get* conn :file-thumbnail
|
|
{:file-id file-id :revn revn}
|
|
{::db/remove-deleted false
|
|
::sql/for-update true})]
|
|
(if (some? thumb)
|
|
(do
|
|
(when (not= (:id media) (:media-id thumb))
|
|
(sto/touch-object! storage (:media-id thumb)))
|
|
(db/update! conn :file-thumbnail
|
|
{:media-id (:id media)
|
|
:deleted-at (:deleted-at file)
|
|
:updated-at tnow
|
|
:props props}
|
|
{:file-id file-id :revn revn}))
|
|
(db/insert! conn :file-thumbnail
|
|
{:file-id file-id
|
|
:revn revn
|
|
:created-at tnow
|
|
:updated-at tnow
|
|
:deleted-at (:deleted-at file)
|
|
:props props
|
|
:media-id (:id media)}))
|
|
media)))]
|
|
|
|
(when result
|
|
{:uri (files/resolve-public-uri (:id result))
|
|
:id (:id result)}))))
|