mirror of
https://github.com/penpot/penpot.git
synced 2026-11-21 17:50:38 +00:00
* ♻️ Rework audit-log consumers and jobs substrate Fold the implementation, follow-up fixes, migrations and tests into one code commit. AI-assisted-by: gpt-6-luna * 📚 Consolidate 2.20.0 upgrade runbooks Keep the operational upgrade procedure and its supporting runbooks in a separate documentation commit. AI-assisted-by: gpt-6-luna * ♻️ Remove the audit-log flag and derive persistence at boot Persistence now follows the consumer flags instead of a separate switch. app.config computes cf/audit-persist? once at boot from :nexus and :admin-console, and every write site reads that value, so request paths no longer check flags per event. With no consumer enabled no full rows are written; telemetry shadows and webhook deliveries keep their own flags. The frontend wire signal stays :audit-log as a derived value. Update the 2.20 runbooks, memories and the devenv flags to match, and migrate the tests that used to toggle :audit-log directly. AI-assisted-by: hy4-flash-free * 🐛 Accept error-reporting batches and precompute sink flags push-audit-events now accepts crash batches with :error-reporting alone, so the frontend signal from get-environment-data matches a real destination. The reporters receive unhandled-exception events without storing any row. All per-event sink checks read booleans computed once at boot: webhooks, telemetry and error reporting join audit-persist? in app.config. The test fixture and with-audit-config rebind all four values together, and the tests that toggled those flags directly were migrated. Address F2 of the review of the previous commit. F1 stays deferred to a separate plan. AI-assisted-by: hy4-flash-free * 💄 Match the layout cljfmt expects in the flags set The first comment of the varia set sat on its own line after the opening delimiter, and cljfmt joins such a comment to the delimiter line, so the Common Tests lint step failed on common/src/app/common/flags.cljc. Keep the comment on the opening line, the same shape the login and email sets already use. Note in mem:scripts/ci that the backend lint lints ../common/src but does not format-check it, so common has to be passed to scripts/ci as well. AI-assisted-by: deepseek-v4-flash
287 lines
11 KiB
Clojure
287 lines
11 KiB
Clojure
;; This Source Code Form is subject to the terms of the Mozilla Public
|
|
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
;;
|
|
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
|
|
|
(ns app.tasks.delete-object
|
|
"A generic task for object deletion cascade handling"
|
|
(:require
|
|
[app.common.data :as d]
|
|
[app.common.logging :as l]
|
|
[app.common.schema :as sm]
|
|
[app.common.time :as ct]
|
|
[app.db :as db]
|
|
[app.db.sql :as-alias sql]
|
|
[app.jobs :as jobs]
|
|
[app.loggers.audit :as audit]
|
|
[app.rpc.commands.files :as files]
|
|
[app.rpc.commands.profile :as profile]
|
|
[app.storage :as sto]
|
|
[integrant.core :as ig]))
|
|
|
|
(def ^:dynamic *team-deletion* false)
|
|
|
|
(def ^:dynamic *deleting-file-ids*
|
|
"Ids of the files the running cascade is going to delete. Bound by
|
|
the `:project` method with its own files so `:file` only absorbs the
|
|
dependents that survive: absorbing into files that die in the same
|
|
run is wasted work. Team deletions need no such set: they bind
|
|
`*team-deletion*` instead, which skips absorb entirely."
|
|
nil)
|
|
|
|
(def ^:dynamic *cascade*
|
|
"Accumulates the ids marked deleted by the current cascade execution,
|
|
grouped by entity. Bound by the task entry point; nil on direct calls
|
|
(e.g. objects-gc, whose purges are announced by their own procs)."
|
|
nil)
|
|
|
|
(defn- record
|
|
[kind id]
|
|
(when (some? *cascade*)
|
|
(swap! *cascade* update kind conj id)))
|
|
|
|
(defn- top-exists?
|
|
"The cascade marks blindly, so an unknown top id would report phantom
|
|
rows. Every queue submission and SREPL call names a real row; only
|
|
check, never change behavior: a missing row just skips the run."
|
|
[{:keys [::db/conn]} {:keys [object id]}]
|
|
(let [object (when (some? object) (keyword (name object)))]
|
|
(if (contains? #{:team :project :file :profile} object)
|
|
(some? (db/get* conn object {:id id}
|
|
{::db/remove-deleted false
|
|
::sql/columns [:id]}))
|
|
true)))
|
|
|
|
(defmulti delete-object
|
|
(fn [_ props] (:object props)))
|
|
|
|
(defmethod delete-object :snapshot
|
|
[{:keys [::db/conn] :as cfg} {:keys [id file-id deleted-at]}]
|
|
(l/trc :obj "snapshot" :id (str id) :file-id (str file-id)
|
|
:deleted-at (ct/format-inst deleted-at))
|
|
|
|
(db/update! conn :file-change
|
|
{:deleted-at deleted-at
|
|
:updated-at (ct/now)}
|
|
{:id id :file-id file-id}
|
|
{::db/return-keys false})
|
|
|
|
(db/update! conn :file-data
|
|
{:deleted-at deleted-at}
|
|
{:id id :file-id file-id :type "snapshot"}
|
|
{::db/return-keys false}))
|
|
|
|
(defmethod delete-object :file
|
|
[{:keys [::db/conn] :as cfg} {:keys [id deleted-at file-ids]}]
|
|
(when-let [file (db/get* conn :file {:id id}
|
|
{::db/remove-deleted false
|
|
::sql/columns [:id :is-shared]})]
|
|
|
|
(l/trc :obj "file" :id (str id)
|
|
:deleted-at (ct/format-inst deleted-at))
|
|
|
|
(db/update! conn :file
|
|
{:deleted-at deleted-at
|
|
:is-shared false}
|
|
{:id id}
|
|
{::db/return-keys false})
|
|
|
|
(when (and (:is-shared file)
|
|
;; Team deletions skip absorb entirely: every file dies
|
|
;; in the same cascade, so no survivor can absorb.
|
|
(not *team-deletion*))
|
|
;; NOTE: we don't prevent file deletion on absorb operation failure.
|
|
;; Only live dependents outside this cascade matter: if the whole
|
|
;; closure dies together there is nothing to absorb and the
|
|
;; relations die with their files (both FKs are ON DELETE CASCADE).
|
|
;; DEPRECATED `:file-ids`: jobs queued by the previous release
|
|
;; already deleted the relations in the RPC, so the live lookup
|
|
;; finds nothing. Trust their snapshot for one release; new jobs
|
|
;; never carry the key.
|
|
(let [doomed (or *deleting-file-ids* #{})
|
|
dependents (if (seq file-ids)
|
|
(into [] (remove doomed) file-ids)
|
|
(into [] (remove doomed)
|
|
(files/get-referenced-files conn id)))]
|
|
(when (seq dependents)
|
|
;; The relations are dropped unconditionally, before the absorb
|
|
;; runs: they are dead weight once the library is being
|
|
;; deleted, regardless of whether the absorb succeeds. A
|
|
;; failure here fails the job instead of being swallowed,
|
|
;; like every other write in the cascade.
|
|
(db/delete! conn :file-library-rel {:library-file-id id})
|
|
;; Runs in the caller transaction: a nested transaction here
|
|
;; would join it anyway, so the call goes directly. The
|
|
;; savepoint contains an absorb failure: it rolls back the
|
|
;; whole absorb and the cascade goes on.
|
|
;; Nothing retries the absorb itself: the job succeeds and the
|
|
;; dependents keep the data they had before the absorb ran.
|
|
(let [sp (db/savepoint conn "absorb-library")]
|
|
(try
|
|
(files/absorb-library cfg id :file-ids dependents)
|
|
(catch Throwable cause
|
|
(db/rollback! conn sp)
|
|
(l/warn :hint "error on absorbing library"
|
|
:file-id id
|
|
:cause cause))
|
|
(finally
|
|
(db/release! conn sp)))))))
|
|
|
|
;; Mark file change to be deleted, but only the rows that do not carry a
|
|
;; date yet. A revision is born already scheduled for deletion one hour
|
|
;; later, and rewriting its date here would push it out to the trash
|
|
;; lifetime; the collector would still take it, just much later than it
|
|
;; should.
|
|
(db/update! conn :file-change
|
|
{:deleted-at deleted-at
|
|
:updated-at (ct/now)}
|
|
{:file-id id :deleted-at nil}
|
|
{::db/return-keys false})
|
|
|
|
;; Mark file data fragment to be deleted
|
|
(db/update! conn :file-data
|
|
{:deleted-at deleted-at}
|
|
{:file-id id}
|
|
{::db/return-keys false})
|
|
|
|
;; Mark file media objects to be deleted
|
|
(db/update! conn :file-media-object
|
|
{:deleted-at deleted-at}
|
|
{:file-id id}
|
|
{::db/return-keys false})
|
|
|
|
;; Mark thumbnails to be deleted
|
|
(db/update! conn :file-thumbnail
|
|
{:deleted-at deleted-at}
|
|
{:file-id id}
|
|
{::db/return-keys false})
|
|
|
|
(db/update! conn :file-tagged-object-thumbnail
|
|
{:deleted-at deleted-at}
|
|
{:file-id id}
|
|
{::db/return-keys false})
|
|
|
|
(record :files id)))
|
|
|
|
(defmethod delete-object :project
|
|
[{:keys [::db/conn] :as cfg} {:keys [id deleted-at]}]
|
|
(l/trc :obj "project" :id (str id)
|
|
:deleted-at (ct/format-inst deleted-at))
|
|
|
|
(db/update! conn :project
|
|
{:deleted-at deleted-at}
|
|
{:id id}
|
|
{::db/return-keys false})
|
|
|
|
(let [files (db/query conn :file
|
|
{:project-id id}
|
|
{::db/columns [:id :deleted-at]})]
|
|
;; Every file below dies in this run: absorbing a library into
|
|
;; them is wasted work, so the files only absorb survivors.
|
|
(binding [*deleting-file-ids* (into #{} d/xf:map-id files)]
|
|
(doseq [file files]
|
|
(delete-object cfg (assoc file
|
|
:object :file
|
|
:deleted-at deleted-at)))))
|
|
|
|
(record :projects id))
|
|
|
|
(defmethod delete-object :team
|
|
[{:keys [::db/conn] :as cfg} {:keys [id deleted-at]}]
|
|
(l/trc :obj "team" :id (str id)
|
|
:deleted-at (ct/format-inst deleted-at))
|
|
(db/update! conn :team
|
|
{:deleted-at deleted-at}
|
|
{:id id}
|
|
{::db/return-keys false})
|
|
|
|
(db/update! conn :team-font-variant
|
|
{:deleted-at deleted-at}
|
|
{:team-id id}
|
|
{::db/return-keys false})
|
|
|
|
(binding [*team-deletion* true]
|
|
(doseq [project (db/query conn :project
|
|
{:team-id id}
|
|
{::db/columns [:id :deleted-at]})]
|
|
(delete-object cfg (assoc project
|
|
:object :project
|
|
:deleted-at deleted-at))))
|
|
|
|
(record :teams id))
|
|
|
|
(defmethod delete-object :profile
|
|
[{:keys [::db/conn] :as cfg} {:keys [id deleted-at]}]
|
|
(l/trc :obj "profile" :id (str id)
|
|
:deleted-at (ct/format-inst deleted-at))
|
|
|
|
(db/update! conn :profile
|
|
{:deleted-at deleted-at}
|
|
{:id id}
|
|
{::db/return-keys false})
|
|
|
|
(doseq [team (profile/get-owned-teams conn id)]
|
|
(jobs/heartbeat cfg)
|
|
(delete-object cfg (assoc team
|
|
:object :team
|
|
:deleted-at deleted-at))))
|
|
|
|
(defmethod delete-object :default
|
|
[_cfg props]
|
|
(l/wrn :obj (:object props) :hint "not implementation found"))
|
|
|
|
(def schema:delete-object-params
|
|
[:map
|
|
[:object [:enum :snapshot :team :project :profile :file]]
|
|
[:deleted-at ::ct/inst]
|
|
[:id ::sm/uuid]
|
|
[:file-id {:optional true} ::sm/uuid]
|
|
;; DEPRECATED: only jobs queued before the cascade owned the
|
|
;; relations carry it (their RPC already deleted them). Remove once
|
|
;; no such rows can be left.
|
|
[:file-ids {:optional true} [:vector ::sm/uuid]]])
|
|
|
|
(defn execute-delete-object
|
|
"Plain job handler: run the delete-object multimethod on the provided
|
|
params, tracking the cascade and reporting it as an audit event when
|
|
anything was marked. Runs inside the caller transaction (the job-def
|
|
handler opens it with `tx-run!`)."
|
|
[cfg params]
|
|
(when (top-exists? cfg params)
|
|
(binding [*cascade* (atom {:teams [] :projects [] :files []})]
|
|
(delete-object cfg params)
|
|
(let [{:keys [teams projects files]} @*cascade*]
|
|
(when (or (seq teams) (seq projects) (seq files))
|
|
;; Flat team ids, so a consumer can key on them without
|
|
;; joining production tables.
|
|
(let [team-ids (audit/get-team-ids cfg
|
|
{:team-ids teams
|
|
:project-ids projects
|
|
:file-ids files})]
|
|
(audit/submit cfg {:name "delete-object-with-cascade"
|
|
:type "action"
|
|
:props {:object (:object params)
|
|
:id (:id params)
|
|
:deleted-at (:deleted-at params)
|
|
:team-ids (vec team-ids)
|
|
:cascade {:team-ids teams
|
|
:project-ids projects
|
|
:file-ids files}}
|
|
:context {:triggered-by "delete-object"}})))))))
|
|
|
|
(defmethod ig/assert-key ::job-def
|
|
[_ params]
|
|
(assert (db/pool? (::db/pool params)) "expected a valid database pool")
|
|
(assert (sto/valid-storage? (::sto/storage params)) "expected valid storage to be provided"))
|
|
|
|
(defmethod ig/init-key ::job-def
|
|
[_ cfg]
|
|
{::jobs/name :delete-object
|
|
::jobs/schema schema:delete-object-params
|
|
::jobs/handler
|
|
(fn [_context params]
|
|
(db/tx-run! cfg execute-delete-object params))
|
|
::jobs/decoder (sm/decoder schema:delete-object-params sm/json-transformer)
|
|
::jobs/validator (sm/validator schema:delete-object-params)})
|