mirror of
https://github.com/penpot/penpot.git
synced 2026-11-22 18:30:34 +00:00
* ♻️ Rework audit-log consumers and jobs substrate Fold the implementation, follow-up fixes, migrations and tests into one code commit. AI-assisted-by: gpt-6-luna * 📚 Consolidate 2.20.0 upgrade runbooks Keep the operational upgrade procedure and its supporting runbooks in a separate documentation commit. AI-assisted-by: gpt-6-luna * ♻️ Remove the audit-log flag and derive persistence at boot Persistence now follows the consumer flags instead of a separate switch. app.config computes cf/audit-persist? once at boot from :nexus and :admin-console, and every write site reads that value, so request paths no longer check flags per event. With no consumer enabled no full rows are written; telemetry shadows and webhook deliveries keep their own flags. The frontend wire signal stays :audit-log as a derived value. Update the 2.20 runbooks, memories and the devenv flags to match, and migrate the tests that used to toggle :audit-log directly. AI-assisted-by: hy4-flash-free * 🐛 Accept error-reporting batches and precompute sink flags push-audit-events now accepts crash batches with :error-reporting alone, so the frontend signal from get-environment-data matches a real destination. The reporters receive unhandled-exception events without storing any row. All per-event sink checks read booleans computed once at boot: webhooks, telemetry and error reporting join audit-persist? in app.config. The test fixture and with-audit-config rebind all four values together, and the tests that toggled those flags directly were migrated. Address F2 of the review of the previous commit. F1 stays deferred to a separate plan. AI-assisted-by: hy4-flash-free * 💄 Match the layout cljfmt expects in the flags set The first comment of the varia set sat on its own line after the opening delimiter, and cljfmt joins such a comment to the delimiter line, so the Common Tests lint step failed on common/src/app/common/flags.cljc. Keep the comment on the opening line, the same shape the login and email sets already use. Note in mem:scripts/ci that the backend lint lints ../common/src but does not format-check it, so common has to be passed to scripts/ci as well. AI-assisted-by: deepseek-v4-flash
342 lines
16 KiB
Clojure
342 lines
16 KiB
Clojure
;; This Source Code Form is subject to the terms of the Mozilla Public
|
|
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
;;
|
|
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
|
|
|
(ns backend-tests.objects-gc-test
|
|
(:require
|
|
[app.common.logging :as l]
|
|
[app.common.time :as ct]
|
|
[app.common.uuid :as uuid]
|
|
[app.config :as cf]
|
|
[app.db :as db]
|
|
[app.features.file-changes :as fchanges]
|
|
[app.rpc :as-alias rpc]
|
|
[app.storage :as sto]
|
|
[app.tasks.objects-gc :as ogc]
|
|
[backend-tests.helpers :as th]
|
|
[clojure.test :as t]
|
|
[mockery.core :refer [with-mocks]]))
|
|
|
|
(t/use-fixtures :once th/state-init)
|
|
(t/use-fixtures :each th/database-reset)
|
|
|
|
(defn- create-deleted-file!
|
|
"Creates a file and applies the deletion cascade on it (the same task the RPC
|
|
uses), so its children are marked as deleted for the given date. Returns the
|
|
file."
|
|
[profile i deleted-at]
|
|
(let [file (th/create-file* i {:profile-id (:id profile)
|
|
:project-id (:default-project-id profile)})]
|
|
(th/run-task! :delete-object {:object :file
|
|
:deleted-at deleted-at
|
|
:id (:id file)})
|
|
file))
|
|
|
|
(defn- block-file!
|
|
"Leaves an unmarked `file_media_object` pointing at the file, which is what
|
|
happens in production: a child that the deletion cascade did not mark. Its
|
|
foreign key towards `file` is `NO ACTION`, so it blocks the file deletion."
|
|
[file]
|
|
(let [media (sto/put-object! (::sto/storage th/*system*)
|
|
{::sto/content (sto/content "media")
|
|
:content-type "text/plain"
|
|
:bucket "file-media-object"})]
|
|
(th/create-file-media-object* {:file-id (:id file)
|
|
:media-id (:id media)})))
|
|
|
|
(defn- find-file
|
|
[id]
|
|
(th/db-get :file {:id id} {::db/remove-deleted false}))
|
|
|
|
(defn- chunk-error
|
|
"The first captured log record that reports a skipped chunk for `obj`,
|
|
waiting for it because logging is asynchronous."
|
|
[records obj]
|
|
(let [deadline (+ (System/currentTimeMillis) 5000)]
|
|
(loop []
|
|
(let [record (first (filter (fn [record]
|
|
(let [props (::l/props record)]
|
|
(and (= obj (get props :obj))
|
|
(= "error on chunk deletion, skipping it"
|
|
(get props :hint)))))
|
|
@records))]
|
|
(cond
|
|
record record
|
|
(> (System/currentTimeMillis) deadline) nil
|
|
:else (do (Thread/sleep 10) (recur)))))))
|
|
|
|
(t/deftest deletes-the-pending-files
|
|
(let [profile (th/create-profile* 1)
|
|
file (create-deleted-file! profile 1 (ct/in-past "10m"))
|
|
result (th/run-task! :objects-gc {})]
|
|
|
|
(t/is (pos? (:processed result)))
|
|
(t/is (nil? (find-file (:id file))))))
|
|
|
|
(t/deftest deletes-the-pending-file-changes
|
|
;; The pending file changes are locked per file, with a constant `file_id`,
|
|
;; so the rows of a file that is pending deletion are gone after the run.
|
|
(let [profile (th/create-profile* 1)
|
|
file (th/create-file* 1 {:profile-id (:id profile)
|
|
:project-id (:default-project-id profile)})
|
|
|
|
_ (th/db-insert! :file-change
|
|
{:file-id (:id file)
|
|
:kind fchanges/revision-kind
|
|
:deleted-at (ct/in-past "10m")})
|
|
|
|
result (th/run-task! :objects-gc {})]
|
|
|
|
(t/is (pos? (:processed result)))
|
|
(t/is (nil? (th/db-get :file-change {:file-id (:id file)}
|
|
{::db/remove-deleted false})))))
|
|
|
|
(t/deftest deletes-the-pending-file-changes-of-several-chunks
|
|
;; More pending rows than the chunk size: the plan walks the cursor and each
|
|
;; chunk locks its own rows, one query per file, with a constant `file_id`
|
|
;; so the planner prunes the partition.
|
|
(let [profile (th/create-profile* 1)
|
|
file (th/create-file* 1 {:profile-id (:id profile)
|
|
:project-id (:default-project-id profile)})
|
|
ids (repeatedly 5 uuid/next)
|
|
_ (doseq [id ids]
|
|
(th/db-insert! :file-change
|
|
{:id id
|
|
:file-id (:id file)
|
|
:kind fchanges/revision-kind
|
|
:deleted-at (ct/in-past "10m")}))
|
|
result (th/run-task! :objects-gc {:chunk-size 2})]
|
|
|
|
(t/is (pos? (:processed result)))
|
|
(t/is (empty? (th/db-query :file-change {:file-id (:id file)}
|
|
{::db/remove-deleted false})))))
|
|
|
|
(t/deftest a-chunk-spanning-several-files-deletes-all-of-them
|
|
;; The rows of one chunk can belong to different files. Each group is locked
|
|
;; with its own `file_id`, and every row is still deleted; buffering the
|
|
;; groups instead of locking them would leave them behind.
|
|
(let [profile (th/create-profile* 1)
|
|
opts {:profile-id (:id profile)
|
|
:project-id (:default-project-id profile)}
|
|
files [(th/create-file* 1 opts)
|
|
(th/create-file* 2 opts)]
|
|
|
|
_ (doseq [file files]
|
|
(dotimes [_ 2]
|
|
(th/db-insert! :file-change
|
|
{:file-id (:id file)
|
|
:kind fchanges/revision-kind
|
|
:deleted-at (ct/in-past "10m")})))
|
|
|
|
;; one chunk holds the rows of both files
|
|
result (th/run-task! :objects-gc {:chunk-size 4})]
|
|
|
|
(t/is (pos? (:processed result)))
|
|
(doseq [file files]
|
|
(t/is (empty? (th/db-query :file-change {:file-id (:id file)}
|
|
{::db/remove-deleted false}))))))
|
|
|
|
(t/deftest profile-cascade-keeps-profile-deletion-date
|
|
;; The cascade marks the owned rows with the deletion date of the profile,
|
|
;; like the RPC delete-profile flow does, not with the timestamp of the run.
|
|
;; The team survives because only the profiles chunk runs here, so its row
|
|
;; still shows that date.
|
|
(let [profile (th/create-profile* 1)
|
|
_ (th/db-update! :profile {:deleted-at (ct/in-past "30m")} {:id (:id profile)})
|
|
|
|
deleted-at (:deleted-at (th/db-get :profile {:id (:id profile)}
|
|
{::db/remove-deleted false}))
|
|
|
|
cfg {::db/pool th/*pool*
|
|
::sto/storage (::sto/storage th/*system*)
|
|
::ogc/threshold deleted-at}
|
|
total (#'ogc/delete-profiles-chunk cfg [(:id profile)])]
|
|
|
|
(t/testing "the profile is gone and the chunk reports it"
|
|
(t/is (= 1 total))
|
|
(t/is (nil? (th/db-get :profile {:id (:id profile)} {::db/remove-deleted false}))))
|
|
|
|
(t/testing "the owned team keeps the deletion date of the profile"
|
|
(t/is (= deleted-at (:deleted-at (th/db-get :team {:id (:default-team-id profile)}
|
|
{::db/remove-deleted false})))))))
|
|
|
|
(t/deftest blocked-chunk-does-not-stop-the-run
|
|
;; Chunks of one file: the blocked file takes down its own chunk and nothing
|
|
;; else, so the clean file of the next chunk is still deleted and the run
|
|
;; finishes. The next run retries the blocked file.
|
|
(let [profile (th/create-profile* 1)
|
|
blocked (create-deleted-file! profile 1 (ct/in-past "30m"))
|
|
clean (create-deleted-file! profile 2 (ct/in-past "20m"))
|
|
|
|
_ (block-file! blocked)
|
|
|
|
records (atom [])
|
|
_ (add-watch l/log-record ::capture
|
|
(fn [_ _ _ record] (swap! records conj record)))
|
|
result (try
|
|
(th/run-task! :objects-gc {:chunk-size 1})
|
|
(finally
|
|
(remove-watch l/log-record ::capture)))]
|
|
|
|
(t/testing "the clean file of the next chunk is deleted"
|
|
(t/is (nil? (find-file (:id clean)))))
|
|
|
|
(t/testing "the blocked file is still pending"
|
|
(t/is (some? (find-file (:id blocked)))))
|
|
|
|
(t/testing "the failed chunk is reported"
|
|
(t/is (some? (chunk-error records "file"))))
|
|
|
|
(t/testing "the run finishes"
|
|
(t/is (pos? (:processed result))))))
|
|
|
|
(defn- submitted-events
|
|
[audit-mock]
|
|
(->> (:call-args-list @audit-mock)
|
|
(map second)))
|
|
|
|
(defn- events-named
|
|
[audit-mock event-name]
|
|
(filter #(= event-name (:name %))
|
|
(submitted-events audit-mock)))
|
|
|
|
(t/deftest gc-emits-hard-delete-objects-for-teams
|
|
;; When the GC physically deletes a team, the projection must drop its row,
|
|
;; so the event carries the purged ids.
|
|
(with-mocks [audit-mock {:target 'app.loggers.audit/submit :return nil}]
|
|
(th/with-audit-config (conj cf/flags :nexus)
|
|
(let [profile (th/create-profile* 1 {:is-active true})
|
|
team (th/create-team* 71 {:profile-id (:id profile)})]
|
|
(th/command! {::th/type :delete-team
|
|
::rpc/profile-id (:id profile)
|
|
:id (:id team)})
|
|
;; Run the queued cascade so the default project is marked deleted too,
|
|
;; then purge everything the way production does.
|
|
(th/run-pending-jobs)
|
|
(th/reset-mock! audit-mock)
|
|
(th/run-task! :objects-gc {:skip-delay true})
|
|
(t/is (nil? (th/db-get :team {:id (:id team)} {::db/remove-deleted false})))
|
|
(let [events (filter #(= #{:team-ids} (set (keys (:props %))))
|
|
(events-named audit-mock "hard-delete-objects"))]
|
|
;; Every chunk carries exactly its own entity key and nothing
|
|
;; else: no flat team-ids are resolved after the delete.
|
|
(t/is (= 1 (count events)))
|
|
(let [event (first events)]
|
|
(t/is (= "command" (:type event)))
|
|
(t/is (= [(:id team)] (get-in event [:props :team-ids])))
|
|
(t/is (= "objects-gc" (get-in event [:context :triggered-by])))))))))
|
|
|
|
(t/deftest gc-emits-hard-delete-objects-for-projects
|
|
(with-mocks [audit-mock {:target 'app.loggers.audit/submit :return nil}]
|
|
(th/with-audit-config (conj cf/flags :nexus)
|
|
(let [profile (th/create-profile* 1 {:is-active true})
|
|
team (th/create-team* 72 {:profile-id (:id profile)})
|
|
project (th/create-project* 72 {:profile-id (:id profile)
|
|
:team-id (:id team)})]
|
|
(th/command! {::th/type :delete-project
|
|
::rpc/profile-id (:id profile)
|
|
:id (:id project)})
|
|
(th/run-pending-jobs)
|
|
(th/reset-mock! audit-mock)
|
|
(th/run-task! :objects-gc {:skip-delay true})
|
|
(t/is (nil? (th/db-get :project {:id (:id project)} {::db/remove-deleted false})))
|
|
(let [events (filter #(contains? (:props %) :project-ids)
|
|
(events-named audit-mock "hard-delete-objects"))]
|
|
(t/is (= 1 (count events)))
|
|
(let [event (first events)]
|
|
(t/is (= "command" (:type event)))
|
|
(t/is (= #{:project-ids} (set (keys (:props event)))))
|
|
(t/is (= [(:id project)] (get-in event [:props :project-ids])))
|
|
(t/is (= "objects-gc" (get-in event [:context :triggered-by])))))))))
|
|
|
|
(t/deftest gc-emits-hard-delete-objects-for-files
|
|
(with-mocks [audit-mock {:target 'app.loggers.audit/submit :return nil}]
|
|
(th/with-audit-config (conj cf/flags :nexus)
|
|
(let [profile (th/create-profile* 1 {:is-active true})
|
|
team (th/create-team* 73 {:profile-id (:id profile)})
|
|
project (th/create-project* 73 {:profile-id (:id profile)
|
|
:team-id (:id team)})
|
|
file (th/create-file* 73 {:profile-id (:id profile)
|
|
:project-id (:id project)})]
|
|
(th/command! {::th/type :delete-file
|
|
::rpc/profile-id (:id profile)
|
|
:id (:id file)})
|
|
(th/run-pending-jobs)
|
|
(th/reset-mock! audit-mock)
|
|
(th/run-task! :objects-gc {:skip-delay true})
|
|
(t/is (nil? (th/db-get :file {:id (:id file)} {::db/remove-deleted false})))
|
|
(let [events (filter #(contains? (:props %) :file-ids)
|
|
(events-named audit-mock "hard-delete-objects"))]
|
|
(t/is (= 1 (count events)))
|
|
(let [event (first events)]
|
|
(t/is (= "command" (:type event)))
|
|
(t/is (= #{:file-ids} (set (keys (:props event)))))
|
|
(t/is (= [(:id file)] (get-in event [:props :file-ids])))
|
|
(t/is (= "objects-gc" (get-in event [:context :triggered-by])))))))))
|
|
|
|
(t/deftest gc-emits-no-event-without-a-consumer
|
|
;; The GC must stay silent when the audit log is off, even though the row is
|
|
;; still purged.
|
|
(with-mocks [audit-mock {:target 'app.loggers.audit/submit :return nil}]
|
|
(let [profile (th/create-profile* 1 {:is-active true})
|
|
team (th/create-team* 74 {:profile-id (:id profile)})]
|
|
(th/command! {::th/type :delete-team
|
|
::rpc/profile-id (:id profile)
|
|
:id (:id team)})
|
|
(th/run-pending-jobs)
|
|
(th/reset-mock! audit-mock)
|
|
(th/run-task! :objects-gc {:skip-delay true})
|
|
(t/is (nil? (th/db-get :team {:id (:id team)} {::db/remove-deleted false})))
|
|
(t/is (empty? (:call-args-list @audit-mock))))))
|
|
|
|
(t/deftest gc-deletes-file-data-and-touches-its-storage-object
|
|
(let [profile (th/create-profile* 1)
|
|
file (th/create-file* 1 {:profile-id (:id profile)
|
|
:project-id (:default-project-id profile)})
|
|
sobject (sto/put-object! (::sto/storage th/*system*)
|
|
{::sto/content (sto/content "file-data")
|
|
:content-type "application/octet-stream"
|
|
:bucket "file-data"
|
|
:file-id (:id file)
|
|
:id (:id file)})]
|
|
|
|
;; `main` always carries `id = file_id`: mark the real row as deleted
|
|
;; instead of inventing a second main row.
|
|
(th/db-update! :file-data
|
|
{:backend "storage"
|
|
:data nil
|
|
:storage-object-id (:id sobject)
|
|
:deleted-at (ct/in-past "30m")}
|
|
{:file-id (:id file) :id (:id file)})
|
|
|
|
(th/run-task! :objects-gc {})
|
|
|
|
(t/testing "the row is purged"
|
|
(t/is (nil? (th/db-get :file-data {:file-id (:id file) :id (:id file)}))))
|
|
|
|
(t/testing "the storage object is touched for the storage GC"
|
|
(t/is (some? (:touched-at (th/db-get :storage-object {:id (:id sobject)})))))))
|
|
|
|
(t/deftest gc-emits-events-only-for-committed-chunks
|
|
(with-mocks [audit-mock {:target 'app.loggers.audit/submit :return nil}]
|
|
(th/with-audit-config (conj cf/flags :nexus)
|
|
(let [profile (th/create-profile* 1)
|
|
rolled-back (create-deleted-file! profile 1 (ct/in-past "30m"))
|
|
blocked (create-deleted-file! profile 2 (ct/in-past "20m"))
|
|
clean (create-deleted-file! profile 3 (ct/in-past "10m"))]
|
|
(block-file! blocked)
|
|
(th/reset-mock! audit-mock)
|
|
(let [result (th/run-task! :objects-gc {:chunk-size 2})
|
|
events (filter #(contains? (:props %) :file-ids)
|
|
(events-named audit-mock "hard-delete-objects"))]
|
|
(t/testing "the failed chunk keeps both files"
|
|
(t/is (some? (find-file (:id rolled-back))))
|
|
(t/is (some? (find-file (:id blocked)))))
|
|
(t/testing "the next chunk commits and reports only its deleted file"
|
|
(t/is (nil? (find-file (:id clean))))
|
|
(t/is (pos? (:processed result)))
|
|
(t/is (= 1 (count events)))
|
|
(t/is (= [(:id clean)] (get-in (first events) [:props :file-ids])))))))))
|