penpot/backend/test/backend_tests/objects_gc_test.clj
Andrey Antukh 90a06e982b
♻️ Rework the audit-log consumers and the jobs substrate (#12701)
* ♻️ Rework audit-log consumers and jobs substrate

Fold the implementation, follow-up fixes, migrations and tests into one

code commit.

AI-assisted-by: gpt-6-luna

* 📚 Consolidate 2.20.0 upgrade runbooks

Keep the operational upgrade procedure and its supporting runbooks in a

separate documentation commit.

AI-assisted-by: gpt-6-luna

* ♻️ Remove the audit-log flag and derive persistence at boot

Persistence now follows the consumer flags instead of a separate switch.
app.config computes cf/audit-persist? once at boot from :nexus and
:admin-console, and every write site reads that value, so request paths
no longer check flags per event. With no consumer enabled no full rows
are written; telemetry shadows and webhook deliveries keep their own
flags. The frontend wire signal stays :audit-log as a derived value.

Update the 2.20 runbooks, memories and the devenv flags to match, and
migrate the tests that used to toggle :audit-log directly.

AI-assisted-by: hy4-flash-free

* 🐛 Accept error-reporting batches and precompute sink flags

push-audit-events now accepts crash batches with :error-reporting alone, so
the frontend signal from get-environment-data matches a real destination.
The reporters receive unhandled-exception events without storing any row.

All per-event sink checks read booleans computed once at boot: webhooks,
telemetry and error reporting join audit-persist? in app.config. The test
fixture and with-audit-config rebind all four values together, and the
tests that toggled those flags directly were migrated.

Address F2 of the review of the previous commit. F1 stays deferred to a
separate plan.

AI-assisted-by: hy4-flash-free

* 💄 Match the layout cljfmt expects in the flags set

The first comment of the varia set sat on its own line after the opening
delimiter, and cljfmt joins such a comment to the delimiter line, so the
Common Tests lint step failed on common/src/app/common/flags.cljc. Keep
the comment on the opening line, the same shape the login and email sets
already use.

Note in mem:scripts/ci that the backend lint lints ../common/src but does
not format-check it, so common has to be passed to scripts/ci as well.

AI-assisted-by: deepseek-v4-flash
2026-11-20 11:12:00 +01:00

342 lines
16 KiB
Clojure

;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; Copyright (c) KALEIDOS INC Sucursal en España SL
(ns backend-tests.objects-gc-test
(:require
[app.common.logging :as l]
[app.common.time :as ct]
[app.common.uuid :as uuid]
[app.config :as cf]
[app.db :as db]
[app.features.file-changes :as fchanges]
[app.rpc :as-alias rpc]
[app.storage :as sto]
[app.tasks.objects-gc :as ogc]
[backend-tests.helpers :as th]
[clojure.test :as t]
[mockery.core :refer [with-mocks]]))
(t/use-fixtures :once th/state-init)
(t/use-fixtures :each th/database-reset)
(defn- create-deleted-file!
"Creates a file and applies the deletion cascade on it (the same task the RPC
uses), so its children are marked as deleted for the given date. Returns the
file."
[profile i deleted-at]
(let [file (th/create-file* i {:profile-id (:id profile)
:project-id (:default-project-id profile)})]
(th/run-task! :delete-object {:object :file
:deleted-at deleted-at
:id (:id file)})
file))
(defn- block-file!
"Leaves an unmarked `file_media_object` pointing at the file, which is what
happens in production: a child that the deletion cascade did not mark. Its
foreign key towards `file` is `NO ACTION`, so it blocks the file deletion."
[file]
(let [media (sto/put-object! (::sto/storage th/*system*)
{::sto/content (sto/content "media")
:content-type "text/plain"
:bucket "file-media-object"})]
(th/create-file-media-object* {:file-id (:id file)
:media-id (:id media)})))
(defn- find-file
[id]
(th/db-get :file {:id id} {::db/remove-deleted false}))
(defn- chunk-error
"The first captured log record that reports a skipped chunk for `obj`,
waiting for it because logging is asynchronous."
[records obj]
(let [deadline (+ (System/currentTimeMillis) 5000)]
(loop []
(let [record (first (filter (fn [record]
(let [props (::l/props record)]
(and (= obj (get props :obj))
(= "error on chunk deletion, skipping it"
(get props :hint)))))
@records))]
(cond
record record
(> (System/currentTimeMillis) deadline) nil
:else (do (Thread/sleep 10) (recur)))))))
(t/deftest deletes-the-pending-files
(let [profile (th/create-profile* 1)
file (create-deleted-file! profile 1 (ct/in-past "10m"))
result (th/run-task! :objects-gc {})]
(t/is (pos? (:processed result)))
(t/is (nil? (find-file (:id file))))))
(t/deftest deletes-the-pending-file-changes
;; The pending file changes are locked per file, with a constant `file_id`,
;; so the rows of a file that is pending deletion are gone after the run.
(let [profile (th/create-profile* 1)
file (th/create-file* 1 {:profile-id (:id profile)
:project-id (:default-project-id profile)})
_ (th/db-insert! :file-change
{:file-id (:id file)
:kind fchanges/revision-kind
:deleted-at (ct/in-past "10m")})
result (th/run-task! :objects-gc {})]
(t/is (pos? (:processed result)))
(t/is (nil? (th/db-get :file-change {:file-id (:id file)}
{::db/remove-deleted false})))))
(t/deftest deletes-the-pending-file-changes-of-several-chunks
;; More pending rows than the chunk size: the plan walks the cursor and each
;; chunk locks its own rows, one query per file, with a constant `file_id`
;; so the planner prunes the partition.
(let [profile (th/create-profile* 1)
file (th/create-file* 1 {:profile-id (:id profile)
:project-id (:default-project-id profile)})
ids (repeatedly 5 uuid/next)
_ (doseq [id ids]
(th/db-insert! :file-change
{:id id
:file-id (:id file)
:kind fchanges/revision-kind
:deleted-at (ct/in-past "10m")}))
result (th/run-task! :objects-gc {:chunk-size 2})]
(t/is (pos? (:processed result)))
(t/is (empty? (th/db-query :file-change {:file-id (:id file)}
{::db/remove-deleted false})))))
(t/deftest a-chunk-spanning-several-files-deletes-all-of-them
;; The rows of one chunk can belong to different files. Each group is locked
;; with its own `file_id`, and every row is still deleted; buffering the
;; groups instead of locking them would leave them behind.
(let [profile (th/create-profile* 1)
opts {:profile-id (:id profile)
:project-id (:default-project-id profile)}
files [(th/create-file* 1 opts)
(th/create-file* 2 opts)]
_ (doseq [file files]
(dotimes [_ 2]
(th/db-insert! :file-change
{:file-id (:id file)
:kind fchanges/revision-kind
:deleted-at (ct/in-past "10m")})))
;; one chunk holds the rows of both files
result (th/run-task! :objects-gc {:chunk-size 4})]
(t/is (pos? (:processed result)))
(doseq [file files]
(t/is (empty? (th/db-query :file-change {:file-id (:id file)}
{::db/remove-deleted false}))))))
(t/deftest profile-cascade-keeps-profile-deletion-date
;; The cascade marks the owned rows with the deletion date of the profile,
;; like the RPC delete-profile flow does, not with the timestamp of the run.
;; The team survives because only the profiles chunk runs here, so its row
;; still shows that date.
(let [profile (th/create-profile* 1)
_ (th/db-update! :profile {:deleted-at (ct/in-past "30m")} {:id (:id profile)})
deleted-at (:deleted-at (th/db-get :profile {:id (:id profile)}
{::db/remove-deleted false}))
cfg {::db/pool th/*pool*
::sto/storage (::sto/storage th/*system*)
::ogc/threshold deleted-at}
total (#'ogc/delete-profiles-chunk cfg [(:id profile)])]
(t/testing "the profile is gone and the chunk reports it"
(t/is (= 1 total))
(t/is (nil? (th/db-get :profile {:id (:id profile)} {::db/remove-deleted false}))))
(t/testing "the owned team keeps the deletion date of the profile"
(t/is (= deleted-at (:deleted-at (th/db-get :team {:id (:default-team-id profile)}
{::db/remove-deleted false})))))))
(t/deftest blocked-chunk-does-not-stop-the-run
;; Chunks of one file: the blocked file takes down its own chunk and nothing
;; else, so the clean file of the next chunk is still deleted and the run
;; finishes. The next run retries the blocked file.
(let [profile (th/create-profile* 1)
blocked (create-deleted-file! profile 1 (ct/in-past "30m"))
clean (create-deleted-file! profile 2 (ct/in-past "20m"))
_ (block-file! blocked)
records (atom [])
_ (add-watch l/log-record ::capture
(fn [_ _ _ record] (swap! records conj record)))
result (try
(th/run-task! :objects-gc {:chunk-size 1})
(finally
(remove-watch l/log-record ::capture)))]
(t/testing "the clean file of the next chunk is deleted"
(t/is (nil? (find-file (:id clean)))))
(t/testing "the blocked file is still pending"
(t/is (some? (find-file (:id blocked)))))
(t/testing "the failed chunk is reported"
(t/is (some? (chunk-error records "file"))))
(t/testing "the run finishes"
(t/is (pos? (:processed result))))))
(defn- submitted-events
[audit-mock]
(->> (:call-args-list @audit-mock)
(map second)))
(defn- events-named
[audit-mock event-name]
(filter #(= event-name (:name %))
(submitted-events audit-mock)))
(t/deftest gc-emits-hard-delete-objects-for-teams
;; When the GC physically deletes a team, the projection must drop its row,
;; so the event carries the purged ids.
(with-mocks [audit-mock {:target 'app.loggers.audit/submit :return nil}]
(th/with-audit-config (conj cf/flags :nexus)
(let [profile (th/create-profile* 1 {:is-active true})
team (th/create-team* 71 {:profile-id (:id profile)})]
(th/command! {::th/type :delete-team
::rpc/profile-id (:id profile)
:id (:id team)})
;; Run the queued cascade so the default project is marked deleted too,
;; then purge everything the way production does.
(th/run-pending-jobs)
(th/reset-mock! audit-mock)
(th/run-task! :objects-gc {:skip-delay true})
(t/is (nil? (th/db-get :team {:id (:id team)} {::db/remove-deleted false})))
(let [events (filter #(= #{:team-ids} (set (keys (:props %))))
(events-named audit-mock "hard-delete-objects"))]
;; Every chunk carries exactly its own entity key and nothing
;; else: no flat team-ids are resolved after the delete.
(t/is (= 1 (count events)))
(let [event (first events)]
(t/is (= "command" (:type event)))
(t/is (= [(:id team)] (get-in event [:props :team-ids])))
(t/is (= "objects-gc" (get-in event [:context :triggered-by])))))))))
(t/deftest gc-emits-hard-delete-objects-for-projects
(with-mocks [audit-mock {:target 'app.loggers.audit/submit :return nil}]
(th/with-audit-config (conj cf/flags :nexus)
(let [profile (th/create-profile* 1 {:is-active true})
team (th/create-team* 72 {:profile-id (:id profile)})
project (th/create-project* 72 {:profile-id (:id profile)
:team-id (:id team)})]
(th/command! {::th/type :delete-project
::rpc/profile-id (:id profile)
:id (:id project)})
(th/run-pending-jobs)
(th/reset-mock! audit-mock)
(th/run-task! :objects-gc {:skip-delay true})
(t/is (nil? (th/db-get :project {:id (:id project)} {::db/remove-deleted false})))
(let [events (filter #(contains? (:props %) :project-ids)
(events-named audit-mock "hard-delete-objects"))]
(t/is (= 1 (count events)))
(let [event (first events)]
(t/is (= "command" (:type event)))
(t/is (= #{:project-ids} (set (keys (:props event)))))
(t/is (= [(:id project)] (get-in event [:props :project-ids])))
(t/is (= "objects-gc" (get-in event [:context :triggered-by])))))))))
(t/deftest gc-emits-hard-delete-objects-for-files
(with-mocks [audit-mock {:target 'app.loggers.audit/submit :return nil}]
(th/with-audit-config (conj cf/flags :nexus)
(let [profile (th/create-profile* 1 {:is-active true})
team (th/create-team* 73 {:profile-id (:id profile)})
project (th/create-project* 73 {:profile-id (:id profile)
:team-id (:id team)})
file (th/create-file* 73 {:profile-id (:id profile)
:project-id (:id project)})]
(th/command! {::th/type :delete-file
::rpc/profile-id (:id profile)
:id (:id file)})
(th/run-pending-jobs)
(th/reset-mock! audit-mock)
(th/run-task! :objects-gc {:skip-delay true})
(t/is (nil? (th/db-get :file {:id (:id file)} {::db/remove-deleted false})))
(let [events (filter #(contains? (:props %) :file-ids)
(events-named audit-mock "hard-delete-objects"))]
(t/is (= 1 (count events)))
(let [event (first events)]
(t/is (= "command" (:type event)))
(t/is (= #{:file-ids} (set (keys (:props event)))))
(t/is (= [(:id file)] (get-in event [:props :file-ids])))
(t/is (= "objects-gc" (get-in event [:context :triggered-by])))))))))
(t/deftest gc-emits-no-event-without-a-consumer
;; The GC must stay silent when the audit log is off, even though the row is
;; still purged.
(with-mocks [audit-mock {:target 'app.loggers.audit/submit :return nil}]
(let [profile (th/create-profile* 1 {:is-active true})
team (th/create-team* 74 {:profile-id (:id profile)})]
(th/command! {::th/type :delete-team
::rpc/profile-id (:id profile)
:id (:id team)})
(th/run-pending-jobs)
(th/reset-mock! audit-mock)
(th/run-task! :objects-gc {:skip-delay true})
(t/is (nil? (th/db-get :team {:id (:id team)} {::db/remove-deleted false})))
(t/is (empty? (:call-args-list @audit-mock))))))
(t/deftest gc-deletes-file-data-and-touches-its-storage-object
(let [profile (th/create-profile* 1)
file (th/create-file* 1 {:profile-id (:id profile)
:project-id (:default-project-id profile)})
sobject (sto/put-object! (::sto/storage th/*system*)
{::sto/content (sto/content "file-data")
:content-type "application/octet-stream"
:bucket "file-data"
:file-id (:id file)
:id (:id file)})]
;; `main` always carries `id = file_id`: mark the real row as deleted
;; instead of inventing a second main row.
(th/db-update! :file-data
{:backend "storage"
:data nil
:storage-object-id (:id sobject)
:deleted-at (ct/in-past "30m")}
{:file-id (:id file) :id (:id file)})
(th/run-task! :objects-gc {})
(t/testing "the row is purged"
(t/is (nil? (th/db-get :file-data {:file-id (:id file) :id (:id file)}))))
(t/testing "the storage object is touched for the storage GC"
(t/is (some? (:touched-at (th/db-get :storage-object {:id (:id sobject)})))))))
(t/deftest gc-emits-events-only-for-committed-chunks
(with-mocks [audit-mock {:target 'app.loggers.audit/submit :return nil}]
(th/with-audit-config (conj cf/flags :nexus)
(let [profile (th/create-profile* 1)
rolled-back (create-deleted-file! profile 1 (ct/in-past "30m"))
blocked (create-deleted-file! profile 2 (ct/in-past "20m"))
clean (create-deleted-file! profile 3 (ct/in-past "10m"))]
(block-file! blocked)
(th/reset-mock! audit-mock)
(let [result (th/run-task! :objects-gc {:chunk-size 2})
events (filter #(contains? (:props %) :file-ids)
(events-named audit-mock "hard-delete-objects"))]
(t/testing "the failed chunk keeps both files"
(t/is (some? (find-file (:id rolled-back))))
(t/is (some? (find-file (:id blocked)))))
(t/testing "the next chunk commits and reports only its deleted file"
(t/is (nil? (find-file (:id clean))))
(t/is (pos? (:processed result)))
(t/is (= 1 (count events)))
(t/is (= [(:id clean)] (get-in (first events) [:props :file-ids])))))))))