github/penpot
1
0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2026-11-08 11:04:44 +00:00
Code Issues Packages Projects Releases Wiki Activity
penpot/backend/test/backend_tests/objects_gc_test.clj
Andrey Antukh 67643340a2 ✨ Partition file_change by file_id 
Partition `file_change` by `HASH (file_id)` in 16 partitions, with the
same bounds as `file_data`, so a join between them is resolved partition
by partition. The primary key becomes `(file_id, id)`, which a
partitioned table needs, and every access filters by `file_id`.

The new table carries a `kind` column (1 snapshot, 2 revision) as a
reserve: the queries still tell them apart by the label, and `kind`
takes over when they are rewritten. The payload of a snapshot stays in
`file_data`; the `data` column is kept for reading old `legacy-db`
payloads.

The four snapshot commands require `file-id`, because it prunes the
partition. The objects GC locks and deletes by the pair, in batches. The
old table is kept as `file_change_legacy` as the way back.

The automatic migration refuses to run above 10 GiB or 2M rows and
points to the runbook; big instances partition online with that runbook,
which copies by id, follows the `updated_at` watermark in delta passes
and checks that no row of the source is missing from the target.

Closes #12416

AI-assisted-by: deepseek-v4.1-flash
2026-11-05 22:40:17 +01:00

310 lines
14 KiB
Clojure
Raw Blame History

;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; Copyright (c) KALEIDOS INC Sucursal en España SL
(ns backend-tests.objects-gc-test
(:require
[app.common.logging :as l]
[app.common.time :as ct]
[app.common.uuid :as uuid]
[app.config :as cf]
[app.db :as db]
[app.features.file-changes :as fchanges]
[app.rpc :as-alias rpc]
[app.storage :as sto]
[app.tasks.objects-gc :as ogc]
[backend-tests.helpers :as th]
[clojure.test :as t]
[mockery.core :refer [with-mocks]]))
(t/use-fixtures :once th/state-init)
(t/use-fixtures :each th/database-reset)
(defn- create-deleted-file!
"Creates a file and applies the deletion cascade on it (the same task the RPC
uses), so its children are marked as deleted for the given date. Returns the
file."
[profile i deleted-at]
(let [file (th/create-file* i {:profile-id (:id profile)
:project-id (:default-project-id profile)})]
(th/run-task! :delete-object {:object :file
:deleted-at deleted-at
:id (:id file)})
file))
(defn- block-file!
"Leaves an unmarked `file_media_object` pointing at the file, which is what
happens in production: a child that the deletion cascade did not mark. Its
foreign key towards `file` is `NO ACTION`, so it blocks the file deletion."
[file]
(let [media (sto/put-object! (::sto/storage th/*system*)
{::sto/content (sto/content "media")
:content-type "text/plain"
:bucket "file-media-object"})]
(th/create-file-media-object* {:file-id (:id file)
:media-id (:id media)})))
(defn- find-file
[id]
(th/db-get :file {:id id} {::db/remove-deleted false}))
(defn- chunk-error
"The first captured log record that reports a skipped chunk for `obj`,
waiting for it because logging is asynchronous."
[records obj]
(let [deadline (+ (System/currentTimeMillis) 5000)]
(loop []
(let [record (first (filter (fn [record]
(let [props (::l/props record)]
(and (= obj (get props :obj))
(= "error on chunk deletion, skipping it"
(get props :hint)))))
@records))]
(cond
record record
(> (System/currentTimeMillis) deadline) nil
:else (do (Thread/sleep 10) (recur)))))))
(t/deftest deletes-the-pending-files
(let [profile (th/create-profile* 1)
file (create-deleted-file! profile 1 (ct/in-past "10m"))
result (th/run-task! :objects-gc {})]
(t/is (pos? (:processed result)))
(t/is (nil? (find-file (:id file))))))
(t/deftest deletes-the-pending-file-changes
;; The pending file changes are locked per file, with a constant `file_id`,
;; so the rows of a file that is pending deletion are gone after the run.
(let [profile (th/create-profile* 1)
file (th/create-file* 1 {:profile-id (:id profile)
:project-id (:default-project-id profile)})
_ (th/db-insert! :file-change
{:file-id (:id file)
:kind fchanges/revision-kind
:deleted-at (ct/in-past "10m")})
result (th/run-task! :objects-gc {})]
(t/is (pos? (:processed result)))
(t/is (nil? (th/db-get :file-change {:file-id (:id file)}
{::db/remove-deleted false})))))
(t/deftest deletes-the-pending-file-changes-of-several-chunks
;; More pending rows than the chunk size: the plan walks the cursor and each
;; chunk locks its own rows, one query per file, with a constant `file_id`
;; so the planner prunes the partition.
(let [profile (th/create-profile* 1)
file (th/create-file* 1 {:profile-id (:id profile)
:project-id (:default-project-id profile)})
ids (repeatedly 5 uuid/next)
_ (doseq [id ids]
(th/db-insert! :file-change
{:id id
:file-id (:id file)
:kind fchanges/revision-kind
:deleted-at (ct/in-past "10m")}))
result (th/run-task! :objects-gc {:chunk-size 2})]
(t/is (pos? (:processed result)))
(t/is (empty? (th/db-query :file-change {:file-id (:id file)}
{::db/remove-deleted false})))))
(t/deftest a-chunk-spanning-several-files-deletes-all-of-them
;; The rows of one chunk can belong to different files. Each group is locked
;; with its own `file_id`, and every row is still deleted; buffering the
;; groups instead of locking them would leave them behind.
(let [profile (th/create-profile* 1)
opts {:profile-id (:id profile)
:project-id (:default-project-id profile)}
files [(th/create-file* 1 opts)
(th/create-file* 2 opts)]
_ (doseq [file files]
(dotimes [_ 2]
(th/db-insert! :file-change
{:file-id (:id file)
:kind fchanges/revision-kind
:deleted-at (ct/in-past "10m")})))
;; one chunk holds the rows of both files
result (th/run-task! :objects-gc {:chunk-size 4})]
(t/is (pos? (:processed result)))
(doseq [file files]
(t/is (empty? (th/db-query :file-change {:file-id (:id file)}
{::db/remove-deleted false}))))))
(t/deftest profile-cascade-keeps-profile-deletion-date
;; The cascade marks the owned rows with the deletion date of the profile,
;; like the RPC delete-profile flow does, not with the timestamp of the run.
;; The team survives because only the profiles chunk runs here, so its row
;; still shows that date.
(let [profile (th/create-profile* 1)
_ (th/db-update! :profile {:deleted-at (ct/in-past "30m")} {:id (:id profile)})
deleted-at (:deleted-at (th/db-get :profile {:id (:id profile)}
{::db/remove-deleted false}))
cfg {::db/pool th/*pool*
::sto/storage (::sto/storage th/*system*)
::ogc/threshold deleted-at}
total (#'ogc/delete-profiles-chunk cfg [(:id profile)])]
(t/testing "the profile is gone and the chunk reports it"
(t/is (= 1 total))
(t/is (nil? (th/db-get :profile {:id (:id profile)} {::db/remove-deleted false}))))
(t/testing "the owned team keeps the deletion date of the profile"
(t/is (= deleted-at (:deleted-at (th/db-get :team {:id (:default-team-id profile)}
{::db/remove-deleted false})))))))
(t/deftest blocked-chunk-does-not-stop-the-run
;; Chunks of one file: the blocked file takes down its own chunk and nothing
;; else, so the clean file of the next chunk is still deleted and the run
;; finishes. The next run retries the blocked file.
(let [profile (th/create-profile* 1)
blocked (create-deleted-file! profile 1 (ct/in-past "30m"))
clean (create-deleted-file! profile 2 (ct/in-past "20m"))
_ (block-file! blocked)
records (atom [])
_ (add-watch l/log-record ::capture
(fn [_ _ _ record] (swap! records conj record)))
result (try
(th/run-task! :objects-gc {:chunk-size 1})
(finally
(remove-watch l/log-record ::capture)))]
(t/testing "the clean file of the next chunk is deleted"
(t/is (nil? (find-file (:id clean)))))
(t/testing "the blocked file is still pending"
(t/is (some? (find-file (:id blocked)))))
(t/testing "the failed chunk is reported"
(t/is (some? (chunk-error records "file"))))
(t/testing "the run finishes"
(t/is (pos? (:processed result))))))
(defn- submitted-events
[audit-mock]
(->> (:call-args-list @audit-mock)
(map second)))
(defn- events-named
[audit-mock event-name]
(filter #(= event-name (:name %))
(submitted-events audit-mock)))
(t/deftest gc-emits-hard-delete-objects-for-teams
;; When the GC physically deletes a team, the projection must drop its row,
;; so the event carries the purged ids.
(with-mocks [audit-mock {:target 'app.loggers.audit/submit :return nil}]
(with-redefs [cf/flags (conj cf/flags :audit-log)]
(let [profile (th/create-profile* 1 {:is-active true})
team (th/create-team* 71 {:profile-id (:id profile)})]
(th/command! {::th/type :delete-team
::rpc/profile-id (:id profile)
:id (:id team)})
;; Run the queued cascade so the default project is marked deleted too,
;; then purge everything the way production does.
(th/run-pending-jobs)
(th/reset-mock! audit-mock)
(th/run-task! :objects-gc {:skip-delay true})
(t/is (nil? (th/db-get :team {:id (:id team)} {::db/remove-deleted false})))
(let [events (filter #(contains? (:props %) :team-ids)
(events-named audit-mock "hard-delete-objects"))]
(t/is (= 1 (count events)))
(let [event (first events)]
(t/is (= "command" (:type event)))
(t/is (= [(:id team)] (get-in event [:props :team-ids])))
(t/is (= "objects-gc" (get-in event [:context :triggered-by])))))))))
(t/deftest gc-emits-hard-delete-objects-for-projects
(with-mocks [audit-mock {:target 'app.loggers.audit/submit :return nil}]
(with-redefs [cf/flags (conj cf/flags :audit-log)]
(let [profile (th/create-profile* 1 {:is-active true})
team (th/create-team* 72 {:profile-id (:id profile)})
project (th/create-project* 72 {:profile-id (:id profile)
:team-id (:id team)})]
(th/command! {::th/type :delete-project
::rpc/profile-id (:id profile)
:id (:id project)})
(th/run-pending-jobs)
(th/reset-mock! audit-mock)
(th/run-task! :objects-gc {:skip-delay true})
(t/is (nil? (th/db-get :project {:id (:id project)} {::db/remove-deleted false})))
(let [events (filter #(contains? (:props %) :project-ids)
(events-named audit-mock "hard-delete-objects"))]
(t/is (= 1 (count events)))
(let [event (first events)]
(t/is (= "command" (:type event)))
(t/is (= [(:id project)] (get-in event [:props :project-ids])))
(t/is (= "objects-gc" (get-in event [:context :triggered-by])))))))))
(t/deftest gc-emits-hard-delete-objects-for-files
(with-mocks [audit-mock {:target 'app.loggers.audit/submit :return nil}]
(with-redefs [cf/flags (conj cf/flags :audit-log)]
(let [profile (th/create-profile* 1 {:is-active true})
team (th/create-team* 73 {:profile-id (:id profile)})
project (th/create-project* 73 {:profile-id (:id profile)
:team-id (:id team)})
file (th/create-file* 73 {:profile-id (:id profile)
:project-id (:id project)})]
(th/command! {::th/type :delete-file
::rpc/profile-id (:id profile)
:id (:id file)})
(th/run-pending-jobs)
(th/reset-mock! audit-mock)
(th/run-task! :objects-gc {:skip-delay true})
(t/is (nil? (th/db-get :file {:id (:id file)} {::db/remove-deleted false})))
(let [events (filter #(contains? (:props %) :file-ids)
(events-named audit-mock "hard-delete-objects"))]
(t/is (= 1 (count events)))
(let [event (first events)]
(t/is (= "command" (:type event)))
(t/is (= [(:id file)] (get-in event [:props :file-ids])))
(t/is (= "objects-gc" (get-in event [:context :triggered-by])))))))))
(t/deftest gc-emits-no-event-without-audit-log-flag
;; The GC must stay silent when the audit log is off, even though the row is
;; still purged.
(with-mocks [audit-mock {:target 'app.loggers.audit/submit :return nil}]
(let [profile (th/create-profile* 1 {:is-active true})
team (th/create-team* 74 {:profile-id (:id profile)})]
(th/command! {::th/type :delete-team
::rpc/profile-id (:id profile)
:id (:id team)})
(th/run-pending-jobs)
(th/reset-mock! audit-mock)
(th/run-task! :objects-gc {:skip-delay true})
(t/is (nil? (th/db-get :team {:id (:id team)} {::db/remove-deleted false})))
(t/is (empty? (:call-args-list @audit-mock))))))
(t/deftest gc-emits-events-only-for-committed-chunks
(with-mocks [audit-mock {:target 'app.loggers.audit/submit :return nil}]
(with-redefs [cf/flags (conj cf/flags :audit-log)]
(let [profile (th/create-profile* 1)
rolled-back (create-deleted-file! profile 1 (ct/in-past "30m"))
blocked (create-deleted-file! profile 2 (ct/in-past "20m"))
clean (create-deleted-file! profile 3 (ct/in-past "10m"))]
(block-file! blocked)
(th/reset-mock! audit-mock)
(let [result (th/run-task! :objects-gc {:chunk-size 2})
events (filter #(contains? (:props %) :file-ids)
(events-named audit-mock "hard-delete-objects"))]
(t/testing "the failed chunk keeps both files"
(t/is (some? (find-file (:id rolled-back))))
(t/is (some? (find-file (:id blocked)))))
(t/testing "the next chunk commits and reports only its deleted file"
(t/is (nil? (find-file (:id clean))))
(t/is (pos? (:processed result)))
(t/is (= 1 (count events)))
(t/is (= [(:id clean)] (get-in (first events) [:props :file-ids])))))))))
Reference in New Issue View Git Blame Copy Permalink