mirror of
https://github.com/penpot/penpot.git
synced 2026-10-29 14:00:37 +00:00
0947e666c0
objects-gc deleted every kind of object inside one transaction per proc, looping until empty, so a single row that a foreign key still references aborted the whole run: the task died on its first file every night, after the earlier steps had already spent around twenty minutes. The per-file savepoint only saved the files proc and left the same shape everywhere else. Now every proc streams its pending keys with db/plan inside a read-only transaction of its own and deletes them chunk by chunk, each chunk in a transaction of its own with FOR UPDATE SKIP LOCKED. A chunk that fails is reported with l/err and skipped, so the run reaches the end and the next run retries the skipped rows. AI-assisted-by: muse-spark-1.3-contributor
126 lines
5.1 KiB
Clojure
126 lines
5.1 KiB
Clojure
;; This Source Code Form is subject to the terms of the Mozilla Public
|
|
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
;;
|
|
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
|
|
|
(ns backend-tests.objects-gc-test
|
|
(:require
|
|
[app.common.logging :as l]
|
|
[app.common.time :as ct]
|
|
[app.db :as db]
|
|
[app.storage :as sto]
|
|
[app.tasks.objects-gc :as ogc]
|
|
[backend-tests.helpers :as th]
|
|
[clojure.test :as t]))
|
|
|
|
(t/use-fixtures :once th/state-init)
|
|
(t/use-fixtures :each th/database-reset)
|
|
|
|
(defn- create-deleted-file!
|
|
"Creates a file and applies the deletion cascade on it (the same task the RPC
|
|
uses), so its children are marked as deleted for the given date. Returns the
|
|
file."
|
|
[profile i deleted-at]
|
|
(let [file (th/create-file* i {:profile-id (:id profile)
|
|
:project-id (:default-project-id profile)})]
|
|
(th/run-task! :delete-object {:object :file
|
|
:deleted-at deleted-at
|
|
:id (:id file)})
|
|
file))
|
|
|
|
(defn- block-file!
|
|
"Leaves an unmarked `file_media_object` pointing at the file, which is what
|
|
happens in production: a child that the deletion cascade did not mark. Its
|
|
foreign key towards `file` is `NO ACTION`, so it blocks the file deletion."
|
|
[file]
|
|
(let [media (sto/put-object! (::sto/storage th/*system*)
|
|
{::sto/content (sto/content "media")
|
|
:content-type "text/plain"
|
|
:bucket "file-media-object"})]
|
|
(th/create-file-media-object* {:file-id (:id file)
|
|
:media-id (:id media)})))
|
|
|
|
(defn- find-file
|
|
[id]
|
|
(th/db-get :file {:id id} {::db/remove-deleted false}))
|
|
|
|
(defn- chunk-error
|
|
"The first captured log record that reports a skipped chunk for `obj`,
|
|
waiting for it because logging is asynchronous."
|
|
[records obj]
|
|
(let [deadline (+ (System/currentTimeMillis) 5000)]
|
|
(loop []
|
|
(let [record (first (filter (fn [record]
|
|
(let [props (::l/props record)]
|
|
(and (= obj (get props :obj))
|
|
(= "error on chunk deletion, skipping it"
|
|
(get props :hint)))))
|
|
@records))]
|
|
(cond
|
|
record record
|
|
(> (System/currentTimeMillis) deadline) nil
|
|
:else (do (Thread/sleep 10) (recur)))))))
|
|
|
|
(t/deftest deletes-the-pending-files
|
|
(let [profile (th/create-profile* 1)
|
|
file (create-deleted-file! profile 1 (ct/in-past "10m"))
|
|
result (th/run-task! :objects-gc {})]
|
|
|
|
(t/is (pos? (:processed result)))
|
|
(t/is (nil? (find-file (:id file))))))
|
|
|
|
(t/deftest profile-cascade-keeps-profile-deletion-date
|
|
;; The cascade marks the owned rows with the deletion date of the profile,
|
|
;; like the RPC delete-profile flow does, not with the timestamp of the run.
|
|
;; The team survives because only the profiles chunk runs here, so its row
|
|
;; still shows that date.
|
|
(let [profile (th/create-profile* 1)
|
|
_ (th/db-update! :profile {:deleted-at (ct/in-past "30m")} {:id (:id profile)})
|
|
|
|
deleted-at (:deleted-at (th/db-get :profile {:id (:id profile)}
|
|
{::db/remove-deleted false}))
|
|
|
|
cfg {::db/pool th/*pool*
|
|
::sto/storage (::sto/storage th/*system*)
|
|
::ogc/timestamp deleted-at}
|
|
total (#'ogc/delete-profiles-chunk cfg [(:id profile)])]
|
|
|
|
(t/testing "the profile is gone and the chunk reports it"
|
|
(t/is (= 1 total))
|
|
(t/is (nil? (th/db-get :profile {:id (:id profile)} {::db/remove-deleted false}))))
|
|
|
|
(t/testing "the owned team keeps the deletion date of the profile"
|
|
(t/is (= deleted-at (:deleted-at (th/db-get :team {:id (:default-team-id profile)}
|
|
{::db/remove-deleted false})))))))
|
|
|
|
(t/deftest blocked-chunk-does-not-stop-the-run
|
|
;; Chunks of one file: the blocked file takes down its own chunk and nothing
|
|
;; else, so the clean file of the next chunk is still deleted and the run
|
|
;; finishes. The next run retries the blocked file.
|
|
(let [profile (th/create-profile* 1)
|
|
blocked (create-deleted-file! profile 1 (ct/in-past "30m"))
|
|
clean (create-deleted-file! profile 2 (ct/in-past "20m"))
|
|
|
|
_ (block-file! blocked)
|
|
|
|
records (atom [])
|
|
_ (add-watch l/log-record ::capture
|
|
(fn [_ _ _ record] (swap! records conj record)))
|
|
result (try
|
|
(th/run-task! :objects-gc {:chunk-size 1})
|
|
(finally
|
|
(remove-watch l/log-record ::capture)))]
|
|
|
|
(t/testing "the clean file of the next chunk is deleted"
|
|
(t/is (nil? (find-file (:id clean)))))
|
|
|
|
(t/testing "the blocked file is still pending"
|
|
(t/is (some? (find-file (:id blocked)))))
|
|
|
|
(t/testing "the failed chunk is reported"
|
|
(t/is (some? (chunk-error records "file"))))
|
|
|
|
(t/testing "the run finishes"
|
|
(t/is (pos? (:processed result))))))
|