diff --git a/backend/src/app/storage.clj b/backend/src/app/storage.clj index 08ddfd76f7..c554955fcd 100644 --- a/backend/src/app/storage.clj +++ b/backend/src/app/storage.clj @@ -163,9 +163,6 @@ backend (:metadata result)))) -(def ^:private sql:retrieve-storage-object - "select * from storage_object where id = ? and (deleted_at is null or deleted_at > now())") - (defn row->storage-object [res] (let [mdata (or (some-> (:metadata res) (db/decode-transit-pgobject)) {})] (impl/storage-object @@ -177,9 +174,15 @@ (keyword (:backend res)) mdata))) -(defn- retrieve-database-object +(def ^:private sql:get-storage-object + "SELECT * + FROM storage_object + WHERE id = ? + AND (deleted_at IS NULL)") + +(defn- get-database-object [conn id] - (some-> (db/exec-one! conn [sql:retrieve-storage-object id]) + (some-> (db/exec-one! conn [sql:get-storage-object id]) (row->storage-object))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -202,7 +205,7 @@ (defn get-object [{:keys [::db/connectable] :as storage} id] (assert (valid-storage? storage)) - (retrieve-database-object connectable id)) + (get-database-object connectable id)) (defn put-object! "Creates a new object with the provided content." diff --git a/backend/src/app/storage/gc_deleted.clj b/backend/src/app/storage/gc_deleted.clj index 51dd18037b..0e707cdc5c 100644 --- a/backend/src/app/storage/gc_deleted.clj +++ b/backend/src/app/storage/gc_deleted.clj @@ -37,7 +37,6 @@ (into #{} (map :id)) (not-empty)))) - (def ^:private sql:delete-sobjects "DELETE FROM storage_object WHERE id = ANY(?::uuid[])") @@ -77,47 +76,37 @@ (d/group-by (comp keyword :backend) :id #{} items)) (def ^:private sql:get-deleted-sobjects - "SELECT s.* FROM storage_object AS s + "SELECT s.* + FROM storage_object AS s WHERE s.deleted_at IS NOT NULL - AND s.deleted_at < now() - ?::interval + AND s.deleted_at <= ? ORDER BY s.deleted_at ASC") (defn- get-buckets - [conn min-age] - (let [age (db/interval min-age)] + [conn] + (let [now (ct/now)] (sequence (comp (partition-all 25) (mapcat group-by-backend)) - (db/cursor conn [sql:get-deleted-sobjects age])))) - + (db/cursor conn [sql:get-deleted-sobjects now])))) (defn- clean-deleted! - [{:keys [::db/conn ::min-age] :as cfg}] + [{:keys [::db/conn] :as cfg}] (reduce (fn [total [backend-id ids]] (let [deleted (delete-in-bulk! cfg backend-id ids)] (+ total (or deleted 0)))) 0 - (get-buckets conn min-age))) + (get-buckets conn))) (defmethod ig/assert-key ::handler [_ params] (assert (sto/valid-storage? (::sto/storage params)) "expect valid storage") (assert (db/pool? (::db/pool params)) "expect valid storage")) -(defmethod ig/expand-key ::handler - [k v] - {k (assoc v ::min-age (ct/duration {:hours 2}))}) - (defmethod ig/init-key ::handler - [_ {:keys [::min-age] :as cfg}] - (fn [{:keys [props] :as task}] - (let [min-age (ct/duration (or (:min-age props) min-age))] - (db/tx-run! cfg (fn [cfg] - (let [cfg (assoc cfg ::min-age min-age) - total (clean-deleted! cfg)] - - (l/inf :hint "task finished" - :min-age (ct/format-duration min-age) - :total total) - - {:deleted total})))))) + [_ cfg] + (fn [_] + (db/tx-run! cfg (fn [cfg] + (let [total (clean-deleted! cfg)] + (l/inf :hint "task finished" :total total) + {:deleted total}))))) diff --git a/backend/src/app/storage/gc_touched.clj b/backend/src/app/storage/gc_touched.clj index c828a69cb3..a906626647 100644 --- a/backend/src/app/storage/gc_touched.clj +++ b/backend/src/app/storage/gc_touched.clj @@ -22,6 +22,8 @@ [app.common.data.macros :as dm] [app.common.exceptions :as ex] [app.common.logging :as l] + [app.common.time :as ct] + [app.config :as cf] [app.db :as db] [app.storage :as-alias sto] [app.storage.impl :as impl] @@ -101,14 +103,15 @@ (def ^:private sql:mark-delete-in-bulk "UPDATE storage_object - SET deleted_at = now(), + SET deleted_at = ?, touched_at = NULL WHERE id = ANY(?::uuid[])") (defn- mark-delete-in-bulk! - [conn ids] - (let [ids (db/create-array conn "uuid" ids)] - (db/exec-one! conn [sql:mark-delete-in-bulk ids]))) + [conn deletion-delay ids] + (let [ids (db/create-array conn "uuid" ids) + now (ct/plus (ct/now) deletion-delay)] + (db/exec-one! conn [sql:mark-delete-in-bulk now ids]))) ;; NOTE: A getter that retrieves the key which will be used for group ;; ids; previously we have no value, then we introduced the @@ -137,18 +140,20 @@ (if-let [{:keys [id] :as object} (first objects)] (if (has-refs? conn object) (do - (l/debug :id (str id) - :status "freeze" - :bucket bucket) + (l/dbg :id (str id) + :status "freeze" + :bucket bucket) (recur (conj to-freeze id) to-delete (rest objects))) (do - (l/debug :id (str id) - :status "delete" - :bucket bucket) + (l/dbg :id (str id) + :status "delete" + :bucket bucket) (recur to-freeze (conj to-delete id) (rest objects)))) - (do + (let [deletion-delay (if (= bucket "tempfile") + (ct/duration {:hours 2}) + (cf/get-deletion-delay))] (some->> (seq to-freeze) (mark-freeze-in-bulk! conn)) - (some->> (seq to-delete) (mark-delete-in-bulk! conn)) + (some->> (seq to-delete) (mark-delete-in-bulk! conn deletion-delay)) [(count to-freeze) (count to-delete)])))) (defn- process-bucket! @@ -173,27 +178,27 @@ [0 0] (d/group-by lookup-bucket identity #{} chunk))) -(def ^:private - sql:get-touched-storage-objects +(def ^:private sql:get-touched-storage-objects "SELECT so.* FROM storage_object AS so WHERE so.touched_at IS NOT NULL + AND so.touched_at <= ? ORDER BY touched_at ASC FOR UPDATE SKIP LOCKED LIMIT 10") (defn get-chunk - [conn] - (->> (db/exec! conn [sql:get-touched-storage-objects]) + [conn timestamp] + (->> (db/exec! conn [sql:get-touched-storage-objects timestamp]) (map impl/decode-row) (not-empty))) (defn- process-touched! - [{:keys [::db/pool] :as cfg}] + [{:keys [::db/pool ::timestamp] :as cfg}] (loop [freezed 0 deleted 0] - (if-let [chunk (get-chunk pool)] + (if-let [chunk (get-chunk pool timestamp)] (let [[nfo ndo] (db/tx-run! cfg process-chunk! chunk)] (recur (long (+ freezed nfo)) (long (+ deleted ndo)))) @@ -209,5 +214,6 @@ (defmethod ig/init-key ::handler [_ cfg] - (fn [_] (process-touched! cfg))) + (fn [_] + (process-touched! (assoc cfg ::timestamp (ct/now))))) diff --git a/backend/src/app/tasks/objects_gc.clj b/backend/src/app/tasks/objects_gc.clj index 75b9c70065..f0b5bc1d1b 100644 --- a/backend/src/app/tasks/objects_gc.clj +++ b/backend/src/app/tasks/objects_gc.clj @@ -18,15 +18,15 @@ (def ^:private sql:get-profiles "SELECT id, photo_id FROM profile WHERE deleted_at IS NOT NULL - AND deleted_at < now() + ?::interval + AND deleted_at <= ? ORDER BY deleted_at ASC LIMIT ? FOR UPDATE SKIP LOCKED") (defn- delete-profiles! - [{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}] - (->> (db/plan conn [sql:get-profiles deletion-threshold chunk-size] {:fetch-size 5}) + [{:keys [::db/conn ::timestamp ::chunk-size ::sto/storage] :as cfg}] + (->> (db/plan conn [sql:get-profiles timestamp chunk-size] {:fetch-size 5}) (reduce (fn [total {:keys [id photo-id]}] (l/trc :obj "profile" :id (str id)) @@ -41,15 +41,15 @@ (def ^:private sql:get-teams "SELECT deleted_at, id, photo_id FROM team WHERE deleted_at IS NOT NULL - AND deleted_at < now() + ?::interval + AND deleted_at <= ? ORDER BY deleted_at ASC LIMIT ? FOR UPDATE SKIP LOCKED") (defn- delete-teams! - [{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}] - (->> (db/plan conn [sql:get-teams deletion-threshold chunk-size] {:fetch-size 5}) + [{:keys [::db/conn ::timestamp ::chunk-size ::sto/storage] :as cfg}] + (->> (db/plan conn [sql:get-teams timestamp chunk-size] {:fetch-size 5}) (reduce (fn [total {:keys [id photo-id deleted-at]}] (l/trc :obj "team" :id (str id) @@ -68,15 +68,15 @@ "SELECT id, team_id, deleted_at, woff1_file_id, woff2_file_id, otf_file_id, ttf_file_id FROM team_font_variant WHERE deleted_at IS NOT NULL - AND deleted_at < now() + ?::interval + AND deleted_at <= ? ORDER BY deleted_at ASC LIMIT ? FOR UPDATE SKIP LOCKED") (defn- delete-fonts! - [{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}] - (->> (db/plan conn [sql:get-fonts deletion-threshold chunk-size] {:fetch-size 5}) + [{:keys [::db/conn ::timestamp ::chunk-size ::sto/storage] :as cfg}] + (->> (db/plan conn [sql:get-fonts timestamp chunk-size] {:fetch-size 5}) (reduce (fn [total {:keys [id team-id deleted-at] :as font}] (l/trc :obj "font-variant" :id (str id) @@ -98,15 +98,15 @@ "SELECT id, deleted_at, team_id FROM project WHERE deleted_at IS NOT NULL - AND deleted_at < now() + ?::interval + AND deleted_at <= ? ORDER BY deleted_at ASC LIMIT ? FOR UPDATE SKIP LOCKED") (defn- delete-projects! - [{:keys [::db/conn ::deletion-threshold ::chunk-size] :as cfg}] - (->> (db/plan conn [sql:get-projects deletion-threshold chunk-size] {:fetch-size 5}) + [{:keys [::db/conn ::timestamp ::chunk-size] :as cfg}] + (->> (db/plan conn [sql:get-projects timestamp chunk-size] {:fetch-size 5}) (reduce (fn [total {:keys [id team-id deleted-at]}] (l/trc :obj "project" :id (str id) @@ -124,15 +124,15 @@ f.project_id FROM file AS f WHERE f.deleted_at IS NOT NULL - AND f.deleted_at < now() + ?::interval + AND f.deleted_at <= ? ORDER BY f.deleted_at ASC LIMIT ? FOR UPDATE SKIP LOCKED") (defn- delete-files! - [{:keys [::db/conn ::deletion-threshold ::chunk-size] :as cfg}] - (->> (db/plan conn [sql:get-files deletion-threshold chunk-size] {:fetch-size 5}) + [{:keys [::db/conn ::timestamp ::chunk-size] :as cfg}] + (->> (db/plan conn [sql:get-files timestamp chunk-size] {:fetch-size 5}) (reduce (fn [total {:keys [id deleted-at project-id] :as file}] (l/trc :obj "file" :id (str id) @@ -148,15 +148,15 @@ "SELECT file_id, revn, media_id, deleted_at FROM file_thumbnail WHERE deleted_at IS NOT NULL - AND deleted_at < now() + ?::interval + AND deleted_at <= ? ORDER BY deleted_at ASC LIMIT ? FOR UPDATE SKIP LOCKED") (defn delete-file-thumbnails! - [{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}] - (->> (db/plan conn [sql:get-file-thumbnails deletion-threshold chunk-size] {:fetch-size 5}) + [{:keys [::db/conn ::timestamp ::chunk-size ::sto/storage] :as cfg}] + (->> (db/plan conn [sql:get-file-thumbnails timestamp chunk-size] {:fetch-size 5}) (reduce (fn [total {:keys [file-id revn media-id deleted-at]}] (l/trc :obj "file-thumbnail" :file-id (str file-id) @@ -175,15 +175,15 @@ "SELECT file_id, object_id, media_id, deleted_at FROM file_tagged_object_thumbnail WHERE deleted_at IS NOT NULL - AND deleted_at < now() + ?::interval + AND deleted_at <= ? ORDER BY deleted_at ASC LIMIT ? FOR UPDATE SKIP LOCKED") (defn delete-file-object-thumbnails! - [{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}] - (->> (db/plan conn [sql:get-file-object-thumbnails deletion-threshold chunk-size] {:fetch-size 5}) + [{:keys [::db/conn ::timestamp ::chunk-size ::sto/storage] :as cfg}] + (->> (db/plan conn [sql:get-file-object-thumbnails timestamp chunk-size] {:fetch-size 5}) (reduce (fn [total {:keys [file-id object-id media-id deleted-at]}] (l/trc :obj "file-object-thumbnail" :file-id (str file-id) @@ -203,15 +203,15 @@ "SELECT id, file_id, media_id, thumbnail_id, deleted_at FROM file_media_object WHERE deleted_at IS NOT NULL - AND deleted_at < now() + ?::interval + AND deleted_at <= ? ORDER BY deleted_at ASC LIMIT ? FOR UPDATE SKIP LOCKED") (defn- delete-file-media-objects! - [{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}] - (->> (db/plan conn [sql:get-file-media-objects deletion-threshold chunk-size] {:fetch-size 5}) + [{:keys [::db/conn ::timestamp ::chunk-size ::sto/storage] :as cfg}] + (->> (db/plan conn [sql:get-file-media-objects timestamp chunk-size] {:fetch-size 5}) (reduce (fn [total {:keys [id file-id deleted-at] :as fmo}] (l/trc :obj "file-media-object" :id (str id) @@ -231,16 +231,15 @@ "SELECT file_id, id, type, deleted_at, metadata, backend FROM file_data WHERE deleted_at IS NOT NULL - AND deleted_at < now() + ?::interval + AND deleted_at <= ? ORDER BY deleted_at ASC LIMIT ? FOR UPDATE SKIP LOCKED") (defn- delete-file-data! - [{:keys [::db/conn ::deletion-threshold ::chunk-size] :as cfg}] - - (->> (db/plan conn [sql:get-file-data deletion-threshold chunk-size] {:fetch-size 5}) + [{:keys [::db/conn ::timestamp ::chunk-size] :as cfg}] + (->> (db/plan conn [sql:get-file-data timestamp chunk-size] {:fetch-size 5}) (reduce (fn [total {:keys [file-id id type deleted-at metadata backend]}] (some->> metadata @@ -266,15 +265,15 @@ "SELECT id, file_id, deleted_at FROM file_change WHERE deleted_at IS NOT NULL - AND deleted_at < now() + ?::interval + AND deleted_at <= ? ORDER BY deleted_at ASC LIMIT ? FOR UPDATE SKIP LOCKED") (defn- delete-file-changes! - [{:keys [::db/conn ::deletion-threshold ::chunk-size] :as cfg}] - (->> (db/plan conn [sql:get-file-change deletion-threshold chunk-size] {:fetch-size 5}) + [{:keys [::db/conn ::timestamp ::chunk-size] :as cfg}] + (->> (db/plan conn [sql:get-file-change timestamp chunk-size] {:fetch-size 5}) (reduce (fn [total {:keys [id file-id deleted-at] :as xlog}] (l/trc :obj "file-change" :id (str id) @@ -322,9 +321,8 @@ (defmethod ig/init-key ::handler [_ cfg] - (fn [{:keys [props] :as task}] - (let [threshold (ct/duration (get props :deletion-threshold 0)) - cfg (assoc cfg ::deletion-threshold (db/interval threshold))] + (fn [_] + (let [cfg (assoc cfg ::timestamp (ct/now))] (loop [procs (map deref deletion-proc-vars) total 0] (if-let [proc-fn (first procs)] diff --git a/backend/test/backend_tests/rpc_file_snapshot_test.clj b/backend/test/backend_tests/rpc_file_snapshot_test.clj index e2c8248815..faedb6bb5e 100644 --- a/backend/test/backend_tests/rpc_file_snapshot_test.clj +++ b/backend/test/backend_tests/rpc_file_snapshot_test.clj @@ -9,6 +9,7 @@ [app.common.features :as cfeat] [app.common.pprint :as pp] [app.common.thumbnails :as thc] + [app.common.time :as ct] [app.common.types.shape :as cts] [app.common.uuid :as uuid] [app.config :as cf] @@ -16,6 +17,7 @@ [app.db.sql :as sql] [app.http :as http] [app.rpc :as-alias rpc] + [app.setup.clock :as clock] [app.storage :as sto] [backend-tests.helpers :as th] [clojure.test :as t] @@ -132,9 +134,10 @@ ;; this will run pending task triggered by deleting user snapshot (th/run-pending-tasks!) - (let [res (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})] - ;; delete 2 snapshots and 2 file data entries - (t/is (= 4 (:processed res)))))))) + (binding [ct/*clock* (clock/fixed (ct/in-future {:days 8}))] + (let [res (th/run-task! :objects-gc {})] + ;; delete 2 snapshots and 2 file data entries + (t/is (= 4 (:processed res))))))))) (t/deftest snapshots-locking (let [profile-1 (th/create-profile* 1 {:is-active true}) diff --git a/backend/test/backend_tests/rpc_file_test.clj b/backend/test/backend_tests/rpc_file_test.clj index d34077bc2d..0c149ec413 100644 --- a/backend/test/backend_tests/rpc_file_test.clj +++ b/backend/test/backend_tests/rpc_file_test.clj @@ -313,7 +313,7 @@ ;; freeze because of the deduplication (we have uploaded 2 times ;; the same files). - (let [res (th/run-task! :storage-gc-touched {:min-age 0})] + (let [res (th/run-task! :storage-gc-touched {})] (t/is (= 2 (:freeze res))) (t/is (= 0 (:delete res)))) @@ -372,14 +372,14 @@ (th/db-exec! ["update file_change set deleted_at = now() where file_id = ? and label is not null" (:id file)]) (th/db-exec! ["update file set has_media_trimmed = false where id = ?" (:id file)]) - (let [res (th/run-task! :objects-gc {:deletion-threshold 0})] + (let [res (th/run-task! :objects-gc {})] ;; this will remove the file change and file data entries for two snapshots (t/is (= 4 (:processed res)))) ;; Rerun the file-gc and objects-gc (t/is (true? (th/run-task! :file-gc {:file-id (:id file)}))) - (let [res (th/run-task! :objects-gc {:deletion-threshold 0})] + (let [res (th/run-task! :objects-gc {})] ;; this will remove the file media objects marked as deleted ;; on prev file-gc (t/is (= 2 (:processed res)))) @@ -387,7 +387,7 @@ ;; Now that file-gc have deleted the file-media-object usage, ;; lets execute the touched-gc task, we should see that two of ;; them are marked to be deleted - (let [res (th/run-task! :storage-gc-touched {:min-age 0})] + (let [res (th/run-task! :storage-gc-touched {})] (t/is (= 0 (:freeze res))) (t/is (= 2 (:delete res)))) @@ -572,7 +572,7 @@ ;; Now that file-gc have deleted the file-media-object usage, ;; lets execute the touched-gc task, we should see that two of ;; them are marked to be deleted. - (let [res (th/run-task! :storage-gc-touched {:min-age 0})] + (let [res (th/run-task! :storage-gc-touched {})] (t/is (= 0 (:freeze res))) (t/is (= 2 (:delete res)))) @@ -665,7 +665,7 @@ ;; because of the deduplication (we have uploaded 2 times the ;; same files). - (let [res (th/run-task! :storage-gc-touched {:min-age 0})] + (let [res (th/run-task! :storage-gc-touched {})] (t/is (= 1 (:freeze res))) (t/is (= 0 (:delete res)))) @@ -715,7 +715,7 @@ ;; Now that objects-gc have deleted the object thumbnail lets ;; execute the touched-gc task - (let [res (th/run-task! "storage-gc-touched" {:min-age 0})] + (let [res (th/run-task! "storage-gc-touched" {})] (t/is (= 1 (:freeze res)))) ;; check file media objects @@ -750,7 +750,7 @@ ;; Now that file-gc have deleted the object thumbnail lets ;; execute the touched-gc task - (let [res (th/run-task! :storage-gc-touched {:min-age 0})] + (let [res (th/run-task! :storage-gc-touched {})] (t/is (= 1 (:delete res)))) ;; check file media objects @@ -922,8 +922,9 @@ (t/is (= 0 (:processed result)))) ;; run permanent deletion - (let [result (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})] - (t/is (= 3 (:processed result)))) + (binding [ct/*clock* (clock/fixed (ct/in-future {:days 8}))] + (let [result (th/run-task! :objects-gc {})] + (t/is (= 3 (:processed result))))) ;; query the list of file libraries of a after hard deletion (let [data {::th/type :get-file-libraries @@ -1134,7 +1135,7 @@ (th/sleep 300) ;; run the task - (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)}))) + (t/is (true? (th/run-task! :file-gc {:file-id (:id file)}))) ;; check that object thumbnails are still here (let [rows (th/db-query :file-tagged-object-thumbnail {:file-id (:id file)})] @@ -1163,7 +1164,7 @@ (t/is (= 2 (count rows)))) ;; run the task again - (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)}))) + (t/is (true? (th/run-task! :file-gc {:file-id (:id file)}))) ;; check that we have all object thumbnails (let [rows (th/db-query :file-tagged-object-thumbnail {:file-id (:id file)})] @@ -1226,7 +1227,7 @@ (t/is (= 2 (count rows))))) (t/testing "gc task" - (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)}))) + (t/is (true? (th/run-task! :file-gc {:file-id (:id file)}))) (let [rows (th/db-query :file-thumbnail {:file-id (:id file)})] (t/is (= 2 (count rows))) @@ -1273,7 +1274,7 @@ ;; The FileGC task will schedule an inner taskq (th/run-pending-tasks!) - (let [res (th/run-task! :storage-gc-touched {:min-age 0})] + (let [res (th/run-task! :storage-gc-touched {})] (t/is (= 2 (:freeze res))) (t/is (= 0 (:delete res)))) @@ -1367,7 +1368,7 @@ ;; we ensure that once object-gc is passed and marked two storage ;; objects to delete - (let [res (th/run-task! :storage-gc-touched {:min-age 0})] + (let [res (th/run-task! :storage-gc-touched {})] (t/is (= 0 (:freeze res))) (t/is (= 2 (:delete res)))) @@ -1489,7 +1490,7 @@ (t/is (some? (not-empty (:objects component)))))) ;; Re-run the file-gc task - (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)}))) + (t/is (true? (th/run-task! :file-gc {:file-id (:id file)}))) (let [row (th/db-get :file {:id (:id file)})] (t/is (true? (:has-media-trimmed row)))) @@ -1519,7 +1520,7 @@ ;; Now, we have deleted the usage of component if we pass file-gc, ;; that component should be deleted - (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)}))) + (t/is (true? (th/run-task! :file-gc {:file-id (:id file)}))) ;; Check that component is properly removed (let [data {::th/type :get-file @@ -1610,8 +1611,8 @@ :component-id c-id})}]) ;; Run the file-gc on file and library - (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file-1)}))) - (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file-2)}))) + (t/is (true? (th/run-task! :file-gc {:file-id (:id file-1)}))) + (t/is (true? (th/run-task! :file-gc {:file-id (:id file-2)}))) ;; Check that component exists (let [data {::th/type :get-file @@ -1684,7 +1685,7 @@ ;; Now, we have deleted the usage of component if we pass file-gc, ;; that component should be deleted - (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file-1)}))) + (t/is (true? (th/run-task! :file-gc {:file-id (:id file-1)}))) ;; Check that component is properly removed (let [data {::th/type :get-file @@ -1833,8 +1834,8 @@ (t/is (not= (:id fill) (:id fmedia))))) ;; Run the file-gc on file and library - (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file-1)}))) - (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file-2)}))) + (t/is (true? (th/run-task! :file-gc {:file-id (:id file-1)}))) + (t/is (true? (th/run-task! :file-gc {:file-id (:id file-2)}))) ;; Now proceed to delete file and absorb it (let [data {::th/type :delete-file diff --git a/backend/test/backend_tests/rpc_file_thumbnails_test.clj b/backend/test/backend_tests/rpc_file_thumbnails_test.clj index b59bdbfcf3..31e6ac5402 100644 --- a/backend/test/backend_tests/rpc_file_thumbnails_test.clj +++ b/backend/test/backend_tests/rpc_file_thumbnails_test.clj @@ -8,12 +8,14 @@ (:require [app.common.pprint :as pp] [app.common.thumbnails :as thc] + [app.common.time :as ct] [app.common.types.shape :as cts] [app.common.uuid :as uuid] [app.config :as cf] [app.db :as db] [app.rpc :as-alias rpc] [app.rpc.commands.auth :as cauth] + [app.setup.clock :as clock] [app.storage :as sto] [app.tokens :as tokens] [backend-tests.helpers :as th] @@ -83,7 +85,8 @@ (t/is (map? (:result out)))) ;; run the task again - (let [res (th/run-task! "storage-gc-touched" {:min-age 0})] + (let [res (binding [ct/*clock* (clock/fixed (ct/in-future {:minutes 31}))] + (th/run-task! "storage-gc-touched" {}))] (t/is (= 2 (:freeze res)))) (let [[row1 row2 :as rows] (th/db-query :file-tagged-object-thumbnail @@ -114,9 +117,9 @@ ;; Run the File GC task that should remove unused file object ;; thumbnails - (th/run-task! :file-gc {:min-age 0 :file-id (:id file)}) + (th/run-task! :file-gc {:file-id (:id file)}) - (let [result (th/run-task! :objects-gc {:min-age 0})] + (let [result (th/run-task! :objects-gc {})] (t/is (= 3 (:processed result)))) ;; check if row2 related thumbnail row still exists @@ -133,7 +136,8 @@ (t/is (some? (sto/get-object storage (:media-id row2)))) ;; run the task again - (let [res (th/run-task! :storage-gc-touched {:min-age 0})] + (let [res (binding [ct/*clock* (clock/fixed (ct/in-future {:minutes 31}))] + (th/run-task! :storage-gc-touched {}))] (t/is (= 1 (:delete res))) (t/is (= 0 (:freeze res)))) @@ -143,8 +147,9 @@ ;; Run the storage gc deleted task, it should permanently delete ;; all storage objects related to the deleted thumbnails - (let [result (th/run-task! :storage-gc-deleted {:min-age 0})] - (t/is (= 1 (:deleted result)))) + (binding [ct/*clock* (clock/fixed (ct/in-future {:days 8}))] + (let [res (th/run-task! :storage-gc-deleted {})] + (t/is (= 1 (:deleted res))))) (t/is (nil? (sto/get-object storage (:media-id row1)))) (t/is (some? (sto/get-object storage (:media-id row2)))) @@ -216,9 +221,9 @@ ;; Run the File GC task that should remove unused file object ;; thumbnails - (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)}))) + (t/is (true? (th/run-task! :file-gc {:file-id (:id file)}))) - (let [result (th/run-task! :objects-gc {:min-age 0})] + (let [result (th/run-task! :objects-gc {})] (t/is (= 2 (:processed result)))) ;; check if row1 related thumbnail row still exists @@ -230,7 +235,7 @@ (t/is (= (:object-id data1) (:object-id row))) (t/is (uuid? (:media-id row1)))) - (let [result (th/run-task! :storage-gc-touched {:min-age 0})] + (let [result (th/run-task! :storage-gc-touched {})] (t/is (= 1 (:delete result)))) ;; Check if storage objects still exists after file-gc @@ -242,8 +247,9 @@ ;; Run the storage gc deleted task, it should permanently delete ;; all storage objects related to the deleted thumbnails - (let [result (th/run-task! :storage-gc-deleted {:min-age 0})] - (t/is (= 1 (:deleted result)))) + (binding [ct/*clock* (clock/fixed (ct/in-future {:days 8}))] + (let [result (th/run-task! :storage-gc-deleted {})] + (t/is (= 1 (:deleted result))))) (t/is (some? (sto/get-object storage (:media-id row2))))))) diff --git a/backend/test/backend_tests/rpc_font_test.clj b/backend/test/backend_tests/rpc_font_test.clj index 08611426df..ef19218314 100644 --- a/backend/test/backend_tests/rpc_font_test.clj +++ b/backend/test/backend_tests/rpc_font_test.clj @@ -6,11 +6,13 @@ (ns backend-tests.rpc-font-test (:require + [app.common.time :as ct] [app.common.uuid :as uuid] [app.config :as cf] [app.db :as db] [app.http :as http] [app.rpc :as-alias rpc] + [app.setup.clock :as clock] [app.storage :as sto] [backend-tests.helpers :as th] [clojure.test :as t] @@ -129,7 +131,7 @@ ;; (th/print-result! out) (t/is (nil? (:error out)))) - (let [res (th/run-task! :storage-gc-touched {:min-age 0})] + (let [res (th/run-task! :storage-gc-touched {})] (t/is (= 6 (:freeze res)))) (let [params {::th/type :delete-font @@ -141,16 +143,17 @@ (t/is (nil? (:error out))) (t/is (nil? (:result out)))) - (let [res (th/run-task! :storage-gc-touched {:min-age 0})] + (let [res (th/run-task! :storage-gc-touched {})] (t/is (= 0 (:freeze res))) (t/is (= 0 (:delete res)))) - (let [res (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})] - (t/is (= 2 (:processed res)))) + (binding [ct/*clock* (clock/fixed (ct/in-future {:days 8}))] + (let [res (th/run-task! :objects-gc {})] + (t/is (= 2 (:processed res)))) - (let [res (th/run-task! :storage-gc-touched {:min-age 0})] - (t/is (= 0 (:freeze res))) - (t/is (= 6 (:delete res)))))) + (let [res (th/run-task! :storage-gc-touched {})] + (t/is (= 0 (:freeze res))) + (t/is (= 6 (:delete res))))))) (t/deftest font-deletion-2 (let [prof (th/create-profile* 1 {:is-active true}) @@ -189,7 +192,7 @@ ;; (th/print-result! out) (t/is (nil? (:error out)))) - (let [res (th/run-task! :storage-gc-touched {:min-age 0})] + (let [res (th/run-task! :storage-gc-touched {})] (t/is (= 6 (:freeze res)))) (let [params {::th/type :delete-font @@ -201,16 +204,17 @@ (t/is (nil? (:error out))) (t/is (nil? (:result out)))) - (let [res (th/run-task! :storage-gc-touched {:min-age 0})] + (let [res (th/run-task! :storage-gc-touched {})] (t/is (= 0 (:freeze res))) (t/is (= 0 (:delete res)))) - (let [res (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})] - (t/is (= 1 (:processed res)))) + (binding [ct/*clock* (clock/fixed (ct/in-future {:days 8}))] + (let [res (th/run-task! :objects-gc {})] + (t/is (= 1 (:processed res)))) - (let [res (th/run-task! :storage-gc-touched {:min-age 0})] - (t/is (= 0 (:freeze res))) - (t/is (= 3 (:delete res)))))) + (let [res (th/run-task! :storage-gc-touched {})] + (t/is (= 0 (:freeze res))) + (t/is (= 3 (:delete res))))))) (t/deftest font-deletion-3 (let [prof (th/create-profile* 1 {:is-active true}) @@ -248,7 +252,7 @@ (t/is (nil? (:error out1))) (t/is (nil? (:error out2))) - (let [res (th/run-task! :storage-gc-touched {:min-age 0})] + (let [res (th/run-task! :storage-gc-touched {})] (t/is (= 6 (:freeze res)))) (let [params {::th/type :delete-font-variant @@ -260,13 +264,14 @@ (t/is (nil? (:error out))) (t/is (nil? (:result out)))) - (let [res (th/run-task! :storage-gc-touched {:min-age 0})] + (let [res (th/run-task! :storage-gc-touched {})] (t/is (= 0 (:freeze res))) (t/is (= 0 (:delete res)))) - (let [res (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})] - (t/is (= 1 (:processed res)))) + (binding [ct/*clock* (clock/fixed (ct/in-future {:days 8}))] + (let [res (th/run-task! :objects-gc {})] + (t/is (= 1 (:processed res)))) - (let [res (th/run-task! :storage-gc-touched {:min-age 0})] - (t/is (= 0 (:freeze res))) - (t/is (= 3 (:delete res)))))) + (let [res (th/run-task! :storage-gc-touched {})] + (t/is (= 0 (:freeze res))) + (t/is (= 3 (:delete res))))))) diff --git a/backend/test/backend_tests/rpc_project_test.clj b/backend/test/backend_tests/rpc_project_test.clj index 6f4704e973..6c75543f05 100644 --- a/backend/test/backend_tests/rpc_project_test.clj +++ b/backend/test/backend_tests/rpc_project_test.clj @@ -6,11 +6,13 @@ (ns backend-tests.rpc-project-test (:require + [app.common.time :as ct] [app.common.uuid :as uuid] [app.config :as cf] [app.db :as db] [app.http :as http] [app.rpc :as-alias rpc] + [app.setup.clock :as clock] [backend-tests.helpers :as th] [clojure.test :as t])) @@ -226,8 +228,9 @@ (t/is (= 0 (count result))))) ;; run permanent deletion - (let [result (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})] - (t/is (= 1 (:processed result)))) + (binding [ct/*clock* (clock/fixed (ct/in-future {:days 8}))] + (let [result (th/run-task! :objects-gc {})] + (t/is (= 1 (:processed result))))) ;; query the list of files of a after hard deletion (let [data {::th/type :get-project-files diff --git a/backend/test/backend_tests/rpc_team_test.clj b/backend/test/backend_tests/rpc_team_test.clj index 7df7ab7d95..12791e761a 100644 --- a/backend/test/backend_tests/rpc_team_test.clj +++ b/backend/test/backend_tests/rpc_team_test.clj @@ -13,6 +13,7 @@ [app.db :as db] [app.http :as http] [app.rpc :as-alias rpc] + [app.setup.clock :as clock] [app.storage :as sto] [app.tokens :as tokens] [backend-tests.helpers :as th] @@ -525,8 +526,9 @@ (t/is (= :not-found (:type edata))))) ;; run permanent deletion - (let [result (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})] - (t/is (= 2 (:processed result)))) + (binding [ct/*clock* (clock/fixed (ct/in-future {:days 8}))] + (let [result (th/run-task! :objects-gc {})] + (t/is (= 2 (:processed result))))) ;; query the list of projects of a after hard deletion (let [data {::th/type :get-projects @@ -581,8 +583,9 @@ (t/is (= 1 (count rows))) (t/is (ct/inst? (:deleted-at (first rows))))) - (let [result (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})] - (t/is (= 7 (:processed result)))))) + (binding [ct/*clock* (clock/fixed (ct/in-future {:days 8}))] + (let [result (th/run-task! :objects-gc {})] + (t/is (= 7 (:processed result))))))) (t/deftest create-team-access-request (with-mocks [mock {:target 'app.email/send! :return nil}] diff --git a/backend/test/backend_tests/storage_test.clj b/backend/test/backend_tests/storage_test.clj index d282b20985..1377dc4f39 100644 --- a/backend/test/backend_tests/storage_test.clj +++ b/backend/test/backend_tests/storage_test.clj @@ -11,6 +11,7 @@ [app.common.uuid :as uuid] [app.db :as db] [app.rpc :as-alias rpc] + [app.setup.clock :as clock] [app.storage :as sto] [backend-tests.helpers :as th] [clojure.test :as t] @@ -53,19 +54,13 @@ (configure-storage-backend)) content (sto/content "content") object (sto/put-object! storage {::sto/content content - ::sto/expired-at (ct/in-future {:seconds 1}) + ::sto/expired-at (ct/in-future {:hours 1}) :content-type "text/plain"})] (t/is (sto/object? object)) (t/is (ct/inst? (:expired-at object))) (t/is (ct/is-after? (:expired-at object) (ct/now))) - (t/is (= object (sto/get-object storage (:id object)))) - - (th/sleep 1000) - (t/is (nil? (sto/get-object storage (:id object)))) - (t/is (nil? (sto/get-object-data storage object))) - (t/is (nil? (sto/get-object-url storage object))) - (t/is (nil? (sto/get-object-path storage object))))) + (t/is (nil? (sto/get-object storage (:id object)))))) (t/deftest put-and-delete-object (let [storage (-> (:app.storage/storage th/*system*) @@ -98,20 +93,25 @@ ::sto/expired-at (ct/now) :content-type "text/plain"}) object2 (sto/put-object! storage {::sto/content content2 - ::sto/expired-at (ct/in-past {:hours 2}) + ::sto/expired-at (ct/in-future {:hours 2}) :content-type "text/plain"}) object3 (sto/put-object! storage {::sto/content content3 - ::sto/expired-at (ct/in-past {:hours 1}) + ::sto/expired-at (ct/in-future {:hours 1}) :content-type "text/plain"})] - - (th/sleep 200) - - (let [res (th/run-task! :storage-gc-deleted {})] - (t/is (= 1 (:deleted res)))) + (binding [ct/*clock* (clock/fixed (ct/in-future {:minutes 0}))] + (let [res (th/run-task! :storage-gc-deleted {})] + (t/is (= 1 (:deleted res))))) (let [res (th/db-exec-one! ["select count(*) from storage_object;"])] - (t/is (= 2 (:count res)))))) + (t/is (= 2 (:count res)))) + + (binding [ct/*clock* (clock/fixed (ct/in-future {:minutes 61}))] + (let [res (th/run-task! :storage-gc-deleted {})] + (t/is (= 1 (:deleted res))))) + + (let [res (th/db-exec-one! ["select count(*) from storage_object;"])] + (t/is (= 1 (:count res)))))) (t/deftest touched-gc-task-1 (let [storage (-> (:app.storage/storage th/*system*) @@ -158,7 +158,7 @@ {:id (:id result-1)}) ;; run the objects gc task for permanent deletion - (let [res (th/run-task! :objects-gc {:min-age 0})] + (let [res (th/run-task! :objects-gc {})] (t/is (= 1 (:processed res)))) ;; check that we still have all the storage objects @@ -182,7 +182,6 @@ (let [res (th/db-exec-one! ["select count(*) from storage_object where deleted_at is not null"])] (t/is (= 0 (:count res))))))) - (t/deftest touched-gc-task-2 (let [storage (-> (:app.storage/storage th/*system*) (configure-storage-backend)) @@ -243,11 +242,12 @@ {:id (:id result-2)}) ;; run the objects gc task for permanent deletion - (let [res (th/run-task! :objects-gc {:min-age 0})] + (let [res (th/run-task! :objects-gc {})] (t/is (= 1 (:processed res)))) ;; revert touched state to all storage objects - (th/db-exec-one! ["update storage_object set touched_at=now()"]) + + (th/db-exec-one! ["update storage_object set touched_at=?" (ct/now)]) ;; Run the task again (let [res (th/run-task! :storage-gc-touched {})] @@ -293,10 +293,10 @@ result-2 (:result out2)] ;; now we proceed to manually mark all storage objects touched - (th/db-exec! ["update storage_object set touched_at=now()"]) + (th/db-exec! ["update storage_object set touched_at=?" (ct/now)]) ;; run the touched gc task - (let [res (th/run-task! "storage-gc-touched" {:min-age 0})] + (let [res (th/run-task! :storage-gc-touched {})] (t/is (= 2 (:freeze res))) (t/is (= 0 (:delete res)))) @@ -305,13 +305,13 @@ (t/is (= 2 (count rows))))) ;; now we proceed to manually delete all file_media_object - (th/db-exec! ["update file_media_object set deleted_at = now()"]) + (th/db-exec! ["update file_media_object set deleted_at = ?" (ct/now)]) - (let [res (th/run-task! "objects-gc" {:min-age 0})] + (let [res (th/run-task! :objects-gc {})] (t/is (= 2 (:processed res)))) ;; run the touched gc task - (let [res (th/run-task! "storage-gc-touched" {:min-age 0})] + (let [res (th/run-task! :storage-gc-touched {})] (t/is (= 0 (:freeze res))) (t/is (= 2 (:delete res))))