♻️ Make storage and other objects deletion task vclock aware

This simplifes the mental model on how it works and simplifies testing
of the related code.

This also normalizes storage object deletion in the same way as the
rest of objects in penpot (now future deletion date on storage object
also means storage object to be deleted).

backend/src/app/storage.clj

@@ -163,9 +163,6 @@
      backend
      (:metadata result))))
 
-(def ^:private sql:retrieve-storage-object
-  "select * from storage_object where id = ? and (deleted_at is null or deleted_at > now())")
-
 (defn row->storage-object [res]
   (let [mdata (or (some-> (:metadata res) (db/decode-transit-pgobject)) {})]
     (impl/storage-object
@@ -177,9 +174,15 @@
      (keyword (:backend res))
      mdata)))
 
-(defn- retrieve-database-object
+(def ^:private sql:get-storage-object
+  "SELECT *
+     FROM storage_object
+    WHERE id = ?
+      AND (deleted_at IS NULL)")
+
+(defn- get-database-object
   [conn id]
-  (some-> (db/exec-one! conn [sql:retrieve-storage-object id])
+  (some-> (db/exec-one! conn [sql:get-storage-object id])
           (row->storage-object)))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -202,7 +205,7 @@
 (defn get-object
   [{:keys [::db/connectable] :as storage}  id]
   (assert (valid-storage? storage))
-  (retrieve-database-object connectable id))
+  (get-database-object connectable id))
 
 (defn put-object!
   "Creates a new object with the provided content."

backend/src/app/storage/gc_deleted.clj

@@ -37,7 +37,6 @@
          (into #{} (map :id))
          (not-empty))))
 
-
 (def ^:private sql:delete-sobjects
   "DELETE FROM storage_object
     WHERE id = ANY(?::uuid[])")
@@ -77,47 +76,37 @@
   (d/group-by (comp keyword :backend) :id #{} items))
 
 (def ^:private sql:get-deleted-sobjects
-  "SELECT s.* FROM storage_object AS s
+  "SELECT s.*
+     FROM storage_object AS s
     WHERE s.deleted_at IS NOT NULL
-      AND s.deleted_at < now() - ?::interval
+      AND s.deleted_at <= ?
     ORDER BY s.deleted_at ASC")
 
 (defn- get-buckets
-  [conn min-age]
+  [conn]
-  (let [age (db/interval min-age)]
+  (let [now (ct/now)]
     (sequence
      (comp (partition-all 25)
            (mapcat group-by-backend))
-     (db/cursor conn [sql:get-deleted-sobjects age]))))
+     (db/cursor conn [sql:get-deleted-sobjects now]))))
-
 
 (defn- clean-deleted!
-  [{:keys [::db/conn ::min-age] :as cfg}]
+  [{:keys [::db/conn] :as cfg}]
   (reduce (fn [total [backend-id ids]]
             (let [deleted (delete-in-bulk! cfg backend-id ids)]
               (+ total (or deleted 0))))
           0
-          (get-buckets conn min-age)))
+          (get-buckets conn)))
 
 (defmethod ig/assert-key ::handler
   [_ params]
   (assert (sto/valid-storage? (::sto/storage params)) "expect valid storage")
   (assert (db/pool? (::db/pool params)) "expect valid storage"))
 
-(defmethod ig/expand-key ::handler
-  [k v]
-  {k (assoc v ::min-age (ct/duration {:hours 2}))})
-
 (defmethod ig/init-key ::handler
-  [_ {:keys [::min-age] :as cfg}]
+  [_ cfg]
-  (fn [{:keys [props] :as task}]
+  (fn [_]
-    (let [min-age (ct/duration (or (:min-age props) min-age))]
+    (db/tx-run! cfg (fn [cfg]
-      (db/tx-run! cfg (fn [cfg]
+                      (let [total (clean-deleted! cfg)]
-                        (let [cfg   (assoc cfg ::min-age min-age)
+                        (l/inf :hint "task finished" :total total)
-                              total (clean-deleted! cfg)]
+                        {:deleted total})))))
-
-                          (l/inf :hint "task finished"
-                                 :min-age (ct/format-duration min-age)
-                                 :total total)
-
-                          {:deleted total}))))))

backend/src/app/storage/gc_touched.clj

@@ -22,6 +22,8 @@
    [app.common.data.macros :as dm]
    [app.common.exceptions :as ex]
    [app.common.logging :as l]
+   [app.common.time :as ct]
+   [app.config :as cf]
    [app.db :as db]
    [app.storage :as-alias sto]
    [app.storage.impl :as impl]
@@ -101,14 +103,15 @@
 
 (def ^:private sql:mark-delete-in-bulk
   "UPDATE storage_object
-      SET deleted_at = now(),
+      SET deleted_at = ?,
           touched_at = NULL
     WHERE id = ANY(?::uuid[])")
 
 (defn- mark-delete-in-bulk!
-  [conn ids]
+  [conn deletion-delay ids]
-  (let [ids (db/create-array conn "uuid" ids)]
+  (let [ids (db/create-array conn "uuid" ids)
-    (db/exec-one! conn [sql:mark-delete-in-bulk ids])))
+        now (ct/plus (ct/now) deletion-delay)]
+    (db/exec-one! conn [sql:mark-delete-in-bulk now ids])))
 
 ;; NOTE: A getter that retrieves the key which will be used for group
 ;; ids; previously we have no value, then we introduced the
@@ -137,18 +140,20 @@
     (if-let [{:keys [id] :as object} (first objects)]
       (if (has-refs? conn object)
         (do
-          (l/debug :id (str id)
+          (l/dbg :id (str id)
-                   :status "freeze"
+                 :status "freeze"
-                   :bucket bucket)
+                 :bucket bucket)
           (recur (conj to-freeze id) to-delete (rest objects)))
         (do
-          (l/debug :id (str id)
+          (l/dbg :id (str id)
-                   :status "delete"
+                 :status "delete"
-                   :bucket bucket)
+                 :bucket bucket)
           (recur to-freeze (conj to-delete id) (rest objects))))
-      (do
+      (let [deletion-delay (if (= bucket "tempfile")
+                             (ct/duration {:hours 2})
+                             (cf/get-deletion-delay))]
         (some->> (seq to-freeze) (mark-freeze-in-bulk! conn))
-        (some->> (seq to-delete) (mark-delete-in-bulk! conn))
+        (some->> (seq to-delete) (mark-delete-in-bulk! conn deletion-delay))
         [(count to-freeze) (count to-delete)]))))
 
 (defn- process-bucket!
@@ -173,27 +178,27 @@
              [0 0]
              (d/group-by lookup-bucket identity #{} chunk)))
 
-(def ^:private
+(def ^:private sql:get-touched-storage-objects
-  sql:get-touched-storage-objects
   "SELECT so.*
      FROM storage_object AS so
     WHERE so.touched_at IS NOT NULL
+      AND so.touched_at <= ?
     ORDER BY touched_at ASC
       FOR UPDATE
      SKIP LOCKED
     LIMIT 10")
 
 (defn get-chunk
-  [conn]
+  [conn timestamp]
-  (->> (db/exec! conn [sql:get-touched-storage-objects])
+  (->> (db/exec! conn [sql:get-touched-storage-objects timestamp])
        (map impl/decode-row)
        (not-empty)))
 
 (defn- process-touched!
-  [{:keys [::db/pool] :as cfg}]
+  [{:keys [::db/pool ::timestamp] :as cfg}]
   (loop [freezed 0
          deleted 0]
-    (if-let [chunk (get-chunk pool)]
+    (if-let [chunk (get-chunk pool timestamp)]
       (let [[nfo ndo] (db/tx-run! cfg process-chunk! chunk)]
         (recur (long (+ freezed nfo))
                (long (+ deleted ndo))))
@@ -209,5 +214,6 @@
 
 (defmethod ig/init-key ::handler
   [_ cfg]
-  (fn [_] (process-touched! cfg)))
+  (fn [_]
+    (process-touched! (assoc cfg ::timestamp (ct/now)))))
 

backend/src/app/tasks/objects_gc.clj

@@ -18,15 +18,15 @@
 (def ^:private sql:get-profiles
   "SELECT id, photo_id FROM profile
     WHERE deleted_at IS NOT NULL
-      AND deleted_at < now() + ?::interval
+      AND deleted_at <= ?
     ORDER BY deleted_at ASC
     LIMIT ?
       FOR UPDATE
      SKIP LOCKED")
 
 (defn- delete-profiles!
-  [{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}]
+  [{:keys [::db/conn ::timestamp ::chunk-size ::sto/storage] :as cfg}]
-  (->> (db/plan conn [sql:get-profiles deletion-threshold chunk-size] {:fetch-size 5})
+  (->> (db/plan conn [sql:get-profiles timestamp chunk-size] {:fetch-size 5})
        (reduce (fn [total {:keys [id photo-id]}]
                  (l/trc :obj "profile" :id (str id))
 
@@ -41,15 +41,15 @@
 (def ^:private sql:get-teams
   "SELECT deleted_at, id, photo_id FROM team
     WHERE deleted_at IS NOT NULL
-      AND deleted_at < now() + ?::interval
+      AND deleted_at <= ?
     ORDER BY deleted_at ASC
     LIMIT ?
       FOR UPDATE
      SKIP LOCKED")
 
 (defn- delete-teams!
-  [{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}]
+  [{:keys [::db/conn ::timestamp ::chunk-size ::sto/storage] :as cfg}]
-  (->> (db/plan conn [sql:get-teams deletion-threshold chunk-size] {:fetch-size 5})
+  (->> (db/plan conn [sql:get-teams timestamp chunk-size] {:fetch-size 5})
        (reduce (fn [total {:keys [id photo-id deleted-at]}]
                  (l/trc :obj "team"
                         :id (str id)
@@ -68,15 +68,15 @@
   "SELECT id, team_id, deleted_at, woff1_file_id, woff2_file_id, otf_file_id, ttf_file_id
      FROM team_font_variant
     WHERE deleted_at IS NOT NULL
-      AND deleted_at < now() + ?::interval
+      AND deleted_at <= ?
     ORDER BY deleted_at ASC
     LIMIT ?
       FOR UPDATE
      SKIP LOCKED")
 
 (defn- delete-fonts!
-  [{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}]
+  [{:keys [::db/conn ::timestamp ::chunk-size ::sto/storage] :as cfg}]
-  (->> (db/plan conn [sql:get-fonts deletion-threshold chunk-size] {:fetch-size 5})
+  (->> (db/plan conn [sql:get-fonts timestamp chunk-size] {:fetch-size 5})
        (reduce (fn [total {:keys [id team-id deleted-at] :as font}]
                  (l/trc :obj "font-variant"
                         :id (str id)
@@ -98,15 +98,15 @@
   "SELECT id, deleted_at, team_id
      FROM project
     WHERE deleted_at IS NOT NULL
-      AND deleted_at < now() + ?::interval
+      AND deleted_at <= ?
     ORDER BY deleted_at ASC
     LIMIT ?
       FOR UPDATE
      SKIP LOCKED")
 
 (defn- delete-projects!
-  [{:keys [::db/conn ::deletion-threshold ::chunk-size] :as cfg}]
+  [{:keys [::db/conn ::timestamp ::chunk-size] :as cfg}]
-  (->> (db/plan conn [sql:get-projects deletion-threshold chunk-size] {:fetch-size 5})
+  (->> (db/plan conn [sql:get-projects timestamp chunk-size] {:fetch-size 5})
        (reduce (fn [total {:keys [id team-id deleted-at]}]
                  (l/trc :obj "project"
                         :id (str id)
@@ -124,15 +124,15 @@
           f.project_id
      FROM file AS f
     WHERE f.deleted_at IS NOT NULL
-      AND f.deleted_at < now() + ?::interval
+      AND f.deleted_at <= ?
     ORDER BY f.deleted_at ASC
     LIMIT ?
       FOR UPDATE
      SKIP LOCKED")
 
 (defn- delete-files!
-  [{:keys [::db/conn ::deletion-threshold ::chunk-size] :as cfg}]
+  [{:keys [::db/conn ::timestamp ::chunk-size] :as cfg}]
-  (->> (db/plan conn [sql:get-files deletion-threshold chunk-size] {:fetch-size 5})
+  (->> (db/plan conn [sql:get-files timestamp chunk-size] {:fetch-size 5})
        (reduce (fn [total {:keys [id deleted-at project-id] :as file}]
                  (l/trc :obj "file"
                         :id (str id)
@@ -148,15 +148,15 @@
   "SELECT file_id, revn, media_id, deleted_at
      FROM file_thumbnail
     WHERE deleted_at IS NOT NULL
-      AND deleted_at < now() + ?::interval
+      AND deleted_at <= ?
     ORDER BY deleted_at ASC
     LIMIT ?
       FOR UPDATE
      SKIP LOCKED")
 
 (defn delete-file-thumbnails!
-  [{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}]
+  [{:keys [::db/conn ::timestamp ::chunk-size ::sto/storage] :as cfg}]
-  (->> (db/plan conn [sql:get-file-thumbnails deletion-threshold chunk-size] {:fetch-size 5})
+  (->> (db/plan conn [sql:get-file-thumbnails timestamp chunk-size] {:fetch-size 5})
        (reduce (fn [total {:keys [file-id revn media-id deleted-at]}]
                  (l/trc :obj "file-thumbnail"
                         :file-id (str file-id)
@@ -175,15 +175,15 @@
   "SELECT file_id, object_id, media_id, deleted_at
      FROM file_tagged_object_thumbnail
     WHERE deleted_at IS NOT NULL
-      AND deleted_at < now() + ?::interval
+      AND deleted_at <= ?
     ORDER BY deleted_at ASC
     LIMIT ?
       FOR UPDATE
      SKIP LOCKED")
 
 (defn delete-file-object-thumbnails!
-  [{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}]
+  [{:keys [::db/conn ::timestamp ::chunk-size ::sto/storage] :as cfg}]
-  (->> (db/plan conn [sql:get-file-object-thumbnails deletion-threshold chunk-size] {:fetch-size 5})
+  (->> (db/plan conn [sql:get-file-object-thumbnails timestamp chunk-size] {:fetch-size 5})
        (reduce (fn [total {:keys [file-id object-id media-id deleted-at]}]
                  (l/trc :obj "file-object-thumbnail"
                         :file-id (str file-id)
@@ -203,15 +203,15 @@
   "SELECT id, file_id, media_id, thumbnail_id, deleted_at
      FROM file_media_object
     WHERE deleted_at IS NOT NULL
-      AND deleted_at < now() + ?::interval
+      AND deleted_at <= ?
     ORDER BY deleted_at ASC
     LIMIT ?
       FOR UPDATE
      SKIP LOCKED")
 
 (defn- delete-file-media-objects!
-  [{:keys [::db/conn ::deletion-threshold ::chunk-size ::sto/storage] :as cfg}]
+  [{:keys [::db/conn ::timestamp ::chunk-size ::sto/storage] :as cfg}]
-  (->> (db/plan conn [sql:get-file-media-objects deletion-threshold chunk-size] {:fetch-size 5})
+  (->> (db/plan conn [sql:get-file-media-objects timestamp chunk-size] {:fetch-size 5})
        (reduce (fn [total {:keys [id file-id deleted-at] :as fmo}]
                  (l/trc :obj "file-media-object"
                         :id (str id)
@@ -231,16 +231,15 @@
   "SELECT file_id, id, type, deleted_at, metadata, backend
      FROM file_data
     WHERE deleted_at IS NOT NULL
-      AND deleted_at < now() + ?::interval
+      AND deleted_at <= ?
     ORDER BY deleted_at ASC
     LIMIT ?
       FOR UPDATE
      SKIP LOCKED")
 
 (defn- delete-file-data!
-  [{:keys [::db/conn ::deletion-threshold ::chunk-size] :as cfg}]
+  [{:keys [::db/conn ::timestamp ::chunk-size] :as cfg}]
-
+  (->> (db/plan conn [sql:get-file-data timestamp chunk-size] {:fetch-size 5})
-  (->> (db/plan conn [sql:get-file-data deletion-threshold chunk-size] {:fetch-size 5})
        (reduce (fn [total {:keys [file-id id type deleted-at metadata backend]}]
 
                  (some->> metadata
@@ -266,15 +265,15 @@
   "SELECT id, file_id, deleted_at
      FROM file_change
     WHERE deleted_at IS NOT NULL
-      AND deleted_at < now() + ?::interval
+      AND deleted_at <= ?
     ORDER BY deleted_at ASC
     LIMIT ?
       FOR UPDATE
      SKIP LOCKED")
 
 (defn- delete-file-changes!
-  [{:keys [::db/conn ::deletion-threshold ::chunk-size] :as cfg}]
+  [{:keys [::db/conn ::timestamp ::chunk-size] :as cfg}]
-  (->> (db/plan conn [sql:get-file-change deletion-threshold chunk-size] {:fetch-size 5})
+  (->> (db/plan conn [sql:get-file-change timestamp chunk-size] {:fetch-size 5})
        (reduce (fn [total {:keys [id file-id deleted-at] :as xlog}]
                  (l/trc :obj "file-change"
                         :id (str id)
@@ -322,9 +321,8 @@
 
 (defmethod ig/init-key ::handler
   [_ cfg]
-  (fn [{:keys [props] :as task}]
+  (fn [_]
-    (let [threshold (ct/duration (get props :deletion-threshold 0))
+    (let [cfg (assoc cfg ::timestamp (ct/now))]
-          cfg       (assoc cfg ::deletion-threshold (db/interval threshold))]
       (loop [procs (map deref deletion-proc-vars)
              total 0]
         (if-let [proc-fn (first procs)]

backend/test/backend_tests/rpc_file_snapshot_test.clj

@@ -9,6 +9,7 @@
    [app.common.features :as cfeat]
    [app.common.pprint :as pp]
    [app.common.thumbnails :as thc]
+   [app.common.time :as ct]
    [app.common.types.shape :as cts]
    [app.common.uuid :as uuid]
    [app.config :as cf]
@@ -16,6 +17,7 @@
    [app.db.sql :as sql]
    [app.http :as http]
    [app.rpc :as-alias rpc]
+   [app.setup.clock :as clock]
    [app.storage :as sto]
    [backend-tests.helpers :as th]
    [clojure.test :as t]
@@ -132,9 +134,10 @@
         ;; this will run pending task triggered by deleting user snapshot
         (th/run-pending-tasks!)
 
-        (let [res (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})]
+        (binding [ct/*clock* (clock/fixed (ct/in-future {:days 8}))]
-          ;; delete 2 snapshots and 2 file data entries
+          (let [res (th/run-task! :objects-gc {})]
-          (t/is (= 4 (:processed res))))))))
+            ;; delete 2 snapshots and 2 file data entries
+            (t/is (= 4 (:processed res)))))))))
 
 (t/deftest snapshots-locking
   (let [profile-1 (th/create-profile* 1 {:is-active true})

backend/test/backend_tests/rpc_file_test.clj

@@ -313,7 +313,7 @@
       ;; freeze because of the deduplication (we have uploaded 2 times
       ;; the same files).
 
-      (let [res (th/run-task! :storage-gc-touched {:min-age 0})]
+      (let [res (th/run-task! :storage-gc-touched {})]
         (t/is (= 2 (:freeze res)))
         (t/is (= 0 (:delete res))))
 
@@ -372,14 +372,14 @@
       (th/db-exec! ["update file_change set deleted_at = now() where file_id = ? and label is not null" (:id file)])
       (th/db-exec! ["update file set has_media_trimmed = false where id = ?" (:id file)])
 
-      (let [res (th/run-task! :objects-gc {:deletion-threshold 0})]
+      (let [res (th/run-task! :objects-gc {})]
         ;; this will remove the file change and file data entries for two snapshots
         (t/is (= 4 (:processed res))))
 
       ;; Rerun the file-gc and objects-gc
       (t/is (true? (th/run-task! :file-gc {:file-id (:id file)})))
 
-      (let [res (th/run-task! :objects-gc {:deletion-threshold 0})]
+      (let [res (th/run-task! :objects-gc {})]
         ;; this will remove the file media objects marked as deleted
         ;; on prev file-gc
         (t/is (= 2 (:processed res))))
@@ -387,7 +387,7 @@
       ;; Now that file-gc have deleted the file-media-object usage,
       ;; lets execute the touched-gc task, we should see that two of
       ;; them are marked to be deleted
-      (let [res (th/run-task! :storage-gc-touched {:min-age 0})]
+      (let [res (th/run-task! :storage-gc-touched {})]
         (t/is (= 0 (:freeze res)))
         (t/is (= 2 (:delete res))))
 
@@ -572,7 +572,7 @@
       ;; Now that file-gc have deleted the file-media-object usage,
       ;; lets execute the touched-gc task, we should see that two of
       ;; them are marked to be deleted.
-      (let [res (th/run-task! :storage-gc-touched {:min-age 0})]
+      (let [res (th/run-task! :storage-gc-touched {})]
         (t/is (= 0 (:freeze res)))
         (t/is (= 2 (:delete res))))
 
@@ -665,7 +665,7 @@
       ;; because of the deduplication (we have uploaded 2 times the
       ;; same files).
 
-      (let [res (th/run-task! :storage-gc-touched {:min-age 0})]
+      (let [res (th/run-task! :storage-gc-touched {})]
         (t/is (= 1 (:freeze res)))
         (t/is (= 0 (:delete res))))
 
@@ -715,7 +715,7 @@
 
       ;; Now that objects-gc have deleted the object thumbnail lets
       ;; execute the touched-gc task
-      (let [res (th/run-task! "storage-gc-touched" {:min-age 0})]
+      (let [res (th/run-task! "storage-gc-touched" {})]
         (t/is (= 1 (:freeze res))))
 
       ;; check file media objects
@@ -750,7 +750,7 @@
 
       ;; Now that file-gc have deleted the object thumbnail lets
       ;; execute the touched-gc task
-      (let [res (th/run-task! :storage-gc-touched {:min-age 0})]
+      (let [res (th/run-task! :storage-gc-touched {})]
         (t/is (= 1 (:delete res))))
 
       ;; check file media objects
@@ -922,8 +922,9 @@
       (t/is (= 0 (:processed result))))
 
     ;; run permanent deletion
-    (let [result (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})]
+    (binding [ct/*clock* (clock/fixed (ct/in-future {:days 8}))]
-      (t/is (= 3 (:processed result))))
+      (let [result (th/run-task! :objects-gc {})]
+        (t/is (= 3 (:processed result)))))
 
     ;; query the list of file libraries of a after hard deletion
     (let [data {::th/type :get-file-libraries
@@ -1134,7 +1135,7 @@
       (th/sleep 300)
 
       ;; run the task
-      (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
+      (t/is (true? (th/run-task! :file-gc {:file-id (:id file)})))
 
       ;; check that object thumbnails are still here
       (let [rows (th/db-query :file-tagged-object-thumbnail {:file-id (:id file)})]
@@ -1163,7 +1164,7 @@
         (t/is (= 2 (count rows))))
 
       ;; run the task again
-      (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
+      (t/is (true? (th/run-task! :file-gc {:file-id (:id file)})))
 
       ;; check that we have all object thumbnails
       (let [rows (th/db-query :file-tagged-object-thumbnail {:file-id (:id file)})]
@@ -1226,7 +1227,7 @@
         (t/is (= 2 (count rows)))))
 
     (t/testing "gc task"
-      (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
+      (t/is (true? (th/run-task! :file-gc {:file-id (:id file)})))
 
       (let [rows (th/db-query :file-thumbnail {:file-id (:id file)})]
         (t/is (= 2 (count rows)))
@@ -1273,7 +1274,7 @@
     ;; The FileGC task will schedule an inner taskq
     (th/run-pending-tasks!)
 
-    (let [res (th/run-task! :storage-gc-touched {:min-age 0})]
+    (let [res (th/run-task! :storage-gc-touched {})]
       (t/is (= 2 (:freeze res)))
       (t/is (= 0 (:delete res))))
 
@@ -1367,7 +1368,7 @@
 
     ;; we ensure that once object-gc is passed and marked two storage
     ;; objects to delete
-    (let [res (th/run-task! :storage-gc-touched {:min-age 0})]
+    (let [res (th/run-task! :storage-gc-touched {})]
       (t/is (= 0 (:freeze res)))
       (t/is (= 2 (:delete res))))
 
@@ -1489,7 +1490,7 @@
         (t/is (some? (not-empty (:objects component))))))
 
     ;; Re-run the file-gc task
-    (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
+    (t/is (true? (th/run-task! :file-gc {:file-id (:id file)})))
     (let [row (th/db-get :file {:id (:id file)})]
       (t/is (true? (:has-media-trimmed row))))
 
@@ -1519,7 +1520,7 @@
 
     ;; Now, we have deleted the usage of component if we pass file-gc,
     ;; that component should be deleted
-    (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
+    (t/is (true? (th/run-task! :file-gc {:file-id (:id file)})))
 
     ;; Check that component is properly removed
     (let [data {::th/type :get-file
@@ -1610,8 +1611,8 @@
               :component-id c-id})}])
 
     ;; Run the file-gc on file and library
-    (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file-1)})))
+    (t/is (true? (th/run-task! :file-gc {:file-id (:id file-1)})))
-    (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file-2)})))
+    (t/is (true? (th/run-task! :file-gc {:file-id (:id file-2)})))
 
     ;; Check that component exists
     (let [data {::th/type :get-file
@@ -1684,7 +1685,7 @@
 
     ;; Now, we have deleted the usage of component if we pass file-gc,
     ;; that component should be deleted
-    (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file-1)})))
+    (t/is (true? (th/run-task! :file-gc {:file-id (:id file-1)})))
 
     ;; Check that component is properly removed
     (let [data {::th/type :get-file
@@ -1833,8 +1834,8 @@
         (t/is (not= (:id fill) (:id fmedia)))))
 
     ;; Run the file-gc on file and library
-    (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file-1)})))
+    (t/is (true? (th/run-task! :file-gc {:file-id (:id file-1)})))
-    (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file-2)})))
+    (t/is (true? (th/run-task! :file-gc {:file-id (:id file-2)})))
 
     ;; Now proceed to delete file and absorb it
     (let [data {::th/type :delete-file

backend/test/backend_tests/rpc_file_thumbnails_test.clj

@@ -8,12 +8,14 @@
   (:require
    [app.common.pprint :as pp]
    [app.common.thumbnails :as thc]
+   [app.common.time :as ct]
    [app.common.types.shape :as cts]
    [app.common.uuid :as uuid]
    [app.config :as cf]
    [app.db :as db]
    [app.rpc :as-alias rpc]
    [app.rpc.commands.auth :as cauth]
+   [app.setup.clock :as clock]
    [app.storage :as sto]
    [app.tokens :as tokens]
    [backend-tests.helpers :as th]
@@ -83,7 +85,8 @@
       (t/is (map? (:result out))))
 
     ;; run the task again
-    (let [res (th/run-task! "storage-gc-touched" {:min-age 0})]
+    (let [res (binding [ct/*clock* (clock/fixed (ct/in-future {:minutes 31}))]
+                (th/run-task! "storage-gc-touched" {}))]
       (t/is (= 2 (:freeze res))))
 
     (let [[row1 row2 :as rows] (th/db-query :file-tagged-object-thumbnail
@@ -114,9 +117,9 @@
 
       ;; Run the File GC task that should remove unused file object
       ;; thumbnails
-      (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})
+      (th/run-task! :file-gc {:file-id (:id file)})
 
-      (let [result (th/run-task! :objects-gc {:min-age 0})]
+      (let [result (th/run-task! :objects-gc {})]
         (t/is (= 3 (:processed result))))
 
       ;; check if row2 related thumbnail row still exists
@@ -133,7 +136,8 @@
       (t/is (some? (sto/get-object storage (:media-id row2))))
 
       ;; run the task again
-      (let [res (th/run-task! :storage-gc-touched {:min-age 0})]
+      (let [res (binding [ct/*clock* (clock/fixed (ct/in-future {:minutes 31}))]
+                  (th/run-task! :storage-gc-touched {}))]
         (t/is (= 1 (:delete res)))
         (t/is (= 0 (:freeze res))))
 
@@ -143,8 +147,9 @@
 
       ;; Run the storage gc deleted task, it should permanently delete
       ;; all storage objects related to the deleted thumbnails
-      (let [result (th/run-task! :storage-gc-deleted {:min-age 0})]
+      (binding [ct/*clock* (clock/fixed (ct/in-future {:days 8}))]
-        (t/is (= 1 (:deleted result))))
+        (let [res (th/run-task! :storage-gc-deleted {})]
+          (t/is (= 1 (:deleted res)))))
 
       (t/is (nil? (sto/get-object storage (:media-id row1))))
       (t/is (some? (sto/get-object storage (:media-id row2))))
@@ -216,9 +221,9 @@
 
       ;; Run the File GC task that should remove unused file object
       ;; thumbnails
-      (t/is (true? (th/run-task! :file-gc {:min-age 0 :file-id (:id file)})))
+      (t/is (true? (th/run-task! :file-gc {:file-id (:id file)})))
 
-      (let [result (th/run-task! :objects-gc {:min-age 0})]
+      (let [result (th/run-task! :objects-gc {})]
         (t/is (= 2 (:processed result))))
 
       ;; check if row1 related thumbnail row still exists
@@ -230,7 +235,7 @@
         (t/is (= (:object-id data1) (:object-id row)))
         (t/is (uuid? (:media-id row1))))
 
-      (let [result (th/run-task! :storage-gc-touched {:min-age 0})]
+      (let [result (th/run-task! :storage-gc-touched {})]
         (t/is (= 1 (:delete result))))
 
       ;; Check if storage objects still exists after file-gc
@@ -242,8 +247,9 @@
 
       ;; Run the storage gc deleted task, it should permanently delete
       ;; all storage objects related to the deleted thumbnails
-      (let [result (th/run-task! :storage-gc-deleted {:min-age 0})]
+      (binding [ct/*clock* (clock/fixed (ct/in-future {:days 8}))]
-        (t/is (= 1 (:deleted result))))
+        (let [result (th/run-task! :storage-gc-deleted {})]
+          (t/is (= 1 (:deleted result)))))
 
       (t/is (some? (sto/get-object storage (:media-id row2)))))))
 

backend/test/backend_tests/rpc_font_test.clj

@@ -6,11 +6,13 @@
 
 (ns backend-tests.rpc-font-test
   (:require
+   [app.common.time :as ct]
    [app.common.uuid :as uuid]
    [app.config :as cf]
    [app.db :as db]
    [app.http :as http]
    [app.rpc :as-alias rpc]
+   [app.setup.clock :as clock]
    [app.storage :as sto]
    [backend-tests.helpers :as th]
    [clojure.test :as t]
@@ -129,7 +131,7 @@
       ;; (th/print-result! out)
       (t/is (nil? (:error out))))
 
-    (let [res (th/run-task! :storage-gc-touched {:min-age 0})]
+    (let [res (th/run-task! :storage-gc-touched {})]
       (t/is (= 6 (:freeze res))))
 
     (let [params {::th/type :delete-font
@@ -141,16 +143,17 @@
       (t/is (nil? (:error out)))
       (t/is (nil? (:result out))))
 
-    (let [res (th/run-task! :storage-gc-touched {:min-age 0})]
+    (let [res (th/run-task! :storage-gc-touched {})]
       (t/is (= 0 (:freeze res)))
       (t/is (= 0 (:delete res))))
 
-    (let [res (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})]
+    (binding [ct/*clock* (clock/fixed (ct/in-future {:days 8}))]
-      (t/is (= 2 (:processed res))))
+      (let [res (th/run-task! :objects-gc {})]
+        (t/is (= 2 (:processed res))))
 
-    (let [res (th/run-task! :storage-gc-touched {:min-age 0})]
+      (let [res (th/run-task! :storage-gc-touched {})]
-      (t/is (= 0 (:freeze res)))
+        (t/is (= 0 (:freeze res)))
-      (t/is (= 6 (:delete res))))))
+        (t/is (= 6 (:delete res)))))))
 
 (t/deftest font-deletion-2
   (let [prof    (th/create-profile* 1 {:is-active true})
@@ -189,7 +192,7 @@
       ;; (th/print-result! out)
       (t/is (nil? (:error out))))
 
-    (let [res (th/run-task! :storage-gc-touched {:min-age 0})]
+    (let [res (th/run-task! :storage-gc-touched {})]
       (t/is (= 6 (:freeze res))))
 
     (let [params {::th/type :delete-font
@@ -201,16 +204,17 @@
       (t/is (nil? (:error out)))
       (t/is (nil? (:result out))))
 
-    (let [res (th/run-task! :storage-gc-touched {:min-age 0})]
+    (let [res (th/run-task! :storage-gc-touched {})]
       (t/is (= 0 (:freeze res)))
       (t/is (= 0 (:delete res))))
 
-    (let [res (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})]
+    (binding [ct/*clock* (clock/fixed (ct/in-future {:days 8}))]
-      (t/is (= 1 (:processed res))))
+      (let [res (th/run-task! :objects-gc {})]
+        (t/is (= 1 (:processed res))))
 
-    (let [res (th/run-task! :storage-gc-touched {:min-age 0})]
+      (let [res (th/run-task! :storage-gc-touched {})]
-      (t/is (= 0 (:freeze res)))
+        (t/is (= 0 (:freeze res)))
-      (t/is (= 3 (:delete res))))))
+        (t/is (= 3 (:delete res)))))))
 
 (t/deftest font-deletion-3
   (let [prof    (th/create-profile* 1 {:is-active true})
@@ -248,7 +252,7 @@
     (t/is (nil? (:error out1)))
     (t/is (nil? (:error out2)))
 
-    (let [res (th/run-task! :storage-gc-touched {:min-age 0})]
+    (let [res (th/run-task! :storage-gc-touched {})]
       (t/is (= 6 (:freeze res))))
 
     (let [params {::th/type :delete-font-variant
@@ -260,13 +264,14 @@
       (t/is (nil? (:error out)))
       (t/is (nil? (:result out))))
 
-    (let [res (th/run-task! :storage-gc-touched {:min-age 0})]
+    (let [res (th/run-task! :storage-gc-touched {})]
       (t/is (= 0 (:freeze res)))
       (t/is (= 0 (:delete res))))
 
-    (let [res (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})]
+    (binding [ct/*clock* (clock/fixed (ct/in-future {:days 8}))]
-      (t/is (= 1 (:processed res))))
+      (let [res (th/run-task! :objects-gc {})]
+        (t/is (= 1 (:processed res))))
 
-    (let [res (th/run-task! :storage-gc-touched {:min-age 0})]
+      (let [res (th/run-task! :storage-gc-touched {})]
-      (t/is (= 0 (:freeze res)))
+        (t/is (= 0 (:freeze res)))
-      (t/is (= 3 (:delete res))))))
+        (t/is (= 3 (:delete res)))))))

backend/test/backend_tests/rpc_project_test.clj

@@ -6,11 +6,13 @@
 
 (ns backend-tests.rpc-project-test
   (:require
+   [app.common.time :as ct]
    [app.common.uuid :as uuid]
    [app.config :as cf]
    [app.db :as db]
    [app.http :as http]
    [app.rpc :as-alias rpc]
+   [app.setup.clock :as clock]
    [backend-tests.helpers :as th]
    [clojure.test :as t]))
 
@@ -226,8 +228,9 @@
         (t/is (= 0 (count result)))))
 
     ;; run permanent deletion
-    (let [result (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})]
+    (binding [ct/*clock* (clock/fixed (ct/in-future {:days 8}))]
-      (t/is (= 1 (:processed result))))
+      (let [result (th/run-task! :objects-gc {})]
+        (t/is (= 1 (:processed result)))))
 
     ;; query the list of files of a after hard deletion
     (let [data {::th/type :get-project-files

backend/test/backend_tests/rpc_team_test.clj

@@ -13,6 +13,7 @@
    [app.db :as db]
    [app.http :as http]
    [app.rpc :as-alias rpc]
+   [app.setup.clock :as clock]
    [app.storage :as sto]
    [app.tokens :as tokens]
    [backend-tests.helpers :as th]
@@ -525,8 +526,9 @@
         (t/is (= :not-found (:type edata)))))
 
     ;; run permanent deletion
-    (let [result (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})]
+    (binding [ct/*clock* (clock/fixed (ct/in-future {:days 8}))]
-      (t/is (= 2 (:processed result))))
+      (let [result (th/run-task! :objects-gc {})]
+        (t/is (= 2 (:processed result)))))
 
     ;; query the list of projects of a after hard deletion
     (let [data {::th/type :get-projects
@@ -581,8 +583,9 @@
       (t/is (= 1 (count rows)))
       (t/is (ct/inst? (:deleted-at (first rows)))))
 
-    (let [result (th/run-task! :objects-gc {:deletion-threshold (cf/get-deletion-delay)})]
+    (binding [ct/*clock* (clock/fixed (ct/in-future {:days 8}))]
-      (t/is (= 7 (:processed result))))))
+      (let [result (th/run-task! :objects-gc {})]
+        (t/is (= 7 (:processed result)))))))
 
 (t/deftest create-team-access-request
   (with-mocks [mock {:target 'app.email/send! :return nil}]

backend/test/backend_tests/storage_test.clj

@@ -11,6 +11,7 @@
    [app.common.uuid :as uuid]
    [app.db :as db]
    [app.rpc :as-alias rpc]
+   [app.setup.clock :as clock]
    [app.storage :as sto]
    [backend-tests.helpers :as th]
    [clojure.test :as t]
@@ -53,19 +54,13 @@
                     (configure-storage-backend))
         content (sto/content "content")
         object  (sto/put-object! storage {::sto/content content
-                                          ::sto/expired-at (ct/in-future {:seconds 1})
+                                          ::sto/expired-at (ct/in-future {:hours 1})
                                           :content-type "text/plain"})]
 
     (t/is (sto/object? object))
     (t/is (ct/inst? (:expired-at object)))
     (t/is (ct/is-after? (:expired-at object) (ct/now)))
-    (t/is (= object (sto/get-object storage (:id object))))
+    (t/is (nil? (sto/get-object storage (:id object))))))
-
-    (th/sleep 1000)
-    (t/is (nil? (sto/get-object storage (:id object))))
-    (t/is (nil? (sto/get-object-data storage object)))
-    (t/is (nil? (sto/get-object-url storage object)))
-    (t/is (nil? (sto/get-object-path storage object)))))
 
 (t/deftest put-and-delete-object
   (let [storage (-> (:app.storage/storage th/*system*)
@@ -98,20 +93,25 @@
                                            ::sto/expired-at (ct/now)
                                            :content-type "text/plain"})
         object2  (sto/put-object! storage {::sto/content content2
-                                           ::sto/expired-at (ct/in-past {:hours 2})
+                                           ::sto/expired-at (ct/in-future {:hours 2})
                                            :content-type "text/plain"})
         object3  (sto/put-object! storage {::sto/content content3
-                                           ::sto/expired-at (ct/in-past {:hours 1})
+                                           ::sto/expired-at (ct/in-future {:hours 1})
                                            :content-type "text/plain"})]
 
-
+    (binding [ct/*clock* (clock/fixed (ct/in-future {:minutes 0}))]
-    (th/sleep 200)
+      (let [res (th/run-task! :storage-gc-deleted {})]
-
+        (t/is (= 1 (:deleted res)))))
-    (let [res (th/run-task! :storage-gc-deleted {})]
-      (t/is (= 1 (:deleted res))))
 
     (let [res (th/db-exec-one! ["select count(*) from storage_object;"])]
-      (t/is (= 2 (:count res))))))
+      (t/is (= 2 (:count res))))
+
+    (binding [ct/*clock* (clock/fixed (ct/in-future {:minutes 61}))]
+      (let [res (th/run-task! :storage-gc-deleted {})]
+        (t/is (= 1 (:deleted res)))))
+
+    (let [res (th/db-exec-one! ["select count(*) from storage_object;"])]
+      (t/is (= 1 (:count res))))))
 
 (t/deftest touched-gc-task-1
   (let [storage (-> (:app.storage/storage th/*system*)
@@ -158,7 +158,7 @@
                      {:id (:id result-1)})
 
       ;; run the objects gc task for permanent deletion
-      (let [res (th/run-task! :objects-gc {:min-age 0})]
+      (let [res (th/run-task! :objects-gc {})]
         (t/is (= 1 (:processed res))))
 
       ;; check that we still have all the storage objects
@@ -182,7 +182,6 @@
       (let [res (th/db-exec-one! ["select count(*) from storage_object where deleted_at is not null"])]
         (t/is (= 0 (:count res)))))))
 
-
 (t/deftest touched-gc-task-2
   (let [storage (-> (:app.storage/storage th/*system*)
                     (configure-storage-backend))
@@ -243,11 +242,12 @@
                        {:id (:id result-2)})
 
         ;; run the objects gc task for permanent deletion
-        (let [res (th/run-task! :objects-gc {:min-age 0})]
+        (let [res (th/run-task! :objects-gc {})]
           (t/is (= 1 (:processed res))))
 
         ;; revert touched state to all storage objects
-        (th/db-exec-one! ["update storage_object set touched_at=now()"])
+
+        (th/db-exec-one! ["update storage_object set touched_at=?" (ct/now)])
 
         ;; Run the task again
         (let [res (th/run-task! :storage-gc-touched {})]
@@ -293,10 +293,10 @@
           result-2 (:result out2)]
 
       ;; now we proceed to manually mark all storage objects touched
-      (th/db-exec! ["update storage_object set touched_at=now()"])
+      (th/db-exec! ["update storage_object set touched_at=?" (ct/now)])
 
       ;; run the touched gc task
-      (let [res (th/run-task! "storage-gc-touched" {:min-age 0})]
+      (let [res (th/run-task! :storage-gc-touched {})]
         (t/is (= 2 (:freeze res)))
         (t/is (= 0 (:delete res))))
 
@@ -305,13 +305,13 @@
         (t/is (= 2 (count rows)))))
 
     ;; now we proceed to manually delete all file_media_object
-    (th/db-exec! ["update file_media_object set deleted_at = now()"])
+    (th/db-exec! ["update file_media_object set deleted_at = ?" (ct/now)])
 
-    (let [res (th/run-task! "objects-gc" {:min-age 0})]
+    (let [res (th/run-task! :objects-gc {})]
       (t/is (= 2 (:processed res))))
 
     ;; run the touched gc task
-    (let [res (th/run-task! "storage-gc-touched" {:min-age 0})]
+    (let [res (th/run-task! :storage-gc-touched {})]
       (t/is (= 0 (:freeze res)))
       (t/is (= 2 (:delete res))))