github/penpot
1
0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2026-04-25 11:18:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

✨ Add 2h min-age threshold to storage/gc_touched task

Browse Source 
Skip storage objects touched less than 2 hours ago, matching the pattern
used by upload-session-gc. Update all affected tests to advance the clock
past the threshold using ct/*clock* bindings.
This commit is contained in:
Andrey Antukh 2026-04-09 09:38:18 +00:00
parent f716995ffd
commit 97d234a566
5 changed files with 71 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
backend/src/app/storage/gc_touched.clj
View File

@ -213,8 +213,13 @@
[_ params]
(assert (db/pool? (::db/pool params)) "expect valid storage"))
(defmethod ig/init-key ::handler
[_ cfg]
(fn [_]
(process-touched! (assoc cfg ::timestamp (ct/now)))))
(defmethod ig/expand-key ::handler
[k v]
{k (merge {::min-age (ct/duration {:hours 2})} v)})
(defmethod ig/init-key ::handler
[_ {:keys [::min-age] :as cfg}]
(fn [_]
(let [threshold (ct/minus (ct/now) min-age)]
(process-touched! (assoc cfg ::timestamp threshold)))))

24
backend/test/backend_tests/rpc_file_test.clj
View File

@ -312,7 +312,8 @@
;; freeze because of the deduplication (we have uploaded 2 times
;; the same files).
(let [res (th/run-task! :storage-gc-touched {})]
(let [res (binding [ct/*clock* (ct/fixed-clock (ct/in-future {:hours 3}))]
(th/run-task! :storage-gc-touched {}))]
(t/is (= 2 (:freeze res)))
(t/is (= 0 (:delete res))))
@ -386,7 +387,8 @@
;; Now that file-gc have deleted the file-media-object usage,
;; lets execute the touched-gc task, we should see that two of
;; them are marked to be deleted
(let [res (th/run-task! :storage-gc-touched {})]
(let [res (binding [ct/*clock* (ct/fixed-clock (ct/in-future {:hours 3}))]
(th/run-task! :storage-gc-touched {}))]
(t/is (= 0 (:freeze res)))
(t/is (= 2 (:delete res))))
@ -571,7 +573,8 @@
;; Now that file-gc have deleted the file-media-object usage,
;; lets execute the touched-gc task, we should see that two of
;; them are marked to be deleted.
(let [res (th/run-task! :storage-gc-touched {})]
(let [res (binding [ct/*clock* (ct/fixed-clock (ct/in-future {:hours 3}))]
(th/run-task! :storage-gc-touched {}))]
(t/is (= 0 (:freeze res)))
(t/is (= 2 (:delete res))))
@ -664,7 +667,8 @@
;; because of the deduplication (we have uploaded 2 times the
;; same files).
(let [res (th/run-task! :storage-gc-touched {})]
(let [res (binding [ct/*clock* (ct/fixed-clock (ct/in-future {:hours 3}))]
(th/run-task! :storage-gc-touched {}))]
(t/is (= 1 (:freeze res)))
(t/is (= 0 (:delete res))))
@ -714,7 +718,8 @@
;; Now that objects-gc have deleted the object thumbnail lets
;; execute the touched-gc task
(let [res (th/run-task! "storage-gc-touched" {})]
(let [res (binding [ct/*clock* (ct/fixed-clock (ct/in-future {:hours 3}))]
(th/run-task! "storage-gc-touched" {}))]
(t/is (= 1 (:freeze res))))
;; check file media objects
@ -749,7 +754,8 @@
;; Now that file-gc have deleted the object thumbnail lets
;; execute the touched-gc task
(let [res (th/run-task! :storage-gc-touched {})]
(let [res (binding [ct/*clock* (ct/fixed-clock (ct/in-future {:hours 3}))]
(th/run-task! :storage-gc-touched {}))]
(t/is (= 1 (:delete res))))
;; check file media objects
@ -1319,7 +1325,8 @@
;; The FileGC task will schedule an inner taskq
(th/run-pending-tasks!)
(let [res (th/run-task! :storage-gc-touched {})]
(let [res (binding [ct/*clock* (ct/fixed-clock (ct/in-future {:hours 3}))]
(th/run-task! :storage-gc-touched {}))]
(t/is (= 2 (:freeze res)))
(t/is (= 0 (:delete res))))
@ -1413,7 +1420,8 @@
;; we ensure that once object-gc is passed and marked two storage
;; objects to delete
(let [res (th/run-task! :storage-gc-touched {})]
(let [res (binding [ct/*clock* (ct/fixed-clock (ct/in-future {:hours 3}))]
(th/run-task! :storage-gc-touched {}))]
(t/is (= 0 (:freeze res)))
(t/is (= 2 (:delete res))))

7
backend/test/backend_tests/rpc_file_thumbnails_test.clj
View File

@ -85,7 +85,7 @@
(t/is (map? (:result out))))
;; run the task again
(let [res (binding [ct/*clock* (ct/fixed-clock (ct/in-future {:minutes 31}))]
(let [res (binding [ct/*clock* (ct/fixed-clock (ct/in-future {:hours 3}))]
(th/run-task! "storage-gc-touched" {}))]
(t/is (= 2 (:freeze res))))
@ -136,7 +136,7 @@
(t/is (some? (sto/get-object storage (:media-id row2))))
;; run the task again
(let [res (binding [ct/*clock* (ct/fixed-clock (ct/in-future {:minutes 31}))]
(let [res (binding [ct/*clock* (ct/fixed-clock (ct/in-future {:hours 3}))]
(th/run-task! :storage-gc-touched {}))]
(t/is (= 1 (:delete res)))
(t/is (= 0 (:freeze res))))
@ -235,7 +235,8 @@
(t/is (= (:object-id data1) (:object-id row)))
(t/is (uuid? (:media-id row1))))
(let [result (th/run-task! :storage-gc-touched {})]
(let [result (binding [ct/*clock* (ct/fixed-clock (ct/in-future {:hours 3}))]
(th/run-task! :storage-gc-touched {}))]
(t/is (= 1 (:delete result))))
;; Check if storage objects still exists after file-gc

71
backend/test/backend_tests/rpc_font_test.clj
View File

@ -130,7 +130,8 @@
;; (th/print-result! out)
(t/is (nil? (:error out))))
(let [res (th/run-task! :storage-gc-touched {})]
(let [res (binding [ct/*clock* (ct/fixed-clock (ct/in-future {:hours 3}))]
(th/run-task! :storage-gc-touched {}))]
(t/is (= 6 (:freeze res))))
(let [params {::th/type :delete-font
@ -142,14 +143,16 @@
(t/is (nil? (:error out)))
(t/is (nil? (:result out))))
(let [res (th/run-task! :storage-gc-touched {})]
(let [res (binding [ct/*clock* (ct/fixed-clock (ct/in-future {:hours 3}))]
(th/run-task! :storage-gc-touched {}))]
(t/is (= 0 (:freeze res)))
(t/is (= 0 (:delete res))))
(binding [ct/*clock* (ct/fixed-clock (ct/in-future {:days 8}))]
(let [res (th/run-task! :objects-gc {})]
(t/is (= 2 (:processed res))))
(t/is (= 2 (:processed res)))))
(binding [ct/*clock* (ct/fixed-clock (ct/in-future {:days 8 :hours 3}))]
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 0 (:freeze res)))
(t/is (= 6 (:delete res)))))))
@ -191,7 +194,8 @@
;; (th/print-result! out)
(t/is (nil? (:error out))))
(let [res (th/run-task! :storage-gc-touched {})]
(let [res (binding [ct/*clock* (ct/fixed-clock (ct/in-future {:hours 3}))]
(th/run-task! :storage-gc-touched {}))]
(t/is (= 6 (:freeze res))))
(let [params {::th/type :delete-font
@ -203,14 +207,16 @@
(t/is (nil? (:error out)))
(t/is (nil? (:result out))))
(let [res (th/run-task! :storage-gc-touched {})]
(let [res (binding [ct/*clock* (ct/fixed-clock (ct/in-future {:hours 3}))]
(th/run-task! :storage-gc-touched {}))]
(t/is (= 0 (:freeze res)))
(t/is (= 0 (:delete res))))
(binding [ct/*clock* (ct/fixed-clock (ct/in-future {:days 8}))]
(let [res (th/run-task! :objects-gc {})]
(t/is (= 1 (:processed res))))
(t/is (= 1 (:processed res)))))
(binding [ct/*clock* (ct/fixed-clock (ct/in-future {:days 8 :hours 3}))]
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 0 (:freeze res)))
(t/is (= 3 (:delete res)))))))
@ -220,57 +226,42 @@
team-id (:default-team-id prof)
proj-id (:default-project-id prof)
font-id (uuid/custom 10 1)
data1 (-> (io/resource "backend_tests/test_files/font-1.woff")
(io/read*))
data2 (-> (io/resource "backend_tests/test_files/font-2.woff")
(io/read*))
params1 {::th/type :create-font-variant
::rpc/profile-id (:id prof)
:team-id team-id
:font-id font-id
:font-family "somefont"
:font-weight 400
:font-style "normal"
:data {"font/woff" data1}}
params2 {::th/type :create-font-variant
::rpc/profile-id (:id prof)
:team-id team-id
:font-id font-id
:font-family "somefont"
:font-weight 500
:font-style "normal"
:data {"font/woff" data2}}
data1 (-> (io/resource "backend_tests/test_files/font-1.woff") (io/read*))
data2 (-> (io/resource "backend_tests/test_files/font-2.woff") (io/read*))
params1 {::th/type :create-font-variant ::rpc/profile-id (:id prof)
:team-id team-id :font-id font-id :font-family "somefont"
:font-weight 400 :font-style "normal" :data {"font/woff" data1}}
params2 {::th/type :create-font-variant ::rpc/profile-id (:id prof)
:team-id team-id :font-id font-id :font-family "somefont"
:font-weight 500 :font-style "normal" :data {"font/woff" data2}}
out1 (th/command! params1)
out2 (th/command! params2)]
;; (th/print-result! out1)
(t/is (nil? (:error out1)))
(t/is (nil? (:error out2)))
(let [res (th/run-task! :storage-gc-touched {})]
;; freeze with hours 3 clock
(let [res (binding [ct/*clock* (ct/fixed-clock (ct/in-future {:hours 3}))]
(th/run-task! :storage-gc-touched {}))]
(t/is (= 6 (:freeze res))))
(let [params {::th/type :delete-font-variant
::rpc/profile-id (:id prof)
:team-id team-id
:id (-> out1 :result :id)}
(let [params {::th/type :delete-font-variant ::rpc/profile-id (:id prof)
:team-id team-id :id (-> out1 :result :id)}
out (th/command! params)]
;; (th/print-result! out)
(t/is (nil? (:error out)))
(t/is (nil? (:result out))))
(let [res (th/run-task! :storage-gc-touched {})]
;; no-op with hours 3 clock (nothing touched yet)
(let [res (binding [ct/*clock* (ct/fixed-clock (ct/in-future {:hours 3}))]
(th/run-task! :storage-gc-touched {}))]
(t/is (= 0 (:freeze res)))
(t/is (= 0 (:delete res))))
;; objects-gc at days 8, then storage-gc-touched at days 8 + 3h
(binding [ct/*clock* (ct/fixed-clock (ct/in-future {:days 8}))]
(let [res (th/run-task! :objects-gc {})]
(t/is (= 1 (:processed res))))
(t/is (= 1 (:processed res)))))
(binding [ct/*clock* (ct/fixed-clock (ct/in-future {:days 8 :hours 3}))]
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 0 (:freeze res)))
(t/is (= 3 (:delete res)))))))

17
backend/test/backend_tests/storage_test.clj
View File

@ -169,7 +169,8 @@
(t/is (= 2 (:count res))))
;; run the touched gc task
(let [res (th/run-task! :storage-gc-touched {})]
(let [res (binding [ct/*clock* (ct/fixed-clock (ct/in-future {:hours 3}))]
(th/run-task! :storage-gc-touched {}))]
(t/is (= 2 (:freeze res)))
(t/is (= 0 (:delete res))))
@ -229,7 +230,8 @@
(t/is (nil? (:error out2)))
;; run the touched gc task
(let [res (th/run-task! :storage-gc-touched {})]
(let [res (binding [ct/*clock* (ct/fixed-clock (ct/in-future {:hours 3}))]
(th/run-task! :storage-gc-touched {}))]
(t/is (= 5 (:freeze res)))
(t/is (= 0 (:delete res)))
@ -249,7 +251,8 @@
(th/db-exec-one! ["update storage_object set touched_at=?" (ct/now)])
;; Run the task again
(let [res (th/run-task! :storage-gc-touched {})]
(let [res (binding [ct/*clock* (ct/fixed-clock (ct/in-future {:hours 3}))]
(th/run-task! :storage-gc-touched {}))]
(t/is (= 2 (:freeze res)))
(t/is (= 3 (:delete res))))
@ -295,7 +298,8 @@
(th/db-exec! ["update storage_object set touched_at=?" (ct/now)])
;; run the touched gc task
(let [res (th/run-task! :storage-gc-touched {})]
(let [res (binding [ct/*clock* (ct/fixed-clock (ct/in-future {:hours 3}))]
(th/run-task! :storage-gc-touched {}))]
(t/is (= 2 (:freeze res)))
(t/is (= 0 (:delete res))))
@ -310,7 +314,8 @@
(t/is (= 2 (:processed res))))
;; run the touched gc task
(let [res (th/run-task! :storage-gc-touched {})]
(let [res (binding [ct/*clock* (ct/fixed-clock (ct/in-future {:hours 3}))]
(th/run-task! :storage-gc-touched {}))]
(t/is (= 0 (:freeze res)))
(t/is (= 2 (:delete res))))
@ -336,7 +341,7 @@
(t/is (= 0 (:delete res)))))
(binding [ct/*clock* (ct/fixed-clock (ct/plus now {:minutes 1}))]
(binding [ct/*clock* (ct/fixed-clock (ct/plus now {:hours 3}))]
(let [res (th/run-task! :storage-gc-touched {})]
(t/is (= 0 (:freeze res)))
(t/is (= 1 (:delete res)))))