diff --git a/backend/src/app/storage/gc_touched.clj b/backend/src/app/storage/gc_touched.clj index a3ea21e44d..f00140d04e 100644 --- a/backend/src/app/storage/gc_touched.clj +++ b/backend/src/app/storage/gc_touched.clj @@ -213,8 +213,13 @@ [_ params] (assert (db/pool? (::db/pool params)) "expect valid storage")) -(defmethod ig/init-key ::handler - [_ cfg] - (fn [_] - (process-touched! (assoc cfg ::timestamp (ct/now))))) +(defmethod ig/expand-key ::handler + [k v] + {k (merge {::min-age (ct/duration {:hours 2})} v)}) + +(defmethod ig/init-key ::handler + [_ {:keys [::min-age] :as cfg}] + (fn [_] + (let [threshold (ct/minus (ct/now) min-age)] + (process-touched! (assoc cfg ::timestamp threshold))))) diff --git a/backend/test/backend_tests/rpc_file_test.clj b/backend/test/backend_tests/rpc_file_test.clj index 921477d1b3..281c834256 100644 --- a/backend/test/backend_tests/rpc_file_test.clj +++ b/backend/test/backend_tests/rpc_file_test.clj @@ -312,7 +312,8 @@ ;; freeze because of the deduplication (we have uploaded 2 times ;; the same files). - (let [res (th/run-task! :storage-gc-touched {})] + (let [res (binding [ct/*clock* (ct/fixed-clock (ct/in-future {:hours 3}))] + (th/run-task! :storage-gc-touched {}))] (t/is (= 2 (:freeze res))) (t/is (= 0 (:delete res)))) @@ -386,7 +387,8 @@ ;; Now that file-gc have deleted the file-media-object usage, ;; lets execute the touched-gc task, we should see that two of ;; them are marked to be deleted - (let [res (th/run-task! :storage-gc-touched {})] + (let [res (binding [ct/*clock* (ct/fixed-clock (ct/in-future {:hours 3}))] + (th/run-task! :storage-gc-touched {}))] (t/is (= 0 (:freeze res))) (t/is (= 2 (:delete res)))) @@ -571,7 +573,8 @@ ;; Now that file-gc have deleted the file-media-object usage, ;; lets execute the touched-gc task, we should see that two of ;; them are marked to be deleted. - (let [res (th/run-task! :storage-gc-touched {})] + (let [res (binding [ct/*clock* (ct/fixed-clock (ct/in-future {:hours 3}))] + (th/run-task! :storage-gc-touched {}))] (t/is (= 0 (:freeze res))) (t/is (= 2 (:delete res)))) @@ -664,7 +667,8 @@ ;; because of the deduplication (we have uploaded 2 times the ;; same files). - (let [res (th/run-task! :storage-gc-touched {})] + (let [res (binding [ct/*clock* (ct/fixed-clock (ct/in-future {:hours 3}))] + (th/run-task! :storage-gc-touched {}))] (t/is (= 1 (:freeze res))) (t/is (= 0 (:delete res)))) @@ -714,7 +718,8 @@ ;; Now that objects-gc have deleted the object thumbnail lets ;; execute the touched-gc task - (let [res (th/run-task! "storage-gc-touched" {})] + (let [res (binding [ct/*clock* (ct/fixed-clock (ct/in-future {:hours 3}))] + (th/run-task! "storage-gc-touched" {}))] (t/is (= 1 (:freeze res)))) ;; check file media objects @@ -749,7 +754,8 @@ ;; Now that file-gc have deleted the object thumbnail lets ;; execute the touched-gc task - (let [res (th/run-task! :storage-gc-touched {})] + (let [res (binding [ct/*clock* (ct/fixed-clock (ct/in-future {:hours 3}))] + (th/run-task! :storage-gc-touched {}))] (t/is (= 1 (:delete res)))) ;; check file media objects @@ -1319,7 +1325,8 @@ ;; The FileGC task will schedule an inner taskq (th/run-pending-tasks!) - (let [res (th/run-task! :storage-gc-touched {})] + (let [res (binding [ct/*clock* (ct/fixed-clock (ct/in-future {:hours 3}))] + (th/run-task! :storage-gc-touched {}))] (t/is (= 2 (:freeze res))) (t/is (= 0 (:delete res)))) @@ -1413,7 +1420,8 @@ ;; we ensure that once object-gc is passed and marked two storage ;; objects to delete - (let [res (th/run-task! :storage-gc-touched {})] + (let [res (binding [ct/*clock* (ct/fixed-clock (ct/in-future {:hours 3}))] + (th/run-task! :storage-gc-touched {}))] (t/is (= 0 (:freeze res))) (t/is (= 2 (:delete res)))) diff --git a/backend/test/backend_tests/rpc_file_thumbnails_test.clj b/backend/test/backend_tests/rpc_file_thumbnails_test.clj index 9a856f3210..28134da5ff 100644 --- a/backend/test/backend_tests/rpc_file_thumbnails_test.clj +++ b/backend/test/backend_tests/rpc_file_thumbnails_test.clj @@ -85,7 +85,7 @@ (t/is (map? (:result out)))) ;; run the task again - (let [res (binding [ct/*clock* (ct/fixed-clock (ct/in-future {:minutes 31}))] + (let [res (binding [ct/*clock* (ct/fixed-clock (ct/in-future {:hours 3}))] (th/run-task! "storage-gc-touched" {}))] (t/is (= 2 (:freeze res)))) @@ -136,7 +136,7 @@ (t/is (some? (sto/get-object storage (:media-id row2)))) ;; run the task again - (let [res (binding [ct/*clock* (ct/fixed-clock (ct/in-future {:minutes 31}))] + (let [res (binding [ct/*clock* (ct/fixed-clock (ct/in-future {:hours 3}))] (th/run-task! :storage-gc-touched {}))] (t/is (= 1 (:delete res))) (t/is (= 0 (:freeze res)))) @@ -235,7 +235,8 @@ (t/is (= (:object-id data1) (:object-id row))) (t/is (uuid? (:media-id row1)))) - (let [result (th/run-task! :storage-gc-touched {})] + (let [result (binding [ct/*clock* (ct/fixed-clock (ct/in-future {:hours 3}))] + (th/run-task! :storage-gc-touched {}))] (t/is (= 1 (:delete result)))) ;; Check if storage objects still exists after file-gc diff --git a/backend/test/backend_tests/rpc_font_test.clj b/backend/test/backend_tests/rpc_font_test.clj index 1316b237c9..d68f657c59 100644 --- a/backend/test/backend_tests/rpc_font_test.clj +++ b/backend/test/backend_tests/rpc_font_test.clj @@ -130,7 +130,8 @@ ;; (th/print-result! out) (t/is (nil? (:error out)))) - (let [res (th/run-task! :storage-gc-touched {})] + (let [res (binding [ct/*clock* (ct/fixed-clock (ct/in-future {:hours 3}))] + (th/run-task! :storage-gc-touched {}))] (t/is (= 6 (:freeze res)))) (let [params {::th/type :delete-font @@ -142,14 +143,16 @@ (t/is (nil? (:error out))) (t/is (nil? (:result out)))) - (let [res (th/run-task! :storage-gc-touched {})] + (let [res (binding [ct/*clock* (ct/fixed-clock (ct/in-future {:hours 3}))] + (th/run-task! :storage-gc-touched {}))] (t/is (= 0 (:freeze res))) (t/is (= 0 (:delete res)))) (binding [ct/*clock* (ct/fixed-clock (ct/in-future {:days 8}))] (let [res (th/run-task! :objects-gc {})] - (t/is (= 2 (:processed res)))) + (t/is (= 2 (:processed res))))) + (binding [ct/*clock* (ct/fixed-clock (ct/in-future {:days 8 :hours 3}))] (let [res (th/run-task! :storage-gc-touched {})] (t/is (= 0 (:freeze res))) (t/is (= 6 (:delete res))))))) @@ -191,7 +194,8 @@ ;; (th/print-result! out) (t/is (nil? (:error out)))) - (let [res (th/run-task! :storage-gc-touched {})] + (let [res (binding [ct/*clock* (ct/fixed-clock (ct/in-future {:hours 3}))] + (th/run-task! :storage-gc-touched {}))] (t/is (= 6 (:freeze res)))) (let [params {::th/type :delete-font @@ -203,14 +207,16 @@ (t/is (nil? (:error out))) (t/is (nil? (:result out)))) - (let [res (th/run-task! :storage-gc-touched {})] + (let [res (binding [ct/*clock* (ct/fixed-clock (ct/in-future {:hours 3}))] + (th/run-task! :storage-gc-touched {}))] (t/is (= 0 (:freeze res))) (t/is (= 0 (:delete res)))) (binding [ct/*clock* (ct/fixed-clock (ct/in-future {:days 8}))] (let [res (th/run-task! :objects-gc {})] - (t/is (= 1 (:processed res)))) + (t/is (= 1 (:processed res))))) + (binding [ct/*clock* (ct/fixed-clock (ct/in-future {:days 8 :hours 3}))] (let [res (th/run-task! :storage-gc-touched {})] (t/is (= 0 (:freeze res))) (t/is (= 3 (:delete res))))))) @@ -220,57 +226,42 @@ team-id (:default-team-id prof) proj-id (:default-project-id prof) font-id (uuid/custom 10 1) - - data1 (-> (io/resource "backend_tests/test_files/font-1.woff") - (io/read*)) - - data2 (-> (io/resource "backend_tests/test_files/font-2.woff") - (io/read*)) - params1 {::th/type :create-font-variant - ::rpc/profile-id (:id prof) - :team-id team-id - :font-id font-id - :font-family "somefont" - :font-weight 400 - :font-style "normal" - :data {"font/woff" data1}} - - params2 {::th/type :create-font-variant - ::rpc/profile-id (:id prof) - :team-id team-id - :font-id font-id - :font-family "somefont" - :font-weight 500 - :font-style "normal" - :data {"font/woff" data2}} - + data1 (-> (io/resource "backend_tests/test_files/font-1.woff") (io/read*)) + data2 (-> (io/resource "backend_tests/test_files/font-2.woff") (io/read*)) + params1 {::th/type :create-font-variant ::rpc/profile-id (:id prof) + :team-id team-id :font-id font-id :font-family "somefont" + :font-weight 400 :font-style "normal" :data {"font/woff" data1}} + params2 {::th/type :create-font-variant ::rpc/profile-id (:id prof) + :team-id team-id :font-id font-id :font-family "somefont" + :font-weight 500 :font-style "normal" :data {"font/woff" data2}} out1 (th/command! params1) out2 (th/command! params2)] - - ;; (th/print-result! out1) (t/is (nil? (:error out1))) (t/is (nil? (:error out2))) - (let [res (th/run-task! :storage-gc-touched {})] + ;; freeze with hours 3 clock + (let [res (binding [ct/*clock* (ct/fixed-clock (ct/in-future {:hours 3}))] + (th/run-task! :storage-gc-touched {}))] (t/is (= 6 (:freeze res)))) - (let [params {::th/type :delete-font-variant - ::rpc/profile-id (:id prof) - :team-id team-id - :id (-> out1 :result :id)} + (let [params {::th/type :delete-font-variant ::rpc/profile-id (:id prof) + :team-id team-id :id (-> out1 :result :id)} out (th/command! params)] - ;; (th/print-result! out) (t/is (nil? (:error out))) (t/is (nil? (:result out)))) - (let [res (th/run-task! :storage-gc-touched {})] + ;; no-op with hours 3 clock (nothing touched yet) + (let [res (binding [ct/*clock* (ct/fixed-clock (ct/in-future {:hours 3}))] + (th/run-task! :storage-gc-touched {}))] (t/is (= 0 (:freeze res))) (t/is (= 0 (:delete res)))) + ;; objects-gc at days 8, then storage-gc-touched at days 8 + 3h (binding [ct/*clock* (ct/fixed-clock (ct/in-future {:days 8}))] (let [res (th/run-task! :objects-gc {})] - (t/is (= 1 (:processed res)))) + (t/is (= 1 (:processed res))))) + (binding [ct/*clock* (ct/fixed-clock (ct/in-future {:days 8 :hours 3}))] (let [res (th/run-task! :storage-gc-touched {})] (t/is (= 0 (:freeze res))) (t/is (= 3 (:delete res))))))) diff --git a/backend/test/backend_tests/storage_test.clj b/backend/test/backend_tests/storage_test.clj index cd058af250..027d54ce70 100644 --- a/backend/test/backend_tests/storage_test.clj +++ b/backend/test/backend_tests/storage_test.clj @@ -169,7 +169,8 @@ (t/is (= 2 (:count res)))) ;; run the touched gc task - (let [res (th/run-task! :storage-gc-touched {})] + (let [res (binding [ct/*clock* (ct/fixed-clock (ct/in-future {:hours 3}))] + (th/run-task! :storage-gc-touched {}))] (t/is (= 2 (:freeze res))) (t/is (= 0 (:delete res)))) @@ -229,7 +230,8 @@ (t/is (nil? (:error out2))) ;; run the touched gc task - (let [res (th/run-task! :storage-gc-touched {})] + (let [res (binding [ct/*clock* (ct/fixed-clock (ct/in-future {:hours 3}))] + (th/run-task! :storage-gc-touched {}))] (t/is (= 5 (:freeze res))) (t/is (= 0 (:delete res))) @@ -249,7 +251,8 @@ (th/db-exec-one! ["update storage_object set touched_at=?" (ct/now)]) ;; Run the task again - (let [res (th/run-task! :storage-gc-touched {})] + (let [res (binding [ct/*clock* (ct/fixed-clock (ct/in-future {:hours 3}))] + (th/run-task! :storage-gc-touched {}))] (t/is (= 2 (:freeze res))) (t/is (= 3 (:delete res)))) @@ -295,7 +298,8 @@ (th/db-exec! ["update storage_object set touched_at=?" (ct/now)]) ;; run the touched gc task - (let [res (th/run-task! :storage-gc-touched {})] + (let [res (binding [ct/*clock* (ct/fixed-clock (ct/in-future {:hours 3}))] + (th/run-task! :storage-gc-touched {}))] (t/is (= 2 (:freeze res))) (t/is (= 0 (:delete res)))) @@ -310,7 +314,8 @@ (t/is (= 2 (:processed res)))) ;; run the touched gc task - (let [res (th/run-task! :storage-gc-touched {})] + (let [res (binding [ct/*clock* (ct/fixed-clock (ct/in-future {:hours 3}))] + (th/run-task! :storage-gc-touched {}))] (t/is (= 0 (:freeze res))) (t/is (= 2 (:delete res)))) @@ -336,7 +341,7 @@ (t/is (= 0 (:delete res))))) - (binding [ct/*clock* (ct/fixed-clock (ct/plus now {:minutes 1}))] + (binding [ct/*clock* (ct/fixed-clock (ct/plus now {:hours 3}))] (let [res (th/run-task! :storage-gc-touched {})] (t/is (= 0 (:freeze res))) (t/is (= 1 (:delete res)))))