From da68eae7c681e0f1ae1f993971f747ac9305e3e1 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 8 Nov 2023 15:15:09 +0100 Subject: [PATCH] :sparkles: Add proper climit configuration for file-thumbnails --- backend/resources/climit.edn | 3 + backend/src/app/rpc/climit.clj | 82 +++++++++---------- .../src/app/rpc/commands/files_thumbnails.clj | 14 +++- 3 files changed, 51 insertions(+), 48 deletions(-) diff --git a/backend/resources/climit.edn b/backend/resources/climit.edn index 5802af5e3b..4a485b8f7d 100644 --- a/backend/resources/climit.edn +++ b/backend/resources/climit.edn @@ -10,5 +10,8 @@ :process-font {:permits 4 :queue 32} :process-image {:permits 8 :queue 32} + :file-thumbnail-ops + {:permits 2} + :submit-audit-events-by-profile {:permits 1 :queue 3}} diff --git a/backend/src/app/rpc/climit.clj b/backend/src/app/rpc/climit.clj index 060ab00154..279723d57e 100644 --- a/backend/src/app/rpc/climit.clj +++ b/backend/src/app/rpc/climit.clj @@ -35,7 +35,7 @@ [{:keys [::wrk/executor]} config] (letfn [(load-fn [key] (let [config (get config (nth key 0))] - (l/trace :hint "insert into cache" :key key) + (l/trc :hint "insert into cache" :key key) (pbh/create :permits (or (:permits config) (:concurrency config)) :queue (or (:queue config) (:queue-size config)) :timeout (:timeout config) @@ -43,7 +43,7 @@ :type (:type config :semaphore)))) (on-remove [_ _ cause] - (l/trace :hint "evict from cache" :key key :reason (str cause)))] + (l/trc :hint "evict from cache" :key key :reason (str cause)))] (cache/create :executor :same-thread :on-remove on-remove @@ -71,7 +71,7 @@ [_ {:keys [::path ::mtx/metrics ::wrk/executor] :as cfg}] (when (contains? cf/flags :rpc-climit) (when-let [params (some->> path slurp edn/read-string)] - (l/info :hint "initializing concurrency limit" :config (str path)) + (l/inf :hint "initializing concurrency limit" :config (str path)) (us/verify! ::config params) {::cache (create-bulkhead-cache cfg params) ::config params @@ -99,15 +99,15 @@ (fn [] (let [elapsed (tpoint) stats (pbh/get-stats limiter)] - (l/trace :hint "executed" - :id (name id) - :key key - :fnh (hash f) - :permits (:permits stats) - :queue (:queue stats) - :max-permits (:max-permits stats) - :max-queue (:max-queue stats) - :elapsed (dt/format-duration elapsed)) + (l/trc :hint "executed" + :id (name id) + :key key + :fnh (hash f) + :permits (:permits stats) + :queue (:queue stats) + :max-permits (:max-permits stats) + :max-queue (:max-queue stats) + :elapsed (dt/format-duration elapsed)) (mtx/run! metrics :id :rpc-climit-timing :val (inst-ms elapsed) @@ -116,15 +116,15 @@ (f) (finally (let [elapsed (tpoint)] - (l/trace :hint "finished" - :id (name id) - :key key - :fnh (hash f) - :permits (:permits stats) - :queue (:queue stats) - :max-permits (:max-permits stats) - :max-queue (:max-queue stats) - :elapsed (dt/format-duration elapsed))))))) + (l/trc :hint "finished" + :id (name id) + :key key + :fnh (hash f) + :permits (:permits stats) + :queue (:queue stats) + :max-permits (:max-permits stats) + :max-queue (:max-queue stats) + :elapsed (dt/format-duration elapsed))))))) measure! (fn [stats] (mtx/run! metrics @@ -139,14 +139,14 @@ (try (let [stats (pbh/get-stats limiter)] (measure! stats) - (l/trace :hint "enqueued" - :id (name id) - :key key - :fnh (hash f) - :permits (:permits stats) - :queue (:queue stats) - :max-permits (:max-permits stats) - :max-queue (:max-queue stats)) + (l/trc :hint "enqueued" + :id (name id) + :key key + :fnh (hash f) + :permits (:permits stats) + :queue (:queue stats) + :max-permits (:max-permits stats) + :max-queue (:max-queue stats)) (pbh/invoke! limiter wrapped)) (catch ExceptionInfo cause (let [{:keys [type code]} (ex-data cause)] @@ -193,13 +193,6 @@ (app.rpc.climit/run! (^:once fn* [] ~@body))) `(run! ~instance (^:once fn* [] ~@body)))) -(defmacro with-dispatch - "Dispatch blocking operation to a separated thread protected with - the specified semaphore. - DEPRECATED" - [& params] - `(with-dispatch! ~@params)) - (def noop-fn (constantly nil)) (defn wrap @@ -207,18 +200,19 @@ (if (and (some? climit) (some? id)) (if-let [config (get-in climit [::config id])] (let [cache (::cache climit)] - (l/debug :hint "wrap: instrumenting method" - :limit (name id) - :service-name (::sv/name mdata) - :timeout (:timeout config) - :permits (:permits config) - :queue (:queue config) - :keyed? (some? key-fn)) + (l/dbg :hint "instrumenting method" + :limit (name id) + :service-name (::sv/name mdata) + :timeout (:timeout config) + :permits (:permits config) + :queue (:queue config) + :keyed? (not= key-fn noop-fn)) + (fn [cfg params] (invoke! cache metrics id (key-fn params) (partial f cfg params)))) (do - (l/warn :hint "no config found for specified queue" :id id) + (l/wrn :hint "no config found for specified queue" :id id) f)) f)) diff --git a/backend/src/app/rpc/commands/files_thumbnails.clj b/backend/src/app/rpc/commands/files_thumbnails.clj index d4475a6358..da5511c65e 100644 --- a/backend/src/app/rpc/commands/files_thumbnails.clj +++ b/backend/src/app/rpc/commands/files_thumbnails.clj @@ -20,6 +20,7 @@ [app.loggers.webhooks :as-alias webhooks] [app.media :as media] [app.rpc :as-alias rpc] + [app.rpc.climit :as-alias climit] [app.rpc.commands.files :as files] [app.rpc.commands.teams :as teams] [app.rpc.cond :as-alias cond] @@ -262,8 +263,10 @@ [:tag {:optional true} :string]]) (sv/defmethod ::create-file-object-thumbnail - {:doc/added "1.19" + {::doc/added "1.19" ::doc/module :files + ::climit/id :file-thumbnail-ops + ::climit/key-fn ::rpc/profile-id ::audit/skip true ::sm/params schema:create-file-object-thumbnail} @@ -288,9 +291,8 @@ {:file-id file-id :object-id object-id} {::db/for-update? true})] - (when media-id - (sto/del-object! storage media-id)) + (sto/del-object! storage media-id) (db/delete! conn :file-tagged-object-thumbnail {:file-id file-id :object-id object-id}) @@ -301,8 +303,10 @@ :req-un [::file-id ::object-id])) (sv/defmethod ::delete-file-object-thumbnail - {:doc/added "1.19" + {::doc/added "1.19" ::doc/module :files + ::climit/id :file-thumbnail-ops + ::climit/key-fn ::rpc/profile-id ::audit/skip true} [{:keys [::db/pool] :as cfg} {:keys [::rpc/profile-id file-id object-id]}] @@ -351,6 +355,8 @@ {::doc/added "1.19" ::doc/module :files ::audit/skip true + ::climit/id :file-thumbnail-ops + ::climit/key-fn ::rpc/profile-id ::sm/params [:map {:title "create-file-thumbnail"} [:file-id ::sm/uuid] [:revn :int]