From 98d5789b1b7232fa1d016e25e702b3c09657ead4 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Wed, 22 Dec 2021 19:04:03 +0100 Subject: [PATCH 1/5] :lisptick: Cosmetic changes. --- frontend/src/app/main/ui/releases.cljs | 2 +- frontend/src/app/main/ui/workspace/viewport/frame_grid.cljs | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/frontend/src/app/main/ui/releases.cljs b/frontend/src/app/main/ui/releases.cljs index 207d242d7c..1fee4c1d2c 100644 --- a/frontend/src/app/main/ui/releases.cljs +++ b/frontend/src/app/main/ui/releases.cljs @@ -10,13 +10,13 @@ [app.main.data.users :as du] [app.main.store :as st] [app.main.ui.releases.common :as rc] + [app.main.ui.releases.v1-10] [app.main.ui.releases.v1-4] [app.main.ui.releases.v1-5] [app.main.ui.releases.v1-6] [app.main.ui.releases.v1-7] [app.main.ui.releases.v1-8] [app.main.ui.releases.v1-9] - [app.main.ui.releases.v1-10] [app.util.object :as obj] [app.util.timers :as tm] [rumext.alpha :as mf])) diff --git a/frontend/src/app/main/ui/workspace/viewport/frame_grid.cljs b/frontend/src/app/main/ui/workspace/viewport/frame_grid.cljs index 75404fd6d3..aac177eec2 100644 --- a/frontend/src/app/main/ui/workspace/viewport/frame_grid.cljs +++ b/frontend/src/app/main/ui/workspace/viewport/frame_grid.cljs @@ -11,7 +11,6 @@ [app.common.uuid :as uuid] [app.main.refs :as refs] [app.util.geom.grid :as gg] - [okulary.core :as l] [rumext.alpha :as mf])) (mf/defc square-grid [{:keys [frame zoom grid] :as props}] @@ -67,14 +66,14 @@ (mf/defc grid-display-frame [{:keys [frame zoom]}] - (for [[index {:keys [type display] :as grid}] (->> (:grids frame) + (for [[index grid] (->> (:grids frame) (filter :display) (map-indexed vector))] (let [props #js {:key (str (:id frame) "-grid-" index) :frame frame :zoom zoom :grid grid}] - (case type + (case (:type grid) :square [:> square-grid props] :column [:> layout-grid props] :row [:> layout-grid props])))) From c6054f7ab2910810849e2a0c37ac3fcee5943479 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Thu, 23 Dec 2021 00:04:15 +0100 Subject: [PATCH 2/5] :lipstick: Improve json namespace API (and fix linter). --- backend/src/app/db.clj | 8 +++++--- backend/src/app/http/middleware.clj | 4 +--- backend/src/app/loggers/loki.clj | 2 +- backend/src/app/loggers/mattermost.clj | 2 +- backend/src/app/loggers/zmq.clj | 14 ++++++-------- backend/src/app/msgbus.clj | 12 ++++++------ backend/src/app/storage/impl.clj | 12 ++++++------ backend/src/app/tasks/telemetry.clj | 2 +- backend/src/app/util/json.clj | 25 +++++++++++++++---------- 9 files changed, 42 insertions(+), 39 deletions(-) diff --git a/backend/src/app/db.clj b/backend/src/app/db.clj index 60347cd24a..bcbf372b24 100644 --- a/backend/src/app/db.clj +++ b/backend/src/app/db.clj @@ -27,14 +27,16 @@ com.zaxxer.hikari.HikariConfig com.zaxxer.hikari.HikariDataSource com.zaxxer.hikari.metrics.prometheus.PrometheusMetricsTrackerFactory + java.io.InputStream + java.io.OutputStream java.lang.AutoCloseable java.sql.Connection java.sql.Savepoint org.postgresql.PGConnection org.postgresql.geometric.PGpoint + org.postgresql.jdbc.PgArray org.postgresql.largeobject.LargeObject org.postgresql.largeobject.LargeObjectManager - org.postgresql.jdbc.PgArray org.postgresql.util.PGInterval org.postgresql.util.PGobject)) @@ -356,7 +358,7 @@ val (.getValue o)] (if (or (= typ "json") (= typ "jsonb")) - (json/decode-str val) + (json/read val) val))) (defn decode-transit-pgobject @@ -392,7 +394,7 @@ [data] (doto (org.postgresql.util.PGobject.) (.setType "jsonb") - (.setValue (json/encode-str data)))) + (.setValue (json/write-str data)))) ;; --- Locks diff --git a/backend/src/app/http/middleware.clj b/backend/src/app/http/middleware.clj index 8c8bf11514..de3343dbca 100644 --- a/backend/src/app/http/middleware.clj +++ b/backend/src/app/http/middleware.clj @@ -13,7 +13,6 @@ [app.util.json :as json] [buddy.core.codecs :as bc] [buddy.core.hash :as bh] - [clojure.java.io :as io] [ring.middleware.cookies :refer [wrap-cookies]] [ring.middleware.keyword-params :refer [wrap-keyword-params]] [ring.middleware.multipart-params :refer [wrap-multipart-params]] @@ -36,8 +35,7 @@ (t/read! reader))) (parse-json [body] - (let [reader (io/reader body)] - (json/read reader))) + (json/read body)) (parse [type body] (try diff --git a/backend/src/app/loggers/loki.clj b/backend/src/app/loggers/loki.clj index 5f6980d320..2734ffa436 100644 --- a/backend/src/app/loggers/loki.clj +++ b/backend/src/app/loggers/loki.clj @@ -68,7 +68,7 @@ :timeout 6000 :method :post :headers {"content-type" "application/json"} - :body (json/encode payload)})] + :body (json/write payload)})] (cond (= (:status response) 204) true diff --git a/backend/src/app/loggers/mattermost.clj b/backend/src/app/loggers/mattermost.clj index ca23b92290..340fd7bf1b 100644 --- a/backend/src/app/loggers/mattermost.clj +++ b/backend/src/app/loggers/mattermost.clj @@ -31,7 +31,7 @@ rsp (http/send! {:uri uri :method :post :headers {"content-type" "application/json"} - :body (json/encode-str {:text text})})] + :body (json/write-str {:text text})})] (when (not= (:status rsp) 200) (l/error :hint "error on sending data to mattermost" :response (pr-str rsp)))) diff --git a/backend/src/app/loggers/zmq.clj b/backend/src/app/loggers/zmq.clj index 3f43cab999..0535f8da5b 100644 --- a/backend/src/app/loggers/zmq.clj +++ b/backend/src/app/loggers/zmq.clj @@ -15,8 +15,7 @@ [clojure.core.async :as a] [clojure.spec.alpha :as s] [cuerdas.core :as str] - [integrant.core :as ig] - [jsonista.core :as j]) + [integrant.core :as ig]) (:import org.zeromq.SocketType org.zeromq.ZMQ$Socket @@ -54,10 +53,10 @@ [_ f] (a/close! (::buffer (meta f)))) -(def json-mapper - (j/object-mapper - {:encode-key-fn str/camel - :decode-key-fn (comp keyword str/kebab)})) +(def ^:private json-mapper + (json/mapper + {:encode-key-fn str/camel + :decode-key-fn (comp keyword str/kebab)})) (defn- start-rcv-loop ([] (start-rcv-loop nil)) @@ -70,7 +69,7 @@ (.. socket (setReceiveTimeOut 5000)) (loop [] (let [msg (.recv ^ZMQ$Socket socket) - msg (ex/ignoring (j/read-value msg json-mapper)) + msg (ex/ignoring (json/read msg json-mapper)) msg (if (nil? msg) :empty msg)] (if (a/>!! out msg) (recur) @@ -78,7 +77,6 @@ (.close ^java.lang.AutoCloseable socket) (.close ^java.lang.AutoCloseable zctx)))))))) - (s/def ::logger-name string?) (s/def ::level string?) (s/def ::thread string?) diff --git a/backend/src/app/msgbus.clj b/backend/src/app/msgbus.clj index 146da0c833..dec1335167 100644 --- a/backend/src/app/msgbus.clj +++ b/backend/src/app/msgbus.clj @@ -179,18 +179,18 @@ ;; Add a unique listener to connection (.addListener sub-conn (reify RedisPubSubListener - (message [it pattern topic message]) - (message [it topic message] + (message [_ _pattern _topic _message]) + (message [_ topic message] ;; There are no back pressure, so we use a slidding ;; buffer for cases when the pubsub broker sends ;; more messages that we can process. (let [val {:topic topic :message (blob/decode message)}] (when-not (a/offer! rcv-ch val) (l/warn :msg "dropping message on subscription loop")))) - (psubscribed [it pattern count]) - (punsubscribed [it pattern count]) - (subscribed [it topic count]) - (unsubscribed [it topic count]))) + (psubscribed [_ _pattern _count]) + (punsubscribed [_ _pattern _count]) + (subscribed [_ _topic _count]) + (unsubscribed [_ _topic _count]))) (letfn [(subscribe-to-single-topic [nsubs topic chan] (let [nsubs (if (nil? nsubs) #{chan} (conj nsubs chan))] diff --git a/backend/src/app/storage/impl.clj b/backend/src/app/storage/impl.clj index 4c3a619009..3c9c6a7d0f 100644 --- a/backend/src/app/storage/impl.clj +++ b/backend/src/app/storage/impl.clj @@ -117,11 +117,11 @@ io/IOFactory (make-reader [_ opts] (io/make-reader path opts)) - (make-writer [_ opts] + (make-writer [_ _] (throw (UnsupportedOperationException. "not implemented"))) (make-input-stream [_ opts] (io/make-input-stream path opts)) - (make-output-stream [_ opts] + (make-output-stream [_ _] (throw (UnsupportedOperationException. "not implemented"))) clojure.lang.Counted (count [_] size) @@ -138,11 +138,11 @@ io/IOFactory (make-reader [_ opts] (io/make-reader bais opts)) - (make-writer [_ opts] + (make-writer [_ _] (throw (UnsupportedOperationException. "not implemented"))) (make-input-stream [_ opts] (io/make-input-stream bais opts)) - (make-output-stream [_ opts] + (make-output-stream [_ _] (throw (UnsupportedOperationException. "not implemented"))) clojure.lang.Counted @@ -159,11 +159,11 @@ io/IOFactory (make-reader [_ opts] (io/make-reader is opts)) - (make-writer [_ opts] + (make-writer [_ _] (throw (UnsupportedOperationException. "not implemented"))) (make-input-stream [_ opts] (io/make-input-stream is opts)) - (make-output-stream [_ opts] + (make-output-stream [_ _] (throw (UnsupportedOperationException. "not implemented"))) clojure.lang.Counted diff --git a/backend/src/app/tasks/telemetry.clj b/backend/src/app/tasks/telemetry.clj index 44a232aeb0..f9441be124 100644 --- a/backend/src/app/tasks/telemetry.clj +++ b/backend/src/app/tasks/telemetry.clj @@ -59,7 +59,7 @@ response (http/send! {:method :post :uri (:uri cfg) :headers {"content-type" "application/json"} - :body (json/encode-str data)})] + :body (json/write-str data)})] (when (> (:status response) 206) (ex/raise :type :internal :code :invalid-response diff --git a/backend/src/app/util/json.clj b/backend/src/app/util/json.clj index 0ffd859d1e..edc204c1f5 100644 --- a/backend/src/app/util/json.clj +++ b/backend/src/app/util/json.clj @@ -9,22 +9,27 @@ (:require [jsonista.core :as j])) -(defn encode-str - [v] - (j/write-value-as-string v j/keyword-keys-object-mapper)) +(defn mapper + [params] + (j/object-mapper params)) + +(defn write + ([v] (j/write-value-as-bytes v j/keyword-keys-object-mapper)) + ([v mapper] (j/write-value-as-bytes v mapper))) + +(defn write-str + ([v] (j/write-value-as-string v j/keyword-keys-object-mapper)) + ([v mapper] (j/write-value-as-string v mapper))) + +(defn read + ([v] (j/read-value v j/keyword-keys-object-mapper)) + ([v mapper] (j/read-value v mapper))) (defn encode [v] (j/write-value-as-bytes v j/keyword-keys-object-mapper)) -(defn decode-str - [v] - (j/read-value v j/keyword-keys-object-mapper)) - (defn decode [v] (j/read-value v j/keyword-keys-object-mapper)) -(defn read - [v] - (j/read-value v j/keyword-keys-object-mapper)) From 1839397ebc8424306eb1c99fd0670be82c78bcaa Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Thu, 23 Dec 2021 18:36:58 +0100 Subject: [PATCH 3/5] :sparkles: Minor enhacements on log processing. --- .clj-kondo/config.edn | 3 +- .clj-kondo/hooks/export.clj | 2 - backend/src/app/http.clj | 17 ++----- backend/src/app/http/errors.clj | 73 ++++++++++-------------------- backend/src/app/http/session.clj | 1 - backend/src/app/worker.clj | 9 +--- common/src/app/common/data.cljc | 2 +- common/src/app/common/logging.cljc | 66 +++++++++++++++++++-------- common/src/app/common/spec.cljc | 9 +--- frontend/src/app/main/errors.cljs | 9 +--- 10 files changed, 83 insertions(+), 108 deletions(-) diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.edn index d1e91f24bb..4ba5b0442a 100644 --- a/.clj-kondo/config.edn +++ b/.clj-kondo/config.edn @@ -3,7 +3,8 @@ rumext.alpha/defc clojure.core/defn rumext.alpha/fnc clojure.core/fn app.common.data/export clojure.core/def - app.db/with-atomic clojure.core/with-open} + app.db/with-atomic clojure.core/with-open + app.common.logging/with-context clojure.core/do} :hooks {:analyze-call diff --git a/.clj-kondo/hooks/export.clj b/.clj-kondo/hooks/export.clj index f66d027839..16ab4e76af 100644 --- a/.clj-kondo/hooks/export.clj +++ b/.clj-kondo/hooks/export.clj @@ -74,5 +74,3 @@ ;; (prn "==============" rtype (into {} ?meta)) ;; (prn (api/sexpr result)) {:node result})) - - diff --git a/backend/src/app/http.clj b/backend/src/app/http.clj index 0dc98852b1..3d8185be19 100644 --- a/backend/src/app/http.clj +++ b/backend/src/app/http.clj @@ -90,20 +90,9 @@ (try (handler request) (catch Throwable e - (try - (let [cdata (errors/get-error-context request e)] - (l/update-thread-context! cdata) - (l/error :hint "unhandled exception" - :message (ex-message e) - :error-id (str (:id cdata)) - :cause e)) - {:status 500 :body "internal server error"} - (catch Throwable e - (l/error :hint "unhandled exception" - :message (ex-message e) - :cause e) - {:status 500 :body "internal server error"}))))))) - + (l/with-context (errors/get-error-context request e) + (l/error :hint (ex-message e) :cause e) + {:status 500 :body "internal server error"})))))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Http Main Handler (Router) diff --git a/backend/src/app/http/errors.clj b/backend/src/app/http/errors.clj index ec489054e8..531f59727e 100644 --- a/backend/src/app/http/errors.clj +++ b/backend/src/app/http/errors.clj @@ -7,7 +7,6 @@ (ns app.http.errors "A errors handling for the http server." (:require - [app.common.data :as d] [app.common.exceptions :as ex] [app.common.logging :as l] [app.common.uuid :as uuid] @@ -20,36 +19,18 @@ (get headers "x-real-ip") (get request :remote-addr))) - -(defn- simple-prune - ([s] (simple-prune s (* 1024 1024))) - ([s max-length] - (if (> (count s) max-length) - (str (subs s 0 max-length) " [...]") - s))) - -(defn- stringify-data - [data] - (binding [clojure.pprint/*print-right-margin* 200] - (let [result (with-out-str (clojure.pprint/pprint data))] - (simple-prune result (* 1024 1024))))) - (defn get-error-context [request error] (let [data (ex-data error)] - (d/without-nils - (merge - {:id (str (uuid/next)) - :path (str (:uri request)) - :method (name (:request-method request)) - :hint (or (:hint data) (ex-message error)) - :params (stringify-data (:params request)) - :data (stringify-data (dissoc data :explain)) - :ip-addr (parse-client-ip request) - :explain (str/prune (:explain data) (* 1024 1024) "[...]")} - - (when-let [id (:profile-id request)] - {:profile-id id}) + (merge + {:id (uuid/next) + :path (:uri request) + :method (:request-method request) + :hint (or (:hint data) (ex-message error)) + :params (l/stringify-data (:params request)) + :data (l/stringify-data (dissoc data :explain)) + :ip-addr (parse-client-ip request) + :profile-id (:profile-id request)} (let [headers (:headers request)] {:user-agent (get headers "user-agent") @@ -57,7 +38,7 @@ (when (map? data) {:error-type (:type data) - :error-code (:code data)}))))) + :error-code (:code data)})))) (defmulti handle-exception (fn [err & _rest] @@ -89,13 +70,9 @@ (defmethod handle-exception :assertion [error request] - (let [edata (ex-data error) - cdata (get-error-context request error)] - (l/update-thread-context! cdata) - (l/error :hint "internal error: assertion" - :error-id (str (:id cdata)) - :cause error) - + (let [edata (ex-data error)] + (l/with-context (get-error-context request error) + (l/error :hint "internal error: assertion" :cause error)) {:status 500 :body {:type :server-error :code :assertion @@ -116,12 +93,10 @@ (if (and (ex/exception? (:rollback edata)) (ex/exception? (:handling edata))) (handle-exception (:handling edata) request) - (let [cdata (get-error-context request error)] - (l/update-thread-context! cdata) - (l/error :hint "internal error" - :error-message (ex-message error) - :error-id (str (:id cdata)) - :cause error) + (do + (l/with-context (get-error-context request error) + (l/error :hint (ex-message error) :cause error)) + {:status 500 :body {:type :server-error :code :unexpected @@ -130,15 +105,13 @@ (defmethod handle-exception org.postgresql.util.PSQLException [error request] - (let [cdata (get-error-context request error) - state (.getSQLState ^java.sql.SQLException error)] + (let [state (.getSQLState ^java.sql.SQLException error)] - (l/update-thread-context! cdata) - (l/error :hint "psql exception" - :error-message (ex-message error) - :error-id (str (:id cdata)) - :sql-state state - :cause error) + (l/with-context (get-error-context request error) + (l/error :hint "psql exception" + :error-message (ex-message error) + :state state + :cause error)) (cond (= state "57014") diff --git a/backend/src/app/http/session.clj b/backend/src/app/http/session.clj index db7207d15e..2888961402 100644 --- a/backend/src/app/http/session.clj +++ b/backend/src/app/http/session.clj @@ -73,7 +73,6 @@ (if-let [{:keys [id profile-id] :as session} (retrieve-from-request cfg request)] (do (a/>!! (::events-ch cfg) id) - (l/update-thread-context! {:profile-id profile-id}) (handler (assoc request :profile-id profile-id))) (handler request)))) diff --git a/backend/src/app/worker.clj b/backend/src/app/worker.clj index 49370e164e..6d979da8cf 100644 --- a/backend/src/app/worker.clj +++ b/backend/src/app/worker.clj @@ -266,13 +266,8 @@ (= ::noop (:strategy edata)) (assoc :inc-by 0)) - - (let [cdata (get-error-context error item)] - (l/update-thread-context! cdata) - (l/error :cause error - :hint "unhandled exception on task" - :id (:id cdata)) - + (l/with-context (get-error-context error item) + (l/error :cause error :hint "unhandled exception on task") (if (>= (:retry-num item) (:max-retries item)) {:status :failed :task item :error error} {:status :retry :task item :error error}))))) diff --git a/common/src/app/common/data.cljc b/common/src/app/common/data.cljc index a39c6c38af..d43d9eb304 100644 --- a/common/src/app/common/data.cljc +++ b/common/src/app/common/data.cljc @@ -6,7 +6,7 @@ (ns app.common.data "Data manipulation and query helper functions." - (:refer-clojure :exclude [read-string hash-map merge name]) + (:refer-clojure :exclude [read-string hash-map merge name parse-double]) #?(:cljs (:require-macros [app.common.data])) (:require diff --git a/common/src/app/common/logging.cljc b/common/src/app/common/logging.cljc index a861bcc27d..29116136c0 100644 --- a/common/src/app/common/logging.cljc +++ b/common/src/app/common/logging.cljc @@ -17,9 +17,12 @@ org.apache.logging.log4j.LogManager org.apache.logging.log4j.Logger org.apache.logging.log4j.ThreadContext + org.apache.logging.log4j.CloseableThreadContext org.apache.logging.log4j.message.MapMessage org.apache.logging.log4j.spi.LoggerContext))) +#?(:clj (set! *warn-on-reflection* true)) + #?(:clj (defn build-map-message [m] @@ -37,14 +40,13 @@ (defn get-logger [lname] #?(:clj (.getLogger ^LoggerContext logger-context ^String lname) - :cljs - (glog/getLogger - (cond - (string? lname) lname - (= lname :root) "" - (simple-ident? lname) (name lname) - (qualified-ident? lname) (str (namespace lname) "." (name lname)) - :else (str lname))))) + :cljs (glog/getLogger + (cond + (string? lname) lname + (= lname :root) "" + (simple-ident? lname) (name lname) + (qualified-ident? lname) (str (namespace lname) "." (name lname)) + :else (str lname))))) (defn get-level [level] @@ -151,20 +153,48 @@ ;; CLJ Specific ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +(defn- simple-prune + ([s] (simple-prune s (* 1024 1024))) + ([s max-length] + (if (> (count s) max-length) + (str (subs s 0 max-length) " [...]") + s))) + #?(:clj - (defn update-thread-context! + (defn stringify-data + [val] + (cond + (instance? clojure.lang.Named val) + (name val) + + (string? val) + val + + (coll? val) + (binding [clojure.pprint/*print-right-margin* 120] + (-> (with-out-str (pprint val)) + (simple-prune (* 1024 1024 3)))) + + :else + (str val)))) + +#?(:clj + (defn data->context-map + ^java.util.Map [data] - (run! (fn [[key val]] - (ThreadContext/put - (name key) - (cond - (coll? val) - (binding [clojure.pprint/*print-right-margin* 120] - (with-out-str (pprint val))) - (instance? clojure.lang.Named val) (name val) - :else (str val)))) + (into {} + (comp (filter second) + (map (fn [[key val]] + [(stringify-data key) + (stringify-data val)]))) data))) +#?(:clj + (defmacro with-context + [data & body] + `(with-open [instance# (CloseableThreadContext/putAll (data->context-map ~data))] + ~@body))) + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; CLJS Specific ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; diff --git a/common/src/app/common/spec.cljc b/common/src/app/common/spec.cljc index e7614707ba..5b46c81b26 100644 --- a/common/src/app/common/spec.cljc +++ b/common/src/app/common/spec.cljc @@ -211,13 +211,11 @@ [spec x message context] (if (s/valid? spec x) x - (let [data (s/explain-data spec x) - explain (with-out-str (s/explain-out data))] + (let [data (s/explain-data spec x)] (ex/raise :type :assertion :code :spec-validation :hint message :data data - :explain explain :context context #?@(:cljs [:stack (.-stack (ex-info message {}))]))))) @@ -253,12 +251,9 @@ [spec data] (let [result (s/conform spec data)] (when (= result ::s/invalid) - (let [data (s/explain-data spec data) - explain (with-out-str - (s/explain-out data))] + (let [data (s/explain-data spec data)] (throw (ex/error :type :validation :code :spec-validation - :explain explain :data data)))) result)) diff --git a/frontend/src/app/main/errors.cljs b/frontend/src/app/main/errors.cljs index ba8ca731b7..6514e8332d 100644 --- a/frontend/src/app/main/errors.cljs +++ b/frontend/src/app/main/errors.cljs @@ -81,10 +81,7 @@ (js/console.group "Validation Error:") (ex/ignoring (js/console.info - (with-out-str - (pprint (dissoc error :explain)))) - (when-let [explain (:explain error)] - (js/console.error explain))) + (with-out-str (pprint error)))) (js/console.groupEnd "Validation Error:")) @@ -138,8 +135,7 @@ (defmethod ptk/handle-error :server-error [{:keys [data hint] :as error}] (let [hint (or hint (:hint data) (:message data)) - info (with-out-str (pprint (dissoc data :explain))) - expl (:explain data) + info (with-out-str (pprint data)) msg (str "Internal Server Error: " hint)] (ts/schedule @@ -150,7 +146,6 @@ (js/console.group msg) (js/console.info info) - (when expl (js/console.error expl)) (js/console.groupEnd msg))) (defn on-unhandled-error From 5c6d72b353c70ce823c1e8126aac25039177e271 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Fri, 24 Dec 2021 12:40:44 +0100 Subject: [PATCH 4/5] :sparkles: Improve logging performance and format. --- backend/deps.edn | 3 - backend/dev/user.clj | 7 ++ backend/resources/error-report.tmpl | 40 ++++++--- backend/resources/log4j2-devenv.xml | 2 +- backend/scripts/repl | 1 - backend/src/app/http/errors.clj | 27 +++--- backend/src/app/loggers/database.clj | 11 +-- backend/src/app/loggers/zmq.clj | 22 ++--- common/deps.edn | 1 + common/src/app/common/logging.cljc | 119 +++++++++++++++------------ common/src/app/common/spec.cljc | 24 +++--- 11 files changed, 143 insertions(+), 114 deletions(-) diff --git a/backend/deps.edn b/backend/deps.edn index b7f3295baa..c89f7bb28c 100644 --- a/backend/deps.edn +++ b/backend/deps.edn @@ -32,7 +32,6 @@ metosin/reitit-ring {:mvn/version "0.5.15"} org.postgresql/postgresql {:mvn/version "42.2.23"} com.zaxxer/HikariCP {:mvn/version "5.0.0"} - funcool/datoteka {:mvn/version "2.0.0"} buddy/buddy-core {:mvn/version "1.10.1"} @@ -49,9 +48,7 @@ io.sentry/sentry {:mvn/version "5.1.2"} ;; Pretty Print specs - fipp/fipp {:mvn/version "0.6.24"} pretty-spec/pretty-spec {:mvn/version "0.1.4"} - software.amazon.awssdk/s3 {:mvn/version "2.17.40"}} :paths ["src" "resources"] diff --git a/backend/dev/user.clj b/backend/dev/user.clj index 4f47934a8b..d65cd01cd9 100644 --- a/backend/dev/user.clj +++ b/backend/dev/user.clj @@ -95,3 +95,10 @@ [{:v1 (alength (blob/encode data {:version 1})) :v2 (alength (blob/encode data {:version 2})) :v3 (alength (blob/encode data {:version 3}))}])) + + +(defonce debug-tap + (do + (add-tap #(locking debug-tap + (prn "tap debug:" %))) + 1)) diff --git a/backend/resources/error-report.tmpl b/backend/resources/error-report.tmpl index 452036997e..d6b3a4dbb4 100644 --- a/backend/resources/error-report.tmpl +++ b/backend/resources/error-report.tmpl @@ -130,10 +130,10 @@ {% endif %} - {% if error %} + {% if hint %}
HINT:
-
{{error.message}}
+
{{hint}}
{% endif %} @@ -163,25 +163,39 @@ {% endif %} - {% if explain %} -
-
EXPLAIN:
-
- 
{{explain}}
-
-
- {% endif %} - {% if data %}
-
EDATA:
+
ERROR DATA: 
{{data}}
{% endif %} - {% if error %} + {% if spec-problems %} +
+
SPEC PROBLEMS:
+
+ 
{{spec-problems}}
+
+
+ {% endif %} + + {% if cause %} +
+
TRACE:
+
+ 
{{cause}}
+
+
+ {% elif trace %} +
+
TRACE:
+
+ 
{{trace}}
+
+
+ {% elif error %}
TRACE:
diff --git a/backend/resources/log4j2-devenv.xml b/backend/resources/log4j2-devenv.xml index 1b9dba567b..d07a33d7dd 100644 --- a/backend/resources/log4j2-devenv.xml +++ b/backend/resources/log4j2-devenv.xml @@ -2,7 +2,7 @@ - + diff --git a/backend/scripts/repl b/backend/scripts/repl index f2377e23d9..3ca39aa9c5 100755 --- a/backend/scripts/repl +++ b/backend/scripts/repl @@ -2,7 +2,6 @@ export PENPOT_FLAGS="enable-asserts enable-audit-log $PENPOT_FLAGS" -#-J-Dlog4j2.contextSelector=org.apache.logging.log4j.core.async.AsyncLoggerContextSelector export OPTIONS=" -A:jmx-remote:dev \ diff --git a/backend/src/app/http/errors.clj b/backend/src/app/http/errors.clj index 531f59727e..fac7094471 100644 --- a/backend/src/app/http/errors.clj +++ b/backend/src/app/http/errors.clj @@ -11,6 +11,7 @@ [app.common.logging :as l] [app.common.uuid :as uuid] [clojure.pprint] + [clojure.spec.alpha :as s] [cuerdas.core :as str])) (defn- parse-client-ip @@ -23,22 +24,20 @@ [request error] (let [data (ex-data error)] (merge - {:id (uuid/next) - :path (:uri request) - :method (:request-method request) - :hint (or (:hint data) (ex-message error)) - :params (l/stringify-data (:params request)) - :data (l/stringify-data (dissoc data :explain)) - :ip-addr (parse-client-ip request) - :profile-id (:profile-id request)} + {:id (uuid/next) + :path (:uri request) + :method (:request-method request) + :hint (or (:hint data) (ex-message error)) + :params (l/stringify-data (:params request)) + :spec-problems (some-> data ::s/problems) + :ip-addr (parse-client-ip request) + :profile-id (:profile-id request)} (let [headers (:headers request)] {:user-agent (get headers "user-agent") :frontend-version (get headers "x-frontend-version" "unknown")}) - (when (map? data) - {:error-type (:type data) - :error-code (:code data)})))) + (dissoc data ::s/problems)))) (defmulti handle-exception (fn [err & _rest] @@ -66,17 +65,17 @@ (:explain edata) "\n")} {:status 400 - :body (dissoc edata :data)}))) + :body (dissoc edata ::s/problems)}))) (defmethod handle-exception :assertion [error request] (let [edata (ex-data error)] (l/with-context (get-error-context request error) - (l/error :hint "internal error: assertion" :cause error)) + (l/error :hint (ex-message error) :cause error)) {:status 500 :body {:type :server-error :code :assertion - :data (dissoc edata :data)}})) + :data (dissoc edata ::s/problems)}})) (defmethod handle-exception :not-found [err _] diff --git a/backend/src/app/loggers/database.clj b/backend/src/app/loggers/database.clj index 2685661d82..ca0fb5d6eb 100644 --- a/backend/src/app/loggers/database.clj +++ b/backend/src/app/loggers/database.clj @@ -36,7 +36,7 @@ (db/insert! conn :server-error-report {:id id :content (db/tjson event)}))) -(defn- parse-context +(defn- parse-event-data [event] (reduce-kv (fn [acc k v] @@ -46,12 +46,11 @@ (str/blank? v) acc :else (assoc acc k v))) {} - (:context event))) + event)) (defn parse-event [event] - (-> (parse-context event) - (merge (dissoc event :context)) + (-> (parse-event-data event) (assoc :tenant (cf/get :tenant)) (assoc :host (cf/get :host)) (assoc :public-uri (cf/get :public-uri)) @@ -62,6 +61,7 @@ (aa/with-thread executor (try (let [event (parse-event event)] + (l/debug :hint "registering error on database" :id (:id event)) (persist-on-database! cfg event)) (catch Exception e (l/warn :hint "unexpected exception on database error logger" @@ -74,7 +74,8 @@ [_ {:keys [receiver] :as cfg}] (l/info :msg "initializing database error persistence") (let [output (a/chan (a/sliding-buffer 128) - (filter #(= (:level %) "error")))] + (filter (fn [event] + (= (:logger/level event) "error"))))] (receiver :sub output) (a/go-loop [] (let [msg (a/ (count s) max-length) + (str (subs s 0 max-length) " [...]") + s))) + +#?(:clj + (defn stringify-data + [val] + (cond + (instance? clojure.lang.Named val) + (name val) + + (instance? Throwable val) + (binding [ie/*app-frame-names* [#"app.*"] + ie/*fonts* nil + ie/*traditional* true] + (ie/format-exception val nil)) + + (string? val) + val + + (coll? val) + (binding [clojure.pprint/*print-right-margin* 120] + (-> (with-out-str (pprint val)) + (simple-prune (* 1024 1024 3)))) + + :else + (str val)))) + +#?(:clj + (defn data->context-map + ^java.util.Map + [data] + (into {} + (comp (filter second) + (map (fn [[key val]] + [(stringify-data key) + (stringify-data val)]))) + data))) + +#?(:clj + (defmacro with-context + [data & body] + `(let [data# (data->context-map ~data)] + (with-open [closeable# (CloseableThreadContext/putAll data#)] + ~@body)))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Common +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + (defn get-logger [lname] #?(:clj (.getLogger ^LoggerContext logger-context ^String lname) @@ -89,7 +147,7 @@ :cljs (when glog/ENABLED (when-let [l (get-logger logger)] - (let [level (get-level level) + (let [level (get-level level) record (glog/LogRecord. level message (.getName ^js l))] (when exception (.setException record exception)) (glog/publishLogRecord l record)))))) @@ -100,7 +158,7 @@ (.isEnabled ^Logger logger ^Level level))) (defmacro log - [& {:keys [level cause ::logger ::async ::raw] :as props}] + [& {:keys [level cause ::logger ::async ::raw] :or {async true} :as props}] (if (:ns &env) ; CLJS `(write-log! ~(or logger (str *ns*)) ~level @@ -114,10 +172,12 @@ ~level-sym (get-level ~level)] (if (enabled? ~logger-sym ~level-sym) ~(if async - `(send-off logging-agent - (fn [_#] - (let [message# (or ~raw (build-map-message ~props))] - (write-log! ~logger-sym ~level-sym ~cause message#)))) + `(let [cdata# (ThreadContext/getImmutableContext)] + (send-off logging-agent + (fn [_#] + (with-context (into {:cause ~cause} cdata#) + (->> (or ~raw (build-map-message ~props)) + (write-log! ~logger-sym ~level-sym ~cause)))))) `(let [message# (or ~raw (build-map-message ~props))] (write-log! ~logger-sym ~level-sym ~cause message#)))))))) @@ -149,52 +209,6 @@ (when (:ns &env) `(set-level* ~n ~level)))) -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;; CLJ Specific -;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; - -(defn- simple-prune - ([s] (simple-prune s (* 1024 1024))) - ([s max-length] - (if (> (count s) max-length) - (str (subs s 0 max-length) " [...]") - s))) - -#?(:clj - (defn stringify-data - [val] - (cond - (instance? clojure.lang.Named val) - (name val) - - (string? val) - val - - (coll? val) - (binding [clojure.pprint/*print-right-margin* 120] - (-> (with-out-str (pprint val)) - (simple-prune (* 1024 1024 3)))) - - :else - (str val)))) - -#?(:clj - (defn data->context-map - ^java.util.Map - [data] - (into {} - (comp (filter second) - (map (fn [[key val]] - [(stringify-data key) - (stringify-data val)]))) - data))) - -#?(:clj - (defmacro with-context - [data & body] - `(with-open [instance# (CloseableThreadContext/putAll (data->context-map ~data))] - ~@body))) - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; CLJS Specific ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -243,7 +257,6 @@ (some-> (get-logger name) (glog/setLevel (get-level lvl))))) - #?(:cljs (defn set-levels! [lvls] diff --git a/common/src/app/common/spec.cljc b/common/src/app/common/spec.cljc index 5b46c81b26..5a2f37065d 100644 --- a/common/src/app/common/spec.cljc +++ b/common/src/app/common/spec.cljc @@ -208,28 +208,30 @@ ;; --- Macros (defn spec-assert* - [spec x message context] - (if (s/valid? spec x) - x - (let [data (s/explain-data spec x)] + [spec val hint ctx] + (if (s/valid? spec val) + val + (let [data (s/explain-data spec val)] (ex/raise :type :assertion :code :spec-validation - :hint message - :data data - :context context - #?@(:cljs [:stack (.-stack (ex-info message {}))]))))) - + :hint hint + :ctx ctx + ::s/problems (::s/problems data))))) (defmacro assert "Development only assertion macro." [spec x] (when *assert* (let [nsdata (:ns &env) - context (when nsdata + context (if nsdata {:ns (str (:name nsdata)) :name (pr-str spec) :line (:line &env) - :file (:file (:meta nsdata))}) + :file (:file (:meta nsdata))} + (let [mdata (meta &form)] + {:ns (str (ns-name *ns*)) + :name (pr-str spec) + :line (:line mdata)})) message (str "spec assert: '" (pr-str spec) "'")] `(spec-assert* ~spec ~x ~message ~context)))) From 4bc2d7444d9417a95a998730e3af80f53780503c Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Mon, 27 Dec 2021 09:30:58 +0100 Subject: [PATCH 5/5] :paperclip: Minor changes on dev tools. --- common/src/app/common/logging.cljc | 19 +++-- frontend/deps.edn | 3 +- frontend/dev/bench/core.cljs | 112 ----------------------------- frontend/dev/cljs/user.cljs | 5 ++ frontend/shadow-cljs.edn | 1 - 5 files changed, 16 insertions(+), 124 deletions(-) delete mode 100644 frontend/dev/bench/core.cljs create mode 100644 frontend/dev/cljs/user.cljs diff --git a/common/src/app/common/logging.cljc b/common/src/app/common/logging.cljc index 35e22aca85..f9356e5880 100644 --- a/common/src/app/common/logging.cljc +++ b/common/src/app/common/logging.cljc @@ -11,16 +11,15 @@ [cuerdas.core :as str] #?(:clj [io.aviso.exception :as ie]) #?(:cljs [goog.log :as glog])) - #?(:cljs (:require-macros [app.common.logging])) - #?(:clj - (:import - org.apache.logging.log4j.Level - org.apache.logging.log4j.LogManager - org.apache.logging.log4j.Logger - org.apache.logging.log4j.ThreadContext - org.apache.logging.log4j.CloseableThreadContext - org.apache.logging.log4j.message.MapMessage - org.apache.logging.log4j.spi.LoggerContext))) + #?(:cljs (:require-macros [app.common.logging]) + :clj (:import + org.apache.logging.log4j.Level + org.apache.logging.log4j.LogManager + org.apache.logging.log4j.Logger + org.apache.logging.log4j.ThreadContext + org.apache.logging.log4j.CloseableThreadContext + org.apache.logging.log4j.message.MapMessage + org.apache.logging.log4j.spi.LoggerContext))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; CLJ Specific diff --git a/frontend/deps.edn b/frontend/deps.edn index 5468b8b552..b2f85c8b1f 100644 --- a/frontend/deps.edn +++ b/frontend/deps.edn @@ -22,7 +22,8 @@ :main-opts ["-m" "antq.core"]} :dev - {:extra-deps + {:extra-paths ["dev"] + :extra-deps {thheller/shadow-cljs {:mvn/version "2.15.12"} cider/cider-nrepl {:mvn/version "0.26.0"}}} diff --git a/frontend/dev/bench/core.cljs b/frontend/dev/bench/core.cljs deleted file mode 100644 index 511f93212c..0000000000 --- a/frontend/dev/bench/core.cljs +++ /dev/null @@ -1,112 +0,0 @@ -(ns bench.core - (:require [kdtree.core :as k] - [intervaltree.core :as it] - [cljs.pprint :refer (pprint)] - [cljs.nodejs :as node])) - -(enable-console-print!) - -;; --- Index Initialization Bechmark - -(defn- bench-init-10000 - [] - (println "1000x1000,10 -> 10000 points") - (time - (k/generate 1000 1000 10 10))) - -(defn- bench-init-250000 - [] - (time - (k/generate 5000 5000 10 10))) - -(defn bench-init - [] - (bench-init-10000) - (bench-init-10000) - (bench-init-250000) - (bench-init-250000) - (bench-init-10000) - (bench-init-10000) - (bench-init-250000) - (bench-init-250000)) - -;; --- Nearest Search Benchmark - -(defn- bench-knn-160000 - [] - (let [tree (k/create)] - (k/setup tree 4000 4000 10 10) - (println "KNN Search (160000 points) 1000 times") - (time - (dotimes [i 1000] - (let [pt #js [(rand-int 400) - (rand-int 400)]] - (k/nearest tree pt 2)))))) - - -(defn- bench-knn-360000 - [] - (let [tree (k/create)] - (k/initialize tree 6000 6000 10 10) - (println "KNN Search (360000 points) 1000 times") - (time - (dotimes [i 1000] - (let [pt #js [(rand-int 600) - (rand-int 600)]] - (k/nearest tree pt 2)))))) - -(defn bench-knn - [] - (bench-knn-160000) - (bench-knn-360000)) - -;; --- Accuracity tests - -(defn test-accuracity - [] - (let [tree (k/create)] - (k/setup tree 4000 4000 20 20) - (print "[1742 1419]") - (pprint (js->clj (k/nearest tree #js [1742 1419] 6))) - (print "[1742 1420]") - (pprint (js->clj (k/nearest tree #js [1742 1420] 6))) - )) - -(defn test-interval - [] - (let [tree (it/create)] - (it/add tree #js [1 5]) - (it/add tree #js [5 7]) - (it/add tree #js [-4 -1]) - (it/add tree #js [-10 -3]) - (it/add tree #js [-20 -10]) - (it/add tree #js [20 30]) - (it/add tree #js [3 9]) - (it/add tree #js [100 200]) - (it/add tree #js [1000 2000]) - (it/add tree #js [6 9]) - - (js/console.dir tree #js {"depth" nil}) - (js/console.log "contains", 4, (it/contains tree 4)) - (js/console.log "contains", 0, (it/contains tree 0)) - )) - -(defn main - [& [type]] - (cond - (= type "kd-init") - (bench-init) - - (= type "kd-search") - (bench-knn) - - (= type "kd-test") - (test-accuracity) - - (= type "interval") - (test-interval) - - :else - (println "not implemented"))) - -(set! *main-cli-fn* main) diff --git a/frontend/dev/cljs/user.cljs b/frontend/dev/cljs/user.cljs new file mode 100644 index 0000000000..3a927046b9 --- /dev/null +++ b/frontend/dev/cljs/user.cljs @@ -0,0 +1,5 @@ +(ns cljs.user) + +(defn hello + [] + (js/console.log "hello")) diff --git a/frontend/shadow-cljs.edn b/frontend/shadow-cljs.edn index 942e2c93d1..8d3bc58e09 100644 --- a/frontend/shadow-cljs.edn +++ b/frontend/shadow-cljs.edn @@ -4,7 +4,6 @@ :jvm-opts ["-Xmx700m" "-Xms100m" "-XX:+UseSerialGC" "-XX:-OmitStackTraceInFastThrow"] :dev-http {8888 "classpath:public"} - :builds {:main {:target :browser