From f63e23599a8efe84c78cf2dbf05778665deb8655 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Sat, 9 May 2026 11:00:24 +0200 Subject: [PATCH] :construction: WIP --- backend/src/app/loggers/audit.clj | 189 ++++++++++++------------- backend/src/app/tasks/telemetry.clj | 6 +- backend/test/backend_tests/helpers.clj | 2 +- 3 files changed, 97 insertions(+), 100 deletions(-) diff --git a/backend/src/app/loggers/audit.clj b/backend/src/app/loggers/audit.clj index a8a862e28d..2f7a2d49af 100644 --- a/backend/src/app/loggers/audit.clj +++ b/backend/src/app/loggers/audit.clj @@ -193,7 +193,6 @@ :type :profile-id :ip-addr - :context :props :context :source @@ -215,105 +214,72 @@ (and (simple-keyword? k) (or (uuid? v) (boolean? v) (number? v)))))) -(defn- filter-telemetry-props - [{:keys [source name props] :as params}] - (cond - (and (= source "backend") - (or (= name "login-with-oidc") - (= name "login-with-password") - (= name "register-profile") - (= name "update-profile"))) - (let [props (select-keys props profile-props) - props (into {} xf:filter-telemetry-props props) - props (-> props - (assoc :lang (:lang props)) - (assoc :auth-backend (:auth-backend props)) - (assoc :email-domain (email/get-domain (:email props))) - (d/without-nils))] - (assoc params :props props)) - - :else - (let [props (into {} xf:filter-telemetry-props props)] - (assoc params :props props)))) - -(defn filter-telemetry-context - [{:keys [source context] :as params}] - (let [context (case source - "backend" (select-keys context safe-backend-context-keys) - "frontend" (select-keys context safe-frontend-context-keys) - {})] - (assoc params :context context))) +(declare filter-telemetry-props) +(declare filter-telemetry-context) (defn- process-event [cfg event] - (let [params (normalize-event event)] + (when (contains? cf/flags :audit-log-logger) + (l/log! ::l/logger "app.audit" + ::l/level :info + :profile-id (str (:profile-id event)) + :ip-addr (str (:ip-addr event)) + :type (:type event) + :name (:name event) + :props (json/encode (:props event) :key-fn json/write-camel-key) + :context (json/encode (:context event) :key-fn json/write-camel-key))) - (when (contains? cf/flags :audit-log-logger) - (l/log! ::l/logger "app.audit" - ::l/level :info - :profile-id (str (:profile-id event)) - :ip-addr (str (:ip-addr event)) - :type (:type event) - :name (:name event) - :props (json/encode (:props event) :key-fn json/write-camel-key) - :context (json/encode (:context event) :key-fn json/write-camel-key))) - - (if (contains? cf/flags :audit-log) + (if (contains? cf/flags :audit-log) + ;; NOTE: this operation may cause primary key conflicts on inserts + ;; because of the timestamp precission (two concurrent requests), in + ;; this case we just retry the operation. + (append-audit-entry cfg event) + (when cf/telemetry-enabled? ;; NOTE: this operation may cause primary key conflicts on inserts ;; because of the timestamp precission (two concurrent requests), in ;; this case we just retry the operation. - (append-audit-entry cfg event) - (when cf/telemetry-enabled? - ;; NOTE: this operation may cause primary key conflicts on inserts - ;; because of the timestamp precission (two concurrent requests), in - ;; this case we just retry the operation. - ;; - ;; NOTE: this is only executed when general audit log is disabled; - ;; events are stored stripped of props and ip-addr, tagged with - ;; source="telemetry" so the telemetry task can collect and ship - ;; them. The profile-id is preserved (UUIDs are already anonymous - ;; random identifiers). Only a safe subset of context fields is - ;; kept: initiator, version, client-version and client-user-agent. - ;; Timestamps are truncated to day precision to avoid leaking exact - ;; event timing. - (let [event-name (get event :name) - tday (ct/truncate tnow :days) - event (-> event - (filter-telemetry-props) - (filter-telemetry-context) - (update :created-at ct/truncate :days) - (update :tracked-at ct/truncate :days) - (assoc :ip-addr "0.0.0.0"))] - (append-audit-entry cfg params)))) + ;; + ;; NOTE: this is only executed when general audit log is disabled; + ;; events are stored stripped of props and ip-addr, tagged with + ;; source="telemetry" so the telemetry task can collect and ship + ;; them. The profile-id is preserved (UUIDs are already anonymous + ;; random identifiers). Only a safe subset of context fields is + ;; kept: initiator, version, client-version and client-user-agent. + ;; Timestamps are truncated to day precision to avoid leaking exact + ;; event timing. + (let [event-name (get event :name) + event (-> event + (filter-telemetry-props) + (filter-telemetry-context) + (update :created-at ct/truncate :days) + (update :tracked-at ct/truncate :days) + (assoc :ip-addr "0.0.0.0"))] + (append-audit-entry cfg event)))) - (when (and (contains? cf/flags :webhooks) - (::webhooks/event? event)) - (let [batch-key (::webhooks/batch-key event) - batch-timeout (::webhooks/batch-timeout event) - label (dm/str "rpc:" (:name params)) - label (cond - (ifn? batch-key) (dm/str label ":" (batch-key (::rpc/params event))) - (string? batch-key) (dm/str label ":" batch-key) - :else label) - dedupe? (boolean (and batch-key batch-timeout))] + (when (and (contains? cf/flags :webhooks) + (::webhooks/event? event)) + (let [batch-key (::webhooks/batch-key event) + batch-timeout (::webhooks/batch-timeout event) + label (dm/str "rpc:" (:name event)) + label (cond + (ifn? batch-key) (dm/str label ":" (batch-key (::rpc/params event))) + (string? batch-key) (dm/str label ":" batch-key) + :else label) + dedupe? (boolean (and batch-key batch-timeout))] - (wrk/submit! (-> cfg - (assoc ::wrk/task :process-webhook-event) - (assoc ::wrk/queue :webhooks) - (assoc ::wrk/max-retries 0) - (assoc ::wrk/delay (or batch-timeout 0)) - (assoc ::wrk/dedupe dedupe?) - (assoc ::wrk/label label) - (assoc ::wrk/params (-> params - (dissoc :source) - (dissoc :context) - (dissoc :ip-addr) - (dissoc :type))))))) - params)) - -(defn- or-ts-now - [o] - (if (inst? o) o (ct/now))) + (wrk/submit! (-> cfg + (assoc ::wrk/task :process-webhook-event) + (assoc ::wrk/queue :webhooks) + (assoc ::wrk/max-retries 0) + (assoc ::wrk/delay (or batch-timeout 0)) + (assoc ::wrk/dedupe dedupe?) + (assoc ::wrk/label label) + (assoc ::wrk/params (-> event + (dissoc :source) + (dissoc :context) + (dissoc :ip-addr) + (dissoc :type))))))) + event) (defn submit* "A public API, lower-leve lhan submit, assumes all required fields are filled" @@ -398,14 +364,45 @@ (some? context) (assoc :context context)))) +(defn filter-telemetry-props + [{:keys [source name props] :as params}] + (cond + (and (= source "backend") + (or (= name "login-with-oidc") + (= name "login-with-password") + (= name "register-profile") + (= name "update-profile"))) + (let [props (select-keys props profile-props) + props (into {} xf:filter-telemetry-props props) + props (-> props + (assoc :lang (:lang props)) + (assoc :auth-backend (:auth-backend props)) + (assoc :email-domain (email/get-domain (:email props))) + (d/without-nils))] + (assoc params :props props)) + + ;; FIXME: add frontend identify + + :else + (let [props (into {} xf:filter-telemetry-props props)] + (assoc params :props props)))) + +(defn filter-telemetry-context + [{:keys [source context] :as params}] + (let [context (case source + "backend" (select-keys context safe-backend-context-keys) + "frontend" (select-keys context safe-frontend-context-keys) + {})] + (assoc params :context context))) + (defn submit "Submit an event to be registered under audit-log subsystem" [cfg event] (let [tnow (ct/now) event (-> event (assoc :created-at tnow) - (update :tracked-at or tnow) - (update :ip-addr or "0.0.0.0") + (update :tracked-at d/nilv tnow) + (update :ip-addr d/nilv "0.0.0.0") (assoc :source "backend") (d/without-nils))] (submit* cfg event))) @@ -419,9 +416,9 @@ (let [tnow (ct/now) event (-> event (assoc :created-at tnow) - (update :tracked-at or tnow) - (update :profile-id or uuid/zero) + (update :tracked-at d/nilv tnow) + (update :profile-id d/nilv uuid/zero) (assoc :source "backend") (select-keys event-keys) (check-event))] - (db/run! cfg append-audit-entry params)))) + (db/run! cfg append-audit-entry event)))) diff --git a/backend/src/app/tasks/telemetry.clj b/backend/src/app/tasks/telemetry.clj index aac6494b3d..196ad0d7ef 100644 --- a/backend/src/app/tasks/telemetry.clj +++ b/backend/src/app/tasks/telemetry.clj @@ -145,7 +145,7 @@ (def ^:private sql:get-counters "SELECT name, count(*) AS count FROM audit_log - WHERE source IN ('backend', 'frontend', 'telemetry:backend', 'telemetry:frontend') + WHERE source IN ('backend', 'frontend') AND created_at >= date_trunc('day', now()) AND created_at < date_trunc('day', now()) + interval '1 day' GROUP BY 1 @@ -218,7 +218,7 @@ (def ^:private sql:gc-events "DELETE FROM audit_log - WHERE source IN ('telemetry:backend', 'telemetry:frontend') + WHERE source IN ('backend', 'frontend') AND created_at < now() - interval '7 days'") (defn- gc-events @@ -233,7 +233,7 @@ (def ^:private sql:fetch-telemetry-events "SELECT id, name, type, source, tracked_at, profile_id, context FROM audit_log - WHERE source IN ('telemetry:backend', 'telemetry:frontend') + WHERE source IN ('backend', 'frontend') ORDER BY created_at ASC LIMIT ?") diff --git a/backend/test/backend_tests/helpers.clj b/backend/test/backend_tests/helpers.clj index 8ddb3448a2..5e8fc40c60 100644 --- a/backend/test/backend_tests/helpers.clj +++ b/backend/test/backend_tests/helpers.clj @@ -83,7 +83,7 @@ [next] (with-redefs [app.config/flags (flags/parse flags/default default-flags) app.config/config config - app.loggers.audit/submit! (constantly nil) + app.loggers.audit/submit (constantly nil) app.auth/derive-password identity app.auth/verify-password (fn [a b] {:valid (= a b)}) app.common.features/get-enabled-features (fn [& _] app.common.features/supported-features)]