mirror of
https://github.com/penpot/penpot.git
synced 2026-05-10 18:48:23 +00:00
🚧 WIP
This commit is contained in:
Andrey Antukh
parent
32627efb96
commit
f63e23599a
@ -193,7 +193,6 @@
|
||||
:type
|
||||
:profile-id
|
||||
:ip-addr
|
||||
:context
|
||||
:props
|
||||
:context
|
||||
:source
|
||||
@ -215,105 +214,72 @@
|
||||
(and (simple-keyword? k)
|
||||
(or (uuid? v) (boolean? v) (number? v))))))
|
||||
|
||||
(defn- filter-telemetry-props
|
||||
[{:keys [source name props] :as params}]
|
||||
(cond
|
||||
(and (= source "backend")
|
||||
(or (= name "login-with-oidc")
|
||||
(= name "login-with-password")
|
||||
(= name "register-profile")
|
||||
(= name "update-profile")))
|
||||
(let [props (select-keys props profile-props)
|
||||
props (into {} xf:filter-telemetry-props props)
|
||||
props (-> props
|
||||
(assoc :lang (:lang props))
|
||||
(assoc :auth-backend (:auth-backend props))
|
||||
(assoc :email-domain (email/get-domain (:email props)))
|
||||
(d/without-nils))]
|
||||
(assoc params :props props))
|
||||
|
||||
:else
|
||||
(let [props (into {} xf:filter-telemetry-props props)]
|
||||
(assoc params :props props))))
|
||||
|
||||
(defn filter-telemetry-context
|
||||
[{:keys [source context] :as params}]
|
||||
(let [context (case source
|
||||
"backend" (select-keys context safe-backend-context-keys)
|
||||
"frontend" (select-keys context safe-frontend-context-keys)
|
||||
{})]
|
||||
(assoc params :context context)))
|
||||
(declare filter-telemetry-props)
|
||||
(declare filter-telemetry-context)
|
||||
|
||||
(defn- process-event
|
||||
[cfg event]
|
||||
(let [params (normalize-event event)]
|
||||
(when (contains? cf/flags :audit-log-logger)
|
||||
(l/log! ::l/logger "app.audit"
|
||||
::l/level :info
|
||||
:profile-id (str (:profile-id event))
|
||||
:ip-addr (str (:ip-addr event))
|
||||
:type (:type event)
|
||||
:name (:name event)
|
||||
:props (json/encode (:props event) :key-fn json/write-camel-key)
|
||||
:context (json/encode (:context event) :key-fn json/write-camel-key)))
|
||||
|
||||
(when (contains? cf/flags :audit-log-logger)
|
||||
(l/log! ::l/logger "app.audit"
|
||||
::l/level :info
|
||||
:profile-id (str (:profile-id event))
|
||||
:ip-addr (str (:ip-addr event))
|
||||
:type (:type event)
|
||||
:name (:name event)
|
||||
:props (json/encode (:props event) :key-fn json/write-camel-key)
|
||||
:context (json/encode (:context event) :key-fn json/write-camel-key)))
|
||||
|
||||
(if (contains? cf/flags :audit-log)
|
||||
(if (contains? cf/flags :audit-log)
|
||||
;; NOTE: this operation may cause primary key conflicts on inserts
|
||||
;; because of the timestamp precission (two concurrent requests), in
|
||||
;; this case we just retry the operation.
|
||||
(append-audit-entry cfg event)
|
||||
(when cf/telemetry-enabled?
|
||||
;; NOTE: this operation may cause primary key conflicts on inserts
|
||||
;; because of the timestamp precission (two concurrent requests), in
|
||||
;; this case we just retry the operation.
|
||||
(append-audit-entry cfg event)
|
||||
(when cf/telemetry-enabled?
|
||||
;; NOTE: this operation may cause primary key conflicts on inserts
|
||||
;; because of the timestamp precission (two concurrent requests), in
|
||||
;; this case we just retry the operation.
|
||||
;;
|
||||
;; NOTE: this is only executed when general audit log is disabled;
|
||||
;; events are stored stripped of props and ip-addr, tagged with
|
||||
;; source="telemetry" so the telemetry task can collect and ship
|
||||
;; them. The profile-id is preserved (UUIDs are already anonymous
|
||||
;; random identifiers). Only a safe subset of context fields is
|
||||
;; kept: initiator, version, client-version and client-user-agent.
|
||||
;; Timestamps are truncated to day precision to avoid leaking exact
|
||||
;; event timing.
|
||||
(let [event-name (get event :name)
|
||||
tday (ct/truncate tnow :days)
|
||||
event (-> event
|
||||
(filter-telemetry-props)
|
||||
(filter-telemetry-context)
|
||||
(update :created-at ct/truncate :days)
|
||||
(update :tracked-at ct/truncate :days)
|
||||
(assoc :ip-addr "0.0.0.0"))]
|
||||
(append-audit-entry cfg params))))
|
||||
;;
|
||||
;; NOTE: this is only executed when general audit log is disabled;
|
||||
;; events are stored stripped of props and ip-addr, tagged with
|
||||
;; source="telemetry" so the telemetry task can collect and ship
|
||||
;; them. The profile-id is preserved (UUIDs are already anonymous
|
||||
;; random identifiers). Only a safe subset of context fields is
|
||||
;; kept: initiator, version, client-version and client-user-agent.
|
||||
;; Timestamps are truncated to day precision to avoid leaking exact
|
||||
;; event timing.
|
||||
(let [event-name (get event :name)
|
||||
event (-> event
|
||||
(filter-telemetry-props)
|
||||
(filter-telemetry-context)
|
||||
(update :created-at ct/truncate :days)
|
||||
(update :tracked-at ct/truncate :days)
|
||||
(assoc :ip-addr "0.0.0.0"))]
|
||||
(append-audit-entry cfg event))))
|
||||
|
||||
(when (and (contains? cf/flags :webhooks)
|
||||
(::webhooks/event? event))
|
||||
(let [batch-key (::webhooks/batch-key event)
|
||||
batch-timeout (::webhooks/batch-timeout event)
|
||||
label (dm/str "rpc:" (:name params))
|
||||
label (cond
|
||||
(ifn? batch-key) (dm/str label ":" (batch-key (::rpc/params event)))
|
||||
(string? batch-key) (dm/str label ":" batch-key)
|
||||
:else label)
|
||||
dedupe? (boolean (and batch-key batch-timeout))]
|
||||
(when (and (contains? cf/flags :webhooks)
|
||||
(::webhooks/event? event))
|
||||
(let [batch-key (::webhooks/batch-key event)
|
||||
batch-timeout (::webhooks/batch-timeout event)
|
||||
label (dm/str "rpc:" (:name event))
|
||||
label (cond
|
||||
(ifn? batch-key) (dm/str label ":" (batch-key (::rpc/params event)))
|
||||
(string? batch-key) (dm/str label ":" batch-key)
|
||||
:else label)
|
||||
dedupe? (boolean (and batch-key batch-timeout))]
|
||||
|
||||
(wrk/submit! (-> cfg
|
||||
(assoc ::wrk/task :process-webhook-event)
|
||||
(assoc ::wrk/queue :webhooks)
|
||||
(assoc ::wrk/max-retries 0)
|
||||
(assoc ::wrk/delay (or batch-timeout 0))
|
||||
(assoc ::wrk/dedupe dedupe?)
|
||||
(assoc ::wrk/label label)
|
||||
(assoc ::wrk/params (-> params
|
||||
(dissoc :source)
|
||||
(dissoc :context)
|
||||
(dissoc :ip-addr)
|
||||
(dissoc :type)))))))
|
||||
params))
|
||||
|
||||
(defn- or-ts-now
|
||||
[o]
|
||||
(if (inst? o) o (ct/now)))
|
||||
(wrk/submit! (-> cfg
|
||||
(assoc ::wrk/task :process-webhook-event)
|
||||
(assoc ::wrk/queue :webhooks)
|
||||
(assoc ::wrk/max-retries 0)
|
||||
(assoc ::wrk/delay (or batch-timeout 0))
|
||||
(assoc ::wrk/dedupe dedupe?)
|
||||
(assoc ::wrk/label label)
|
||||
(assoc ::wrk/params (-> event
|
||||
(dissoc :source)
|
||||
(dissoc :context)
|
||||
(dissoc :ip-addr)
|
||||
(dissoc :type)))))))
|
||||
event)
|
||||
|
||||
(defn submit*
|
||||
"A public API, lower-leve lhan submit, assumes all required fields are filled"
|
||||
@ -398,14 +364,45 @@
|
||||
(some? context)
|
||||
(assoc :context context))))
|
||||
|
||||
(defn filter-telemetry-props
|
||||
[{:keys [source name props] :as params}]
|
||||
(cond
|
||||
(and (= source "backend")
|
||||
(or (= name "login-with-oidc")
|
||||
(= name "login-with-password")
|
||||
(= name "register-profile")
|
||||
(= name "update-profile")))
|
||||
(let [props (select-keys props profile-props)
|
||||
props (into {} xf:filter-telemetry-props props)
|
||||
props (-> props
|
||||
(assoc :lang (:lang props))
|
||||
(assoc :auth-backend (:auth-backend props))
|
||||
(assoc :email-domain (email/get-domain (:email props)))
|
||||
(d/without-nils))]
|
||||
(assoc params :props props))
|
||||
|
||||
;; FIXME: add frontend identify
|
||||
|
||||
:else
|
||||
(let [props (into {} xf:filter-telemetry-props props)]
|
||||
(assoc params :props props))))
|
||||
|
||||
(defn filter-telemetry-context
|
||||
[{:keys [source context] :as params}]
|
||||
(let [context (case source
|
||||
"backend" (select-keys context safe-backend-context-keys)
|
||||
"frontend" (select-keys context safe-frontend-context-keys)
|
||||
{})]
|
||||
(assoc params :context context)))
|
||||
|
||||
(defn submit
|
||||
"Submit an event to be registered under audit-log subsystem"
|
||||
[cfg event]
|
||||
(let [tnow (ct/now)
|
||||
event (-> event
|
||||
(assoc :created-at tnow)
|
||||
(update :tracked-at or tnow)
|
||||
(update :ip-addr or "0.0.0.0")
|
||||
(update :tracked-at d/nilv tnow)
|
||||
(update :ip-addr d/nilv "0.0.0.0")
|
||||
(assoc :source "backend")
|
||||
(d/without-nils))]
|
||||
(submit* cfg event)))
|
||||
@ -419,9 +416,9 @@
|
||||
(let [tnow (ct/now)
|
||||
event (-> event
|
||||
(assoc :created-at tnow)
|
||||
(update :tracked-at or tnow)
|
||||
(update :profile-id or uuid/zero)
|
||||
(update :tracked-at d/nilv tnow)
|
||||
(update :profile-id d/nilv uuid/zero)
|
||||
(assoc :source "backend")
|
||||
(select-keys event-keys)
|
||||
(check-event))]
|
||||
(db/run! cfg append-audit-entry params))))
|
||||
(db/run! cfg append-audit-entry event))))
|
||||
|
||||
@ -145,7 +145,7 @@
|
||||
(def ^:private sql:get-counters
|
||||
"SELECT name, count(*) AS count
|
||||
FROM audit_log
|
||||
WHERE source IN ('backend', 'frontend', 'telemetry:backend', 'telemetry:frontend')
|
||||
WHERE source IN ('backend', 'frontend')
|
||||
AND created_at >= date_trunc('day', now())
|
||||
AND created_at < date_trunc('day', now()) + interval '1 day'
|
||||
GROUP BY 1
|
||||
@ -218,7 +218,7 @@
|
||||
|
||||
(def ^:private sql:gc-events
|
||||
"DELETE FROM audit_log
|
||||
WHERE source IN ('telemetry:backend', 'telemetry:frontend')
|
||||
WHERE source IN ('backend', 'frontend')
|
||||
AND created_at < now() - interval '7 days'")
|
||||
|
||||
(defn- gc-events
|
||||
@ -233,7 +233,7 @@
|
||||
(def ^:private sql:fetch-telemetry-events
|
||||
"SELECT id, name, type, source, tracked_at, profile_id, context
|
||||
FROM audit_log
|
||||
WHERE source IN ('telemetry:backend', 'telemetry:frontend')
|
||||
WHERE source IN ('backend', 'frontend')
|
||||
ORDER BY created_at ASC
|
||||
LIMIT ?")
|
||||
|
||||
|
||||
@ -83,7 +83,7 @@
|
||||
[next]
|
||||
(with-redefs [app.config/flags (flags/parse flags/default default-flags)
|
||||
app.config/config config
|
||||
app.loggers.audit/submit! (constantly nil)
|
||||
app.loggers.audit/submit (constantly nil)
|
||||
app.auth/derive-password identity
|
||||
app.auth/verify-password (fn [a b] {:valid (= a b)})
|
||||
app.common.features/get-enabled-features (fn [& _] app.common.features/supported-features)]
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user