github/penpot
1
0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2026-05-22 16:33:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

🚧 WIP

Browse Source
This commit is contained in:
Andrey Antukh 2026-05-09 11:00:24 +02:00
parent 32627efb96
commit f63e23599a
3 changed files with 97 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

189
backend/src/app/loggers/audit.clj
View File

@ -193,7 +193,6 @@
:type :type
:profile-id :profile-id
:ip-addr :ip-addr
:context
:props :props
:context :context
:source :source
@ -215,105 +214,72 @@
(and (simple-keyword? k) (and (simple-keyword? k)
(or (uuid? v) (boolean? v) (number? v)))))) (or (uuid? v) (boolean? v) (number? v))))))
(defn- filter-telemetry-props (declare filter-telemetry-props)
[{:keys [source name props] :as params}] (declare filter-telemetry-context)
(cond
(and (= source "backend")
(or (= name "login-with-oidc")
(= name "login-with-password")
(= name "register-profile")
(= name "update-profile")))
(let [props (select-keys props profile-props)
props (into {} xf:filter-telemetry-props props)
props (-> props
(assoc :lang (:lang props))
(assoc :auth-backend (:auth-backend props))
(assoc :email-domain (email/get-domain (:email props)))
(d/without-nils))]
(assoc params :props props))
:else
(let [props (into {} xf:filter-telemetry-props props)]
(assoc params :props props))))
(defn filter-telemetry-context
[{:keys [source context] :as params}]
(let [context (case source
"backend" (select-keys context safe-backend-context-keys)
"frontend" (select-keys context safe-frontend-context-keys)
{})]
(assoc params :context context)))
(defn- process-event (defn- process-event
[cfg event] [cfg event]
(let [params (normalize-event event)] (when (contains? cf/flags :audit-log-logger)
(l/log! ::l/logger "app.audit"
::l/level :info
:profile-id (str (:profile-id event))
:ip-addr (str (:ip-addr event))
:type (:type event)
:name (:name event)
:props (json/encode (:props event) :key-fn json/write-camel-key)
:context (json/encode (:context event) :key-fn json/write-camel-key)))
(when (contains? cf/flags :audit-log-logger) (if (contains? cf/flags :audit-log)
(l/log! ::l/logger "app.audit" ;; NOTE: this operation may cause primary key conflicts on inserts
::l/level :info ;; because of the timestamp precission (two concurrent requests), in
:profile-id (str (:profile-id event)) ;; this case we just retry the operation.
:ip-addr (str (:ip-addr event)) (append-audit-entry cfg event)
:type (:type event) (when cf/telemetry-enabled?
:name (:name event)
:props (json/encode (:props event) :key-fn json/write-camel-key)
:context (json/encode (:context event) :key-fn json/write-camel-key)))
(if (contains? cf/flags :audit-log)
;; NOTE: this operation may cause primary key conflicts on inserts ;; NOTE: this operation may cause primary key conflicts on inserts
;; because of the timestamp precission (two concurrent requests), in ;; because of the timestamp precission (two concurrent requests), in
;; this case we just retry the operation. ;; this case we just retry the operation.
(append-audit-entry cfg event) ;;
(when cf/telemetry-enabled? ;; NOTE: this is only executed when general audit log is disabled;
;; NOTE: this operation may cause primary key conflicts on inserts ;; events are stored stripped of props and ip-addr, tagged with
;; because of the timestamp precission (two concurrent requests), in ;; source="telemetry" so the telemetry task can collect and ship
;; this case we just retry the operation. ;; them. The profile-id is preserved (UUIDs are already anonymous
;; ;; random identifiers). Only a safe subset of context fields is
;; NOTE: this is only executed when general audit log is disabled; ;; kept: initiator, version, client-version and client-user-agent.
;; events are stored stripped of props and ip-addr, tagged with ;; Timestamps are truncated to day precision to avoid leaking exact
;; source="telemetry" so the telemetry task can collect and ship ;; event timing.
;; them. The profile-id is preserved (UUIDs are already anonymous (let [event-name (get event :name)
;; random identifiers). Only a safe subset of context fields is event (-> event
;; kept: initiator, version, client-version and client-user-agent. (filter-telemetry-props)
;; Timestamps are truncated to day precision to avoid leaking exact (filter-telemetry-context)
;; event timing. (update :created-at ct/truncate :days)
(let [event-name (get event :name) (update :tracked-at ct/truncate :days)
tday (ct/truncate tnow :days) (assoc :ip-addr "0.0.0.0"))]
event (-> event (append-audit-entry cfg event))))
(filter-telemetry-props)
(filter-telemetry-context)
(update :created-at ct/truncate :days)
(update :tracked-at ct/truncate :days)
(assoc :ip-addr "0.0.0.0"))]
(append-audit-entry cfg params))))
(when (and (contains? cf/flags :webhooks) (when (and (contains? cf/flags :webhooks)
(::webhooks/event? event)) (::webhooks/event? event))
(let [batch-key (::webhooks/batch-key event) (let [batch-key (::webhooks/batch-key event)
batch-timeout (::webhooks/batch-timeout event) batch-timeout (::webhooks/batch-timeout event)
label (dm/str "rpc:" (:name params)) label (dm/str "rpc:" (:name event))
label (cond label (cond
(ifn? batch-key) (dm/str label ":" (batch-key (::rpc/params event))) (ifn? batch-key) (dm/str label ":" (batch-key (::rpc/params event)))
(string? batch-key) (dm/str label ":" batch-key) (string? batch-key) (dm/str label ":" batch-key)
:else label) :else label)
dedupe? (boolean (and batch-key batch-timeout))] dedupe? (boolean (and batch-key batch-timeout))]
(wrk/submit! (-> cfg (wrk/submit! (-> cfg
(assoc ::wrk/task :process-webhook-event) (assoc ::wrk/task :process-webhook-event)
(assoc ::wrk/queue :webhooks) (assoc ::wrk/queue :webhooks)
(assoc ::wrk/max-retries 0) (assoc ::wrk/max-retries 0)
(assoc ::wrk/delay (or batch-timeout 0)) (assoc ::wrk/delay (or batch-timeout 0))
(assoc ::wrk/dedupe dedupe?) (assoc ::wrk/dedupe dedupe?)
(assoc ::wrk/label label) (assoc ::wrk/label label)
(assoc ::wrk/params (-> params (assoc ::wrk/params (-> event
(dissoc :source) (dissoc :source)
(dissoc :context) (dissoc :context)
(dissoc :ip-addr) (dissoc :ip-addr)
(dissoc :type))))))) (dissoc :type)))))))
params)) event)
(defn- or-ts-now
[o]
(if (inst? o) o (ct/now)))
(defn submit* (defn submit*
"A public API, lower-leve lhan submit, assumes all required fields are filled" "A public API, lower-leve lhan submit, assumes all required fields are filled"
@ -398,14 +364,45 @@
(some? context) (some? context)
(assoc :context context)))) (assoc :context context))))
(defn filter-telemetry-props
[{:keys [source name props] :as params}]
(cond
(and (= source "backend")
(or (= name "login-with-oidc")
(= name "login-with-password")
(= name "register-profile")
(= name "update-profile")))
(let [props (select-keys props profile-props)
props (into {} xf:filter-telemetry-props props)
props (-> props
(assoc :lang (:lang props))
(assoc :auth-backend (:auth-backend props))
(assoc :email-domain (email/get-domain (:email props)))
(d/without-nils))]
(assoc params :props props))
;; FIXME: add frontend identify
:else
(let [props (into {} xf:filter-telemetry-props props)]
(assoc params :props props))))
(defn filter-telemetry-context
[{:keys [source context] :as params}]
(let [context (case source
"backend" (select-keys context safe-backend-context-keys)
"frontend" (select-keys context safe-frontend-context-keys)
{})]
(assoc params :context context)))
(defn submit (defn submit
"Submit an event to be registered under audit-log subsystem" "Submit an event to be registered under audit-log subsystem"
[cfg event] [cfg event]
(let [tnow (ct/now) (let [tnow (ct/now)
event (-> event event (-> event
(assoc :created-at tnow) (assoc :created-at tnow)
(update :tracked-at or tnow) (update :tracked-at d/nilv tnow)
(update :ip-addr or "0.0.0.0") (update :ip-addr d/nilv "0.0.0.0")
(assoc :source "backend") (assoc :source "backend")
(d/without-nils))] (d/without-nils))]
(submit* cfg event))) (submit* cfg event)))
@ -419,9 +416,9 @@
(let [tnow (ct/now) (let [tnow (ct/now)
event (-> event event (-> event
(assoc :created-at tnow) (assoc :created-at tnow)
(update :tracked-at or tnow) (update :tracked-at d/nilv tnow)
(update :profile-id or uuid/zero) (update :profile-id d/nilv uuid/zero)
(assoc :source "backend") (assoc :source "backend")
(select-keys event-keys) (select-keys event-keys)
(check-event))] (check-event))]
(db/run! cfg append-audit-entry params)))) (db/run! cfg append-audit-entry event))))

6
backend/src/app/tasks/telemetry.clj
View File

@ -145,7 +145,7 @@
(def ^:private sql:get-counters (def ^:private sql:get-counters
"SELECT name, count(*) AS count "SELECT name, count(*) AS count
FROM audit_log FROM audit_log
WHERE source IN ('backend', 'frontend', 'telemetry:backend', 'telemetry:frontend') WHERE source IN ('backend', 'frontend')
AND created_at >= date_trunc('day', now()) AND created_at >= date_trunc('day', now())
AND created_at < date_trunc('day', now()) + interval '1 day' AND created_at < date_trunc('day', now()) + interval '1 day'
GROUP BY 1 GROUP BY 1
@ -218,7 +218,7 @@
(def ^:private sql:gc-events (def ^:private sql:gc-events
"DELETE FROM audit_log "DELETE FROM audit_log
WHERE source IN ('telemetry:backend', 'telemetry:frontend') WHERE source IN ('backend', 'frontend')
AND created_at < now() - interval '7 days'") AND created_at < now() - interval '7 days'")
(defn- gc-events (defn- gc-events
@ -233,7 +233,7 @@
(def ^:private sql:fetch-telemetry-events (def ^:private sql:fetch-telemetry-events
"SELECT id, name, type, source, tracked_at, profile_id, context "SELECT id, name, type, source, tracked_at, profile_id, context
FROM audit_log FROM audit_log
WHERE source IN ('telemetry:backend', 'telemetry:frontend') WHERE source IN ('backend', 'frontend')
ORDER BY created_at ASC ORDER BY created_at ASC
LIMIT ?") LIMIT ?")

2
backend/test/backend_tests/helpers.clj
View File

@ -83,7 +83,7 @@
[next] [next]
(with-redefs [app.config/flags (flags/parse flags/default default-flags) (with-redefs [app.config/flags (flags/parse flags/default default-flags)
app.config/config config app.config/config config
app.loggers.audit/submit! (constantly nil) app.loggers.audit/submit (constantly nil)
app.auth/derive-password identity app.auth/derive-password identity
app.auth/verify-password (fn [a b] {:valid (= a b)}) app.auth/verify-password (fn [a b] {:valid (= a b)})
app.common.features/get-enabled-features (fn [& _] app.common.features/supported-features)] app.common.features/get-enabled-features (fn [& _] app.common.features/supported-features)]