github/penpot
1
0
Fork 0
mirror of https://github.com/penpot/penpot.git synced 2026-05-11 02:58:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

🚧 WIP

Browse Source
This commit is contained in:
Andrey Antukh 2026-05-09 12:57:34 +02:00
parent 42b0b39a31
commit 70f60ef3d1
10 changed files with 695 additions and 170 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
backend/src/app/config.clj
View File

@ -336,12 +336,6 @@
(or (c/get config :file-clean-delay)
(ct/duration {:days 2})))
(def telemetry-enabled?
"True when telemetry is active, either via the :telemetry feature
flag or the legacy :telemetry-enabled config key."
(or (contains? flags :telemetry)
(c/get config :telemetry-enabled)))
(defn get
"A configuration getter. Helps code be more testable."
([key]

179
backend/src/app/loggers/audit.clj
View File

@ -40,7 +40,7 @@
:client-version
:client-user-agent})
(def ^:privarte safe-frontend-context-keys
(def ^:private safe-frontend-context-keys
#{:version
:locale
:browser
@ -70,6 +70,18 @@
:fullname
:lang])
(def ^:private event-keys
#{:id
:name
:type
:profile-id
:ip-addr
:props
:context
:source
:tracked-at
:created-at})
(def reserved-props
#{:session-id
:password
@ -147,6 +159,7 @@
(def ^:private schema:event
[:map {:title "AuditEvent"}
[:id ::sm/uuid]
[:type ::sm/text]
[:name ::sm/text]
[:profile-id ::sm/uuid]
@ -177,27 +190,14 @@
key-id (::http/auth-key-id request)
token-id (::actoken/id request)
token-type (::actoken/type request)]
(d/without-nils
{:external-session-id session-id
:initiator (or key-id "app")
:access-token-id (some-> token-id str)
:access-token-type (some-> token-type str)
:client-event-origin client-event-origin
:client-user-agent client-user-agent
:client-version client-version
:version (:full cf/version)})))
(def ^:private event-keys
#{:id
:name
:type
:profile-id
:ip-addr
:props
:context
:source
:tracked-at
:created-at})
{:external-session-id session-id
:initiator (or key-id "app")
:access-token-id (some-> token-id str)
:access-token-type (some-> token-type str)
:client-event-origin client-event-origin
:client-user-agent client-user-agent
:client-version client-version
:version (:full cf/version)}))
(defn- append-audit-entry
[cfg params]
@ -230,23 +230,15 @@
:context (json/encode (:context event) :key-fn json/write-camel-key)))
(if (contains? cf/flags :audit-log)
;; NOTE: this operation may cause primary key conflicts on inserts
;; because of the timestamp precission (two concurrent requests), in
;; this case we just retry the operation.
(append-audit-entry cfg event)
(when cf/telemetry-enabled?
;; NOTE: this operation may cause primary key conflicts on inserts
;; because of the timestamp precission (two concurrent requests), in
;; this case we just retry the operation.
;;
;; NOTE: this is only executed when general audit log is disabled;
;; events are stored stripped of props and ip-addr, tagged with
;; source="telemetry" so the telemetry task can collect and ship
;; them. The profile-id is preserved (UUIDs are already anonymous
;; random identifiers). Only a safe subset of context fields is
;; kept: initiator, version, client-version and client-user-agent.
;; Timestamps are truncated to day precision to avoid leaking exact
;; event timing.
(when (contains? cf/flags :telemetry)
;; NOTE: this is only executed when general audit log is disabled; events
;; are stored stripped of props and ip-addr, tagged with
;; source="telemetry" so the telemetry task can collect and ship them.
;; The profile-id is preserved (UUIDs are already anonymous random
;; identifiers). Only a safe subset of context fields is kept: initiator,
;; version, client-version and client-user-agent. Timestamps are
;; truncated to day precision to avoid leaking exact event timing.
(let [event-name (get event :name)
event (-> event
(filter-telemetry-props)
@ -286,12 +278,8 @@
"A public API, lower-leve lhan submit, assumes all required fields are filled"
[cfg event]
(try
(let [event (check-event event)
cfg (-> cfg
(assoc ::rtry/when rtry/conflict-exception?)
(assoc ::rtry/max-retries 6)
(assoc ::rtry/label "persist-audit-log"))]
(rtry/invoke! cfg db/tx-run! process-event event))
(let [event (check-event event)]
(db/tx-run! cfg process-event event))
(catch Throwable cause
(l/error :hint "unexpected error processing event" :cause cause))))
@ -299,6 +287,43 @@
;; PUBLIC API
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defn filter-telemetry-props
[{:keys [source name props type] :as params}]
(cond
(or (and (= source "frontend")
(= type "identify"))
(and (= source "backend")
(or (= name "login-with-oidc")
(= name "login-with-password")
(= name "register-profile")
(= name "update-profile"))))
(let [props' (select-keys props profile-props)
props' (into {} xf:filter-telemetry-props props)
props' (-> props'
(assoc :lang (:lang props))
(assoc :auth-backend (:auth-backend props))
(assoc :email-domain (email/get-domain (:email props)))
(d/without-nils))]
(assoc params :props props'))
(and (= source "backend")
(= type "trigger")
(= name "instance-start"))
params
:else
(let [props (into {} xf:filter-telemetry-props props)]
(assoc params :props props))))
(defn filter-telemetry-context
[{:keys [source context] :as params}]
(let [context (case source
"backend" (select-keys context safe-backend-context-keys)
"frontend" (select-keys context safe-frontend-context-keys)
{})]
(assoc params :context context)))
(defn prepare-rpc-event
[cfg mdata params result]
(let [resultm (meta result)
@ -312,12 +337,16 @@
(merge params (::props resultm)))
(clean-props))
context (merge (::context resultm)
(prepare-context-from-request request))
context (-> (::context resultm)
(merge (prepare-context-from-request request))
(assoc :request-id (::rpc/request-id params))
(d/without-nils))
ip-addr (inet/parse-request request)
module (get cfg ::rpc/module)]
{:type (or (::type resultm)
{:id (uuid/next)
:type (or (::type resultm)
(::rpc/type cfg))
:name (or (::name resultm)
(let [sname (::sv/name mdata)]
@ -355,55 +384,28 @@
"Create a base event skeleton with pre-filled some important
data that can be extracted from RPC params object"
[params]
(let [context (some-> params meta ::http/request prepare-context-from-request)
event {:type "action"
:profile-id (::rpc/profile-id params)
:created-at (::rpc/request-at params)
:tracked-at (::rpc/request-at params)
:ip-addr (::rpc/ip-addr params)}]
(cond-> event
(some? context)
(assoc :context context))))
(defn filter-telemetry-props
[{:keys [source name props] :as params}]
(cond
(and (= source "telemetry:backend")
(or (= name "login-with-oidc")
(= name "login-with-password")
(= name "register-profile")
(= name "update-profile")))
(let [props (select-keys props profile-props)
props (into {} xf:filter-telemetry-props props)
props (-> props
(assoc :lang (:lang props))
(assoc :auth-backend (:auth-backend props))
(assoc :email-domain (email/get-domain (:email props)))
(d/without-nils))]
(assoc params :props props))
;; FIXME: add frontend identify
:else
(let [props (into {} xf:filter-telemetry-props props)]
(assoc params :props props))))
(defn filter-telemetry-context
[{:keys [source context] :as params}]
(let [context (case source
"backend" (select-keys context safe-backend-context-keys)
"frontend" (select-keys context safe-frontend-context-keys)
{})]
(assoc params :context context)))
(let [context (some-> params meta ::http/request prepare-context-from-request)
context (assoc context :request-id (::rpc/request-id params))
request-at (::rpc/request-at params)]
{:type "action"
:profile-id (::rpc/profile-id params)
:created-at request-at
:tracked-at request-at
:ip-addr (::rpc/ip-addr params)
:context (d/without-nils context)}))
(defn submit
"Submit an event to be registered under audit-log subsystem"
[cfg event]
(let [tnow (ct/now)
event (-> event
(update :id (fn [id] (or id (uuid/next))))
(assoc :created-at tnow)
(update :profile-id d/nilv uuid/zero)
(update :tracked-at d/nilv tnow)
(update :ip-addr d/nilv "0.0.0.0")
(update :props d/nilv {})
(update :context d/nilv {})
(assoc :source "backend")
(d/without-nils))]
(submit* cfg event)))
@ -416,9 +418,12 @@
(when (contains? cf/flags :audit-log)
(let [tnow (ct/now)
event (-> event
(update :id (fn [id] (or id (uuid/next))))
(assoc :created-at tnow)
(update :tracked-at d/nilv tnow)
(update :profile-id d/nilv uuid/zero)
(update :props d/nilv {})
(update :context d/nilv {})
(assoc :source "backend")
(select-keys event-keys)
(check-event))]

1
backend/src/app/rpc.clj
View File

@ -105,6 +105,7 @@
(assoc ::handler-name handler-name)
(assoc ::ip-addr ip-addr)
(assoc ::request-at (ct/now))
(assoc ::request-id (uuid/next))
(assoc ::session-id (some-> session-id uuid/parse*))
(assoc ::cond/key etag)
(cond-> (uuid? profile-id)

15
backend/src/app/rpc/commands/audit.clj
View File

@ -23,7 +23,8 @@
[app.rpc.doc :as-alias doc]
[app.rpc.helpers :as rph]
[app.util.inet :as inet]
[app.util.services :as sv]))
[app.util.services :as sv]
[clojure.set :as set]))
(def ^:private event-columns
[:id
@ -97,10 +98,10 @@
(-> event
(update :created-at ct/truncate :days)
(update :tracked-at ct/truncate :days)
(assoc :source "telemetry:frontend")
(assoc :ip-addr "0.0.0.0")
(audit/filter-telemetry-props)
(audit/filter-telemetry-context))))
(audit/filter-telemetry-context)
(assoc :ip-addr "0.0.0.0")
(assoc :source "telemetry:frontend"))))
(map event->row)))
(defn- handle-events
@ -122,7 +123,7 @@
;; When telemetry is enabled (and audit-log is NOT), store anonymized
;; frontend events so the telemetry task can ship them in batches.
(when cf/telemetry-enabled?
(when (contains? cf/flags :telemetry)
(when-let [rows (-> (sequence xf:map-telemetry-event-row events)
(not-empty))]
(db/insert-many! pool :audit-log event-columns rows))))))
@ -158,7 +159,7 @@
::doc/skip true
::doc/added "1.17"}
[{:keys [::db/pool] :as cfg} params]
(let [telemetry? cf/telemetry-enabled?
(let [telemetry? (contains? cf/flags :telemetry)
audit-log? (contains? cf/flags :audit-log)
enabled? (and (not (db/read-only? pool))
(or audit-log? telemetry?))]
@ -185,4 +186,4 @@
::doc/skip true
::doc/added "1.20"}
[_cfg _params]
cf/flags)
(set/intersection cf/flags #{:audit-log :telemetry}))

33
backend/src/app/setup.clj
View File

@ -11,7 +11,9 @@
[app.common.logging :as l]
[app.common.schema :as sm]
[app.common.uuid :as uuid]
[app.config :as cf]
[app.db :as db]
[app.loggers.audit :as audit]
[app.main :as-alias main]
[app.setup.keys :as keys]
[app.setup.templates]
@ -35,22 +37,20 @@
(into {})))
(defn- handle-instance-id
[instance-id conn read-only?]
[instance-id conn]
(or instance-id
(let [instance-id (uuid/random)]
(when-not read-only?
(try
(db/insert! conn :server-prop
{:id "instance-id"
:preload true
:content (db/tjson instance-id)})
(catch Throwable cause
(l/warn :hint "unable to persist instance-id"
:instance-id instance-id
:cause cause))))
(try
(db/insert! conn :server-prop
{:id "instance-id"
:preload true
:content (db/tjson instance-id)})
(catch Throwable cause
(l/warn :hint "unable to persist instance-id"
:instance-id instance-id
:cause cause)))
instance-id)))
(def sql:add-prop
"INSERT INTO server_prop (id, content, preload)
VALUES (?, ?, ?)
@ -77,7 +77,12 @@
(assert (db/pool? (::db/pool params)) "expected valid database pool"))
(defmethod ig/init-key ::props
[_ {:keys [::db/pool ::key] :as cfg}]
[_ {:keys [::key] :as cfg}]
(audit/submit cfg {:type "trigger"
:name "instance-start"
:props {:version (:full cf/version)
:flags (mapv name cf/flags)
:public-uri (str (cf/get :public-uri))}})
(db/tx-run! cfg (fn [{:keys [::db/conn]}]
(db/xact-lock! conn 0)
@ -91,7 +96,7 @@
(-> (get-all-props conn)
(assoc :secret-key secret)
(assoc :tokens-key (keys/derive secret :salt "tokens"))
(update :instance-id handle-instance-id conn (db/read-only? pool)))))))
(update :instance-id handle-instance-id conn))))))
(defmethod ig/init-key ::shared-keys
[_ {:keys [::props] :as cfg}]

8
backend/src/app/srepl/main.clj
View File

@ -595,7 +595,7 @@
(audit/insert main/system
{:name "delete-project"
:type "action"
:type "action"
:props {:id project-id}
:context {:triggered-by "srepl"
:cause "explicit call to delete-project!"}
@ -650,8 +650,8 @@
:type "action"
:props {:id team-id}
:context {:triggered-by "srepl"
:cause "explicit call to delete-profile!"}
:tracked-at tnow})
:cause "explicit call to delete-profile!"}
:tracked-at tnow})
(wrk/invoke! (-> main/system
(assoc ::wrk/task :delete-object)
@ -706,7 +706,7 @@
{:name "delete-profile"
:type "action"
:context {:triggered-by "srepl"
:cause "explicit call to delete-profile!"}
:cause "explicit call to delete-profile!"}
:tracked-at tnow})
(wrk/invoke! (-> main/system

11
backend/src/app/tasks/telemetry.clj
View File

@ -165,7 +165,7 @@
"taiga"
(cf/get :telemetry-referer))]
(-> {:referer referer
:public-uri (cf/get :public-uri)
:public-uri (str (cf/get :public-uri))
:total-teams (get-num-teams conn)
:total-projects (get-num-projects conn)
:total-files (get-num-files conn)
@ -231,21 +231,22 @@
:count (:next.jdbc/update-count result)))))
(def ^:private sql:fetch-telemetry-events
"SELECT id, name, type, source, tracked_at, profile_id, context
"SELECT id, name, type, source, tracked_at, profile_id, props, context
FROM audit_log
WHERE source LIKE 'telemetry:%'
ORDER BY created_at ASC
LIMIT ?")
(defn- row->event
[{:keys [name type source tracked-at profile-id context]}]
[{:keys [name type source tracked-at profile-id props context]}]
(d/without-nils
{:name name
:type type
:source source
:tracked-at tracked-at
:profile-id profile-id
:context (not-empty (db/decode-transit-pgobject context))}))
:props (or (some-> props db/decode-transit-pgobject) {})
:context (or (some-> context db/decode-transit-pgobject) {})}))
(defn- encode-batch
"Encode a sequence of event maps into a fressian+zstd base64 string
@ -315,7 +316,7 @@
(let [params (:props task)
send? (get params :send? true)
enabled? (or (get params :enabled? false)
cf/telemetry-enabled?)
(contains? cf/flags :telemetry))
subs (get-subscriptions cfg)]

221
backend/test/backend_tests/rpc_audit_test.clj
View File

@ -11,6 +11,7 @@
[app.common.uuid :as uuid]
[app.config :as cf]
[app.db :as db]
[app.loggers.audit :as audit]
[app.rpc :as-alias rpc]
[backend-tests.helpers :as th]
[clojure.test :as t]
@ -105,8 +106,7 @@
;; When telemetry is enabled and audit-log is NOT, frontend events
;; must be stored with source="telemetry", empty props, zeroed ip,
;; and context filtered to safe keys only.
(with-redefs [cf/flags #{:telemetry}
cf/telemetry-enabled? true]
(with-redefs [cf/flags #{:telemetry}]
(let [prof (th/create-profile* 1 {:is-active true})
team-id (:default-team-id prof)
proj-id (:default-project-id prof)
@ -164,8 +164,7 @@
(t/deftest push-events-audit-log-flag-prevents-telemetry-rows
;; When the :audit-log flag is active, telemetry-mode storage must
;; NOT happen — events go to full audit log instead.
(with-redefs [cf/flags #{:audit-log :telemetry}
cf/telemetry-enabled? true]
(with-redefs [cf/flags #{:audit-log :telemetry}]
(let [prof (th/create-profile* 1 {:is-active true})
params {::th/type :push-audit-events
::rpc/profile-id (:id prof)
@ -193,8 +192,7 @@
(t/deftest push-events-disabled-when-no-flags-and-no-telemetry
;; When neither audit-log nor telemetry is enabled, no rows should
;; be stored.
(with-redefs [cf/flags #{}
cf/telemetry-enabled? false]
(with-redefs [cf/flags #{}]
(let [prof (th/create-profile* 1 {:is-active true})
params {::th/type :push-audit-events
::rpc/profile-id (:id prof)
@ -208,3 +206,214 @@
(t/is (nil? (:error out)))
(t/is (= 0 (count (th/db-exec! ["select * from audit_log"])))))))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; PURE HELPER UNIT TESTS
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(t/deftest extract-utm-params-utm
;; UTM params are namespaced under :penpot
(let [result (audit/extract-utm-params {:utm_source "google"
:utm_medium "cpc"
:utm_campaign "spring"
:other "ignored"})]
(t/is (= "google" (:penpot/utm-source result)))
(t/is (= "cpc" (:penpot/utm-medium result)))
(t/is (= "spring" (:penpot/utm-campaign result)))
(t/is (not (contains? result :other)))))
(t/deftest extract-utm-params-mtm
;; MTM params are also namespaced under :penpot
(let [result (audit/extract-utm-params {:mtm_source "newsletter"
:mtm_medium "email"})]
(t/is (= "newsletter" (:penpot/mtm-source result)))
(t/is (= "email" (:penpot/mtm-medium result)))))
(t/deftest extract-utm-params-empty
(t/is (= {} (audit/extract-utm-params {})))
(t/is (= {} (audit/extract-utm-params {:foo "bar" :baz 42}))))
(t/deftest profile->props-selects-and-merges
;; Selects profile-props keys and merges with (:props profile)
(let [profile {:id (uuid/next)
:fullname "John"
:email "john@example.com"
:is-active true
:lang "en"
:deleted-field "gone"
:props {:custom-key "custom-val"
:newsletter-updates true}}
result (audit/profile->props profile)]
;; Selected keys from profile
(t/is (= "John" (:fullname result)))
(t/is (= "john@example.com" (:email result)))
(t/is (true? (:is-active result)))
(t/is (= "en" (:lang result)))
;; Merged from (:props profile)
(t/is (= "custom-val" (:custom-key result)))
(t/is (true? (:newsletter-updates result)))
;; Keys not in profile-props are excluded
(t/is (not (contains? result :deleted-field)))))
(t/deftest profile->props-removes-nils
(let [profile {:id (uuid/next) :fullname nil :email "a@b.com"}
result (audit/profile->props profile)]
(t/is (not (contains? result :fullname)))
(t/is (= "a@b.com" (:email result)))))
(t/deftest clean-props-removes-reserved
;; Reserved props (:session-id, :password, :old-password, :token) are stripped
(let [props {:name "test"
:session-id "sess-123"
:password "secret"
:old-password "old-secret"
:token "tok-456"
:valid-key "kept"}
result (audit/clean-props props)]
(t/is (= "test" (:name result)))
(t/is (= "kept" (:valid-key result)))
(t/is (not (contains? result :session-id)))
(t/is (not (contains? result :password)))
(t/is (not (contains? result :old-password)))
(t/is (not (contains? result :token)))))
(t/deftest clean-props-removes-qualified-keys
;; Qualified keywords (namespaced) are stripped
(let [props {:simple "kept"
::namespaced "stripped"
:app.rpc/also-stripped true}
result (audit/clean-props props)]
(t/is (= "kept" (:simple result)))
(t/is (not (contains? result ::namespaced)))
(t/is (not (contains? result :app.rpc/also-stripped)))))
(t/deftest clean-props-removes-nils
(let [props {:a nil :b "val" :c nil}
result (audit/clean-props props)]
(t/is (= "val" (:b result)))
(t/is (not (contains? result :a)))
(t/is (not (contains? result :c)))))
(t/deftest get-external-session-id-valid
(let [request (reify yetti.request/IRequest
(get-header [_ name]
(case name "x-external-session-id" "abc-123")))]
(t/is (= "abc-123" (audit/get-external-session-id request)))))
(t/deftest get-external-session-id-nil-when-missing
(let [request (reify yetti.request/IRequest
(get-header [_ _] nil))]
(t/is (nil? (audit/get-external-session-id request)))))
(t/deftest get-external-session-id-nil-when-null-string
(let [request (reify yetti.request/IRequest
(get-header [_ name]
(case name "x-external-session-id" "null")))]
(t/is (nil? (audit/get-external-session-id request)))))
(t/deftest get-external-session-id-nil-when-blank
(let [request (reify yetti.request/IRequest
(get-header [_ name]
(case name "x-external-session-id" " ")))]
(t/is (nil? (audit/get-external-session-id request)))))
(t/deftest get-external-session-id-nil-when-too-long
(let [long-id (apply str (repeat 300 "x"))
request (reify yetti.request/IRequest
(get-header [_ name]
(case name "x-external-session-id" long-id)))]
(t/is (nil? (audit/get-external-session-id request)))))
(t/deftest get-client-user-agent-valid
(let [request (reify yetti.request/IRequest
(get-header [_ name]
(case name "user-agent" "Mozilla/5.0 (Test)")))]
(t/is (= "Mozilla/5.0 (Test)" (audit/get-client-user-agent request)))))
(t/deftest get-client-user-agent-nil-when-missing
(let [request (reify yetti.request/IRequest
(get-header [_ _] nil))]
(t/is (nil? (audit/get-client-user-agent request)))))
(t/deftest get-client-user-agent-truncates-long
(let [long-ua (apply str (repeat 600 "x"))
request (reify yetti.request/IRequest
(get-header [_ name]
(case name "user-agent" long-ua)))]
(t/is (<= (count (audit/get-client-user-agent request)) 500))))
(t/deftest get-client-event-origin-valid
(let [get-client-event-origin (ns-resolve 'app.loggers.audit 'get-client-event-origin)
request (reify yetti.request/IRequest
(get-header [_ name]
(case name "x-event-origin" "workspace")))]
(t/is (= "workspace" (get-client-event-origin request)))))
(t/deftest get-client-event-origin-nil-when-null
(let [get-client-event-origin (ns-resolve 'app.loggers.audit 'get-client-event-origin)
request (reify yetti.request/IRequest
(get-header [_ name]
(case name "x-event-origin" "null")))]
(t/is (nil? (get-client-event-origin request)))))
(t/deftest get-client-event-origin-nil-when-blank
(let [get-client-event-origin (ns-resolve 'app.loggers.audit 'get-client-event-origin)
request (reify yetti.request/IRequest
(get-header [_ name]
(case name "x-event-origin" " ")))]
(t/is (nil? (get-client-event-origin request)))))
(t/deftest get-client-event-origin-truncates-long
(let [get-client-event-origin (ns-resolve 'app.loggers.audit 'get-client-event-origin)
long-origin (apply str (repeat 300 "a"))
request (reify yetti.request/IRequest
(get-header [_ name]
(case name "x-event-origin" long-origin)))]
(t/is (<= (count (get-client-event-origin request)) 200))))
(t/deftest get-client-version-valid
(let [get-client-version (ns-resolve 'app.loggers.audit 'get-client-version)
request (reify yetti.request/IRequest
(get-header [_ name]
(case name "x-frontend-version" "2.0.0")))]
(t/is (= "2.0.0" (get-client-version request)))))
(t/deftest get-client-version-nil-when-null
(let [get-client-version (ns-resolve 'app.loggers.audit 'get-client-version)
request (reify yetti.request/IRequest
(get-header [_ name]
(case name "x-frontend-version" "null")))]
(t/is (nil? (get-client-version request)))))
(t/deftest get-client-version-nil-when-blank
(let [get-client-version (ns-resolve 'app.loggers.audit 'get-client-version)
request (reify yetti.request/IRequest
(get-header [_ name]
(case name "x-frontend-version" " ")))]
(t/is (nil? (get-client-version request)))))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; INSERT DEFAULTS
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(t/deftest insert-only-runs-with-audit-log-flag
;; insert must be a no-op when :audit-log flag is not set
(with-redefs [app.config/flags #{:telemetry}]
(audit/insert th/*system* {:name "test" :type "action"})
(t/is (= 0 (count (th/db-exec! ["select * from audit_log"]))))))
(t/deftest insert-sets-defaults
;; insert must set defaults and persist when :audit-log is set
(with-redefs [app.config/flags #{:audit-log}]
(audit/insert th/*system* {:name "test-action" :type "action"})
(let [[row] (->> (th/db-exec! ["select * from audit_log"])
(mapv decode-row))]
(t/is (some? row))
(t/is (= "test-action" (:name row)))
(t/is (= "action" (:type row)))
(t/is (= "backend" (:source row)))
(t/is (some? (:id row)))
(t/is (some? (:created-at row)))
(t/is (some? (:tracked-at row)))
(t/is (= {} (:props row)))
(t/is (= {} (:context row))))))

383
backend/test/backend_tests/tasks_telemetry_test.clj
View File

@ -13,12 +13,23 @@
[app.loggers.audit :as audit]
[app.tasks.telemetry :as telemetry]
[app.util.blob :as blob]
[app.util.json :as json]
[backend-tests.helpers :as th]
[clojure.test :as t]
[mockery.core :refer [with-mocks]]))
[mockery.core :refer [with-mocks]]
[promesa.exec :as px]))
(t/use-fixtures :once th/state-init)
(t/use-fixtures :each th/database-reset)
;; Mock px/sleep for all tests to avoid 10s random delays.
;; Composed with database-reset so both apply.
(defn- test-fixture [next]
(th/database-reset
(fn []
(with-redefs [px/sleep (constantly nil)]
(next)))))
(t/use-fixtures :each test-fixture)
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; HELPERS
@ -67,6 +78,7 @@
(t/is (:called? @mock))
(let [[_ data] (-> @mock :call-args)]
(t/is (= :telemetry-legacy-report (:type data)))
(t/is (contains? data :subscriptions))
(t/is (= [(:email prof)] (:subscriptions data)))
(t/is (contains? data :stats))
@ -86,7 +98,10 @@
(t/is (contains? stats :max-projects-on-team))
(t/is (contains? stats :avg-files-on-project))
(t/is (contains? stats :email-domains))
(t/is (= ["nodomain.com"] (:email-domains stats))))
(t/is (= ["nodomain.com"] (:email-domains stats)))
;; public-uri must be a string
(t/is (string? (:public-uri stats)))
(t/is (not-empty (:public-uri stats))))
(t/is (contains? data :version))
(t/is (contains? data :instance-id))))))
@ -225,7 +240,8 @@
(t/is (contains? ev :type))
(t/is (contains? ev :source))
(t/is (contains? ev :profile-id))
(t/is (not (contains? ev :props)))
;; props are present but empty (stripped at ingest time)
(t/is (= {} (:props ev)))
(t/is (not (contains? ev :ip-addr)))))))))
(t/deftest test-batch-encoding-is-decodable
@ -383,7 +399,7 @@
:browser-version "120.0"
:os "Linux"
:version "2.0.0"}]
;; Simulate what app.loggers.audit/handle-event! does in mode C
;; Simulate what app.loggers.audit/process-event does in mode C
(th/db-insert! :audit-log
{:id (uuid/next)
:name "create-project"
@ -444,8 +460,9 @@
(t/is (contains? ev :source))
(t/is (contains? ev :tracked-at))
(t/is (contains? ev :profile-id))
;; props and ip-addr must be stripped
(t/is (not (contains? ev :props)))
;; props are present but empty (stripped at ingest time)
(t/is (= {} (:props ev)))
;; ip-addr is stripped
(t/is (not (contains? ev :ip-addr)))
;; context may be present and must not contain session-linking keys
(when-let [ctx (:context ev)]
@ -457,14 +474,20 @@
(t/deftest test-telemetry-rows-have-day-precision-timestamps
;; Telemetry events must be stored with timestamps truncated to day
;; precision so that exact event timing cannot be inferred.
(with-redefs [cf/flags #{:telemetry}
cf/telemetry-enabled? true]
(let [handle-event! (ns-resolve 'app.loggers.audit 'handle-event!)
(with-redefs [cf/flags #{:telemetry}]
(let [process-event (ns-resolve 'app.loggers.audit 'process-event)
profile (th/create-profile* 1 {:is-active true})
event {::audit/type "action"
::audit/name "create-project"
::audit/profile-id (:id profile)}]
(db/tx-run! th/*system* handle-event! event)
tnow (ct/now)
event {:type "action"
:name "create-project"
:profile-id (:id profile)
:source "backend"
:props {}
:context {}
:created-at tnow
:tracked-at tnow
:ip-addr "0.0.0.0"}]
(db/tx-run! th/*system* process-event event)
(let [[row] (th/db-exec! ["SELECT * FROM audit_log WHERE source = 'telemetry:backend'"])]
(t/is (some? row))
(let [created-at (:created-at row)
@ -475,24 +498,28 @@
(t/is (= day-now tracked-at)))))))
(t/deftest test-backend-ingest-full-row-shape
;; Verify the full row shape stored by handle-event! in telemetry mode:
;; Verify the full row shape stored by process-event in telemetry mode:
;; source=telemetry:backend, empty props, zeroed ip, context filtered to safe
;; backend keys only, profile-id preserved, timestamps truncated.
(with-redefs [cf/flags #{:telemetry}
cf/telemetry-enabled? true]
(let [handle-event! (ns-resolve 'app.loggers.audit 'handle-event!)
(with-redefs [cf/flags #{:telemetry}]
(let [process-event (ns-resolve 'app.loggers.audit 'process-event)
profile (th/create-profile* 1 {:is-active true})
event {::audit/type "action"
::audit/name "create-project"
::audit/profile-id (:id profile)
::audit/context {:initiator "app"
:version "2.0.0"
:client-version "1.0"
:client-user-agent "Mozilla/5.0"
:external-session-id "should-be-stripped"
:session "also-stripped"}
::audit/props {:some-prop "value"}}]
(db/tx-run! th/*system* handle-event! event)
tnow (ct/now)
event {:type "action"
:name "create-project"
:profile-id (:id profile)
:source "backend"
:context {:initiator "app"
:version "2.0.0"
:client-version "1.0"
:client-user-agent "Mozilla/5.0"
:external-session-id "should-be-stripped"
:session "also-stripped"}
:props {:some-prop "value"}
:created-at tnow
:tracked-at tnow
:ip-addr "0.0.0.0"}]
(db/tx-run! th/*system* process-event event)
(let [[row] (th/db-exec! ["SELECT * FROM audit_log WHERE source = 'telemetry:backend'"])]
(t/is (some? row))
@ -523,12 +550,12 @@
(t/is (not (contains? ctx :session))))))))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; FILTER-SAFE-CONTEXT UNIT TESTS
;; FILTER-TELEMETRY-CONTEXT UNIT TESTS
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(t/deftest test-filter-safe-context-keeps-browser-fields
(t/deftest test-filter-telemetry-context-keeps-browser-fields
;; Safe environment fields must survive the filter.
(let [filter-safe-context (ns-resolve 'app.rpc.commands.audit 'filter-safe-context)
(let [filter-telemetry-context (ns-resolve 'app.loggers.audit 'filter-telemetry-context)
ctx {:browser "Chrome"
:browser-version "120.0"
:engine "Blink"
@ -542,7 +569,7 @@
:screen-width 1920
:screen-height 1080
:event-origin "workspace"}
result (filter-safe-context ctx)]
result (:context (filter-telemetry-context {:source "frontend" :context ctx}))]
(t/is (= "Chrome" (:browser result)))
(t/is (= "120.0" (:browser-version result)))
(t/is (= "Windows 11" (:os result)))
@ -550,9 +577,9 @@
(t/is (= "workspace" (:event-origin result)))
(t/is (= 1920 (:screen-width result)))))
(t/deftest test-filter-safe-context-strips-pii-keys
(t/deftest test-filter-telemetry-context-strips-pii-keys
;; Session-linking and access-token fields must be removed.
(let [filter-safe-context (ns-resolve 'app.rpc.commands.audit 'filter-safe-context)
(let [filter-telemetry-context (ns-resolve 'app.loggers.audit 'filter-telemetry-context)
ctx {:browser "Firefox"
:session "abc-session-id"
:external-session-id "ext-123"
@ -560,7 +587,7 @@
:initiator "app"
:access-token-id "tok-456"
:access-token-type "api-key"}
result (filter-safe-context ctx)]
result (:context (filter-telemetry-context {:source "frontend" :context ctx}))]
(t/is (= "Firefox" (:browser result)))
(t/is (not (contains? result :session)))
(t/is (not (contains? result :external-session-id)))
@ -569,7 +596,285 @@
(t/is (not (contains? result :access-token-id)))
(t/is (not (contains? result :access-token-type)))))
(t/deftest test-filter-safe-context-empty-input
(t/deftest test-filter-telemetry-context-empty-input
;; An empty context should return an empty map without error.
(let [filter-safe-context (ns-resolve 'app.rpc.commands.audit 'filter-safe-context)]
(t/is (= {} (filter-safe-context {})))))
(let [filter-telemetry-context (ns-resolve 'app.loggers.audit 'filter-telemetry-context)]
(t/is (= {} (:context (filter-telemetry-context {:source "frontend" :context {}}))))))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; FILTER-TELEMETRY-PROPS UNIT TESTS
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(t/deftest test-filter-telemetry-props-login-event-keeps-safe-profile-fields
;; Login/register/update events carry safe profile-derived fields:
;; :lang, :auth-backend, :email-domain. Raw :email is stripped.
(let [ftp (ns-resolve 'app.loggers.audit 'filter-telemetry-props)]
;; backend login-with-password
(let [result (ftp {:source "backend"
:name "login-with-password"
:type "action"
:props {:email "user@example.com"
:fullname "John Doe"
:lang "en"
:auth-backend "password"
:id (uuid/next)}})]
(t/is (= "en" (get-in result [:props :lang])))
(t/is (= "password" (get-in result [:props :auth-backend])))
(t/is (= "example.com" (get-in result [:props :email-domain])))
;; Raw email and fullname are stripped
(t/is (not (contains? (:props result) :email)))
(t/is (not (contains? (:props result) :fullname)))
;; UUID values survive the xf:filter-telemetry-props filter
(t/is (some? (get-in result [:props :id]))))
;; backend register-profile
(let [result (ftp {:source "backend"
:name "register-profile"
:type "action"
:props {:email "new@corp.org"
:lang "es"
:auth-backend "oidc"}})]
(t/is (= "es" (get-in result [:props :lang])))
(t/is (= "oidc" (get-in result [:props :auth-backend])))
(t/is (= "corp.org" (get-in result [:props :email-domain]))))
;; backend login-with-oidc
(let [result (ftp {:source "backend"
:name "login-with-oidc"
:type "action"
:props {:email "u@corp.io" :lang "fr" :auth-backend "oidc"}})]
(t/is (= "fr" (get-in result [:props :lang])))
(t/is (= "oidc" (get-in result [:props :auth-backend])))
(t/is (= "corp.io" (get-in result [:props :email-domain]))))
;; backend update-profile
(let [result (ftp {:source "backend"
:name "update-profile"
:type "action"
:props {:email "u@corp.io" :lang "de"}})]
(t/is (= "de" (get-in result [:props :lang])))
(t/is (= "corp.io" (get-in result [:props :email-domain]))))))
(t/deftest test-filter-telemetry-props-frontend-identify-keeps-safe-profile-fields
;; Frontend identify events also carry safe profile-derived fields.
(let [ftp (ns-resolve 'app.loggers.audit 'filter-telemetry-props)]
(let [result (ftp {:source "frontend"
:name "signin"
:type "identify"
:props {:email "user@example.com"
:fullname "Jane Doe"
:lang "pt"
:auth-backend "password"
:some-string "should-be-stripped"}})]
(t/is (= "pt" (get-in result [:props :lang])))
(t/is (= "password" (get-in result [:props :auth-backend])))
(t/is (= "example.com" (get-in result [:props :email-domain])))
;; PII stripped
(t/is (not (contains? (:props result) :email)))
(t/is (not (contains? (:props result) :fullname)))
;; String values that are not UUID/boolean/number are stripped
(t/is (not (contains? (:props result) :some-string))))))
(t/deftest test-filter-telemetry-props-instance-start-passthrough
;; instance-start trigger events pass through as-is.
(let [ftp (ns-resolve 'app.loggers.audit 'filter-telemetry-props)
props {:total-teams 5 :total-users 42 :version "2.0"}
result (ftp {:source "backend"
:name "instance-start"
:type "trigger"
:props props})]
(t/is (= props (:props result)))))
(t/deftest test-filter-telemetry-props-generic-event-keeps-uuid-boolean-number
;; Generic events (navigate, create-file, etc.) keep only entries
;; whose values are UUIDs, booleans, or numbers.
(let [ftp (ns-resolve 'app.loggers.audit 'filter-telemetry-props)
id (uuid/next)
result (ftp {:source "frontend"
:name "navigate"
:type "action"
:props {:project-id id
:team-id id
:route "dashboard-files"
:count 42
:active true
:label "should-be-stripped"}})]
;; UUIDs survive
(t/is (= id (get-in result [:props :project-id])))
(t/is (= id (get-in result [:props :team-id])))
;; Numbers survive
(t/is (= 42 (get-in result [:props :count])))
;; Booleans survive
(t/is (true? (get-in result [:props :active])))
;; Strings are stripped
(t/is (not (contains? (:props result) :route)))
(t/is (not (contains? (:props result) :label)))))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; SEND-EVENT-BATCH PAYLOAD STRUCTURE
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(t/deftest test-send-event-batch-payload-structure
;; Verify the HTTP request sent by send-event-batch carries the
;; correct outer wrapper: :type, :version, :instance-id, :events.
(let [captured-request (atom nil)]
(with-mocks [legacy-mock {:target 'app.tasks.telemetry/make-legacy-request
:return nil}
http-mock {:target 'app.http.client/req
:return {:status 200}}]
(with-redefs [cf/flags #{:telemetry}]
(insert-telemetry-row! "navigate")
(insert-telemetry-row! "create-file")
(th/run-task! :telemetry {:send? true :enabled? true})
;; http/req was called (by both send-legacy-data and send-event-batch)
(t/is (:called? @http-mock))
;; Find the call whose body contains :telemetry-events
(let [calls (filter (fn [args]
(let [[_ request] args
body (:body request)]
(and (string? body)
(re-find #"telemetry-events" body))))
(:call-args-list @http-mock))]
(t/is (= 1 (count calls)))
(let [[_ request] (first calls)
body (json/decode (:body request))]
;; Outer payload fields
(t/is (= "telemetry-events" (name (:type body))))
(t/is (string? (:version body)))
(t/is (some? (:instance-id body)))
;; :events is a base64-encoded blob
(t/is (string? (:events body)))
(t/is (pos? (count (:events body))))))))))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; TASK BRANCH COVERAGE
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(t/deftest test-enabled-no-subs-no-events-legacy-still-sends
;; When telemetry is enabled, there are no newsletter subscriptions
;; and no audit_log rows, the legacy report must still be sent.
(with-mocks [legacy-mock {:target 'app.tasks.telemetry/make-legacy-request
:return nil}
batch-mock {:target 'app.tasks.telemetry/send-event-batch
:return true}]
(with-redefs [cf/flags #{:telemetry}]
;; No profiles with newsletter-updates, no telemetry rows
(th/run-task! :telemetry {:send? true :enabled? true})
;; Legacy report was sent
(t/is (:called? @legacy-mock))
(let [[_ data] (:call-args @legacy-mock)]
(t/is (= :telemetry-legacy-report (:type data)))
(t/is (contains? data :stats))
;; No subscriptions in the payload
(t/is (not (contains? data :subscriptions))))
;; No events to batch-send
(t/is (not (:called? @batch-mock))))))
(t/deftest test-legacy-succeeds-batch-fails
;; The legacy report and event batch are independent paths.
;; When the batch endpoint fails, the legacy report must still
;; have been sent successfully.
(with-mocks [legacy-mock {:target 'app.tasks.telemetry/make-legacy-request
:return nil}
batch-mock {:target 'app.tasks.telemetry/send-event-batch
:return false}]
(with-redefs [cf/flags #{:telemetry}]
(insert-telemetry-row! "navigate")
(th/run-task! :telemetry {:send? true :enabled? true})
;; Legacy report was sent
(t/is (:called? @legacy-mock))
(let [[_ data] (:call-args @legacy-mock)]
(t/is (= :telemetry-legacy-report (:type data))))
;; Batch send was attempted but failed
(t/is (:called? @batch-mock))
;; Row still present (not deleted on failure)
(t/is (= 1 (count-telemetry-rows))))))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; GC + BATCH FAILURE INTERACTION
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(t/deftest test-gc-runs-even-when-batch-fails
;; GC must purge stale events regardless of whether the subsequent
;; batch send succeeds or fails.
(with-mocks [legacy-mock {:target 'app.tasks.telemetry/make-legacy-request
:return nil}
batch-mock {:target 'app.tasks.telemetry/send-event-batch
:return false}]
(with-redefs [cf/flags #{:telemetry}]
(let [eight-days (ct/minus (ct/now) (ct/duration {:days 8}))
one-day (ct/minus (ct/now) (ct/duration {:days 1}))]
;; Stale events (should be GC'd)
(insert-telemetry-row! "stale-1" {:created-at eight-days :tracked-at eight-days})
(insert-telemetry-row! "stale-2" {:created-at eight-days :tracked-at eight-days})
;; Fresh event (should survive GC but fail to send)
(insert-telemetry-row! "fresh" {:created-at one-day :tracked-at one-day})
(t/is (= 3 (count-telemetry-rows)))
(th/run-task! :telemetry {:send? true :enabled? true})
;; Batch send was attempted (and failed)
(t/is (:called? @batch-mock))
;; Stale rows were purged by GC, fresh row remains
(t/is (= 1 (count-telemetry-rows)))
(t/is (= "fresh" (:name (first (th/db-exec! ["SELECT name FROM audit_log WHERE source LIKE 'telemetry:%'"])))))))))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; ROW->EVENT CONTEXT GUARANTEE
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(t/deftest test-row->event-always-includes-context
;; row->event must always include :context as a map, even when the
;; DB column contains an empty transit object.
(let [row->event (ns-resolve 'app.tasks.telemetry 'row->event)]
;; With non-empty context
(let [ev (row->event {:name "test" :type "action" :source "telemetry:backend"
:tracked-at (ct/now) :profile-id uuid/zero
:context (db/tjson {:browser "Chrome"})})]
(t/is (contains? ev :context))
(t/is (= {:browser "Chrome"} (:context ev))))
;; With empty context ({} in transit)
(let [ev (row->event {:name "test" :type "action" :source "telemetry:backend"
:tracked-at (ct/now) :profile-id uuid/zero
:context (db/tjson {})})]
(t/is (contains? ev :context))
(t/is (= {} (:context ev))))))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; NO DUPLICATE EVENTS ON SUCCESS
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(t/deftest test-no-duplicate-events-after-successful-send
;; After a successful batch send, the sent rows must be deleted.
;; Running the task again must NOT re-send the same events.
(let [send-count (atom 0)]
(with-mocks [legacy-mock {:target 'app.tasks.telemetry/make-legacy-request
:return nil}]
(with-redefs [cf/flags #{:telemetry}
telemetry/send-event-batch
(fn [_cfg _batch]
(swap! send-count inc)
true)]
(insert-telemetry-row! "navigate")
(insert-telemetry-row! "create-file")
(t/is (= 2 (count-telemetry-rows)))
;; First run: sends and deletes
(th/run-task! :telemetry {:send? true :enabled? true})
(t/is (= 1 @send-count))
(t/is (= 0 (count-telemetry-rows)))
;; Second run: no events to send
(th/run-task! :telemetry {:send? true :enabled? true})
(t/is (= 1 @send-count)) ;; still 1, not 2
(t/is (= 0 (count-telemetry-rows)))))))

8
frontend/src/app/main/data/event.cljs
View File

@ -378,8 +378,12 @@
(l/debug :hint "event instrumentation initialized")
;; Fetch backend flags and only start event collection if
;; :audit-log or :telemetry is enabled
;; :audit-log or :telemetry is enabled. On RPC failure, proceed
;; with event collection anyway (backend will reject if truly disabled).
(->> (rp/cmd! :get-enabled-flags)
(rx/catch (fn [cause]
(l/debug :hint "unable to fetch backend flags, proceeding with event collection" :cause cause)
(rx/of #{:telemetry})))
(rx/mapcat (fn [flags]
(if (or (contains? flags :audit-log)
(contains? flags :telemetry))
@ -471,7 +475,7 @@
(fn []
(l/debug :hint "events batching stream terminated")))))
(fn [cause]
(l/debug :hint "unable to fetch backend flags" :cause cause))))))))
(l/warn :hint "unexpected error during event collection initialization" :cause cause))))))))
(defn event
[props]