mirror of
https://github.com/penpot/penpot.git
synced 2026-05-22 16:33:55 +00:00
d56c452288
When the :telemetry flag is ON and :audit-log is OFF, frontend and backend events are stored anonymously in the audit_log table and shipped in compressed batches by the existing telemetry task. Stored rows strip props and ip-addr but preserve the profile-id, since Penpot profile UUIDs are already anonymous random identifiers with no PII attached. Timestamps are truncated to day precision to avoid leaking exact event timing. Only a safe subset of context fields is preserved: - Backend events: initiator, version, client-version, client-user-agent - Frontend events: browser, os, locale, screen metrics and event-origin Backend (app.loggers.audit): - Store backend telemetry events with source='telemetry', the safe context subset described above, and timestamps truncated to day precision via ct/truncate. Frontend RPC (app.rpc.commands.audit): - Add filter-safe-context to retain only the allowed frontend context fields. - Add xf:map-telemetry-event-row transducer that anonymises frontend events before inserting them. - push-audit-events now accepts events when telemetry is active. Telemetry task (app.tasks.telemetry): - gc-telemetry-events: enforces a 100,000-row safety cap by dropping the oldest rows first. - collect-and-send-audit-events: loop that fetches up to 10,000 rows per iteration, encodes and sends each page, deletes it on success, and stops immediately on failure leaving remaining rows for retry. - send-event-batch: POSTs a fressian+zstd batch (base64-encoded via blob/encode-str) to the telemetry endpoint, including instance-id and profile-id per event. - delete-sent-events: deletes successfully shipped rows by id. Blob utilities (app.util.blob): - Add blob/encode-str and blob/decode-str: convenience wrappers that combine blob encoding with base64 for JSON-safe string transport. Database: - Add index on audit_log (source, created_at ASC) to support efficient queries for telemetry batch collection. Tests (backend-tests.tasks-telemetry-test): - 21 tests, 94 assertions covering all code paths: disabled/enabled telemetry, no-events no-op, happy-path batch send and delete, failure retention, payload anonymity, context stripping, timestamp day precision, batch encoding round-trip, multi-page iteration, GC cap enforcement. Signed-off-by: Andrey Antukh <niwi@niwi.nz>
107 lines
3.9 KiB
Clojure
107 lines
3.9 KiB
Clojure
;; This Source Code Form is subject to the terms of the Mozilla Public
|
|
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
;;
|
|
;; Copyright (c) KALEIDOS INC
|
|
|
|
(ns backend-tests.util-blob-test
|
|
(:require
|
|
[app.util.blob :as blob]
|
|
[clojure.string :as str]
|
|
[clojure.test :as t]))
|
|
|
|
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
;; encode-str / decode-str round-trip
|
|
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
|
|
(t/deftest encode-str-roundtrip-empty-map
|
|
(let [data {}]
|
|
(t/is (= data (blob/decode-str (blob/encode-str data))))))
|
|
|
|
(t/deftest encode-str-roundtrip-empty-vector
|
|
(let [data []]
|
|
(t/is (= data (blob/decode-str (blob/encode-str data))))))
|
|
|
|
(t/deftest encode-str-roundtrip-nil
|
|
(let [data nil]
|
|
(t/is (= data (blob/decode-str (blob/encode-str data))))))
|
|
|
|
(t/deftest encode-str-roundtrip-simple-map
|
|
(let [data {:name "penpot" :version 42}]
|
|
(t/is (= data (blob/decode-str (blob/encode-str data))))))
|
|
|
|
(t/deftest encode-str-roundtrip-nested-structure
|
|
(let [data {:users [{:name "Alice" :tags #{"admin" "active"}}
|
|
{:name "Bob" :tags #{"user"}}]
|
|
:config {:debug false :timeout 3000}}]
|
|
(t/is (= data (blob/decode-str (blob/encode-str data))))))
|
|
|
|
(t/deftest encode-str-roundtrip-vector-of-maps
|
|
(let [data [{:name "navigate" :type "action" :source "telemetry"}
|
|
{:name "create-file" :type "action" :source "telemetry"}]]
|
|
(t/is (= data (blob/decode-str (blob/encode-str data))))))
|
|
|
|
(t/deftest encode-str-roundtrip-keywords-and-strings
|
|
(let [data {:keyword/value :foo
|
|
:string/value "hello world"
|
|
:boolean/value true
|
|
:nil/value nil}]
|
|
(t/is (= data (blob/decode-str (blob/encode-str data))))))
|
|
|
|
(t/deftest encode-str-roundtrip-numeric-types
|
|
(let [data {:int 42
|
|
:neg -7
|
|
:zero 0
|
|
:big 9999999999}]
|
|
(t/is (= data (blob/decode-str (blob/encode-str data))))))
|
|
|
|
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
;; URL-safe encoding properties
|
|
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
|
|
(t/deftest encode-str-url-safe-no-unsafe-chars
|
|
;; URL-safe base64 must not contain +, /, or padding =
|
|
(let [data {:a (apply str (repeat 100 "x"))
|
|
:b (range 200)
|
|
:c {"key" "value with special chars: @#$%^&*()"}}
|
|
encoded (blob/encode-str data)]
|
|
(t/is (not (str/includes? encoded "+")))
|
|
(t/is (not (str/includes? encoded "/")))
|
|
(t/is (not (str/includes? encoded "=")))))
|
|
|
|
(t/deftest encode-str-url-safe-roundtrip-after-encoding
|
|
;; Ensure the URL-safe encoding still round-trips correctly
|
|
(let [data {:payload (vec (range 500))
|
|
:nested {:a {:b {:c "deep"}}}}
|
|
encoded (blob/encode-str data)
|
|
decoded (blob/decode-str encoded)]
|
|
(t/is (= data decoded))))
|
|
|
|
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
;; version-specific encoding
|
|
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
|
|
(t/deftest encode-str-with-version-4
|
|
(let [data {:events [{:name "click"} {:name "scroll"}]}
|
|
encoded (blob/encode-str data {:version 4})
|
|
decoded (blob/decode-str encoded)]
|
|
(t/is (= data decoded))))
|
|
|
|
(t/deftest encode-str-with-version-5
|
|
(let [data {:events [{:name "click"} {:name "scroll"}]}
|
|
encoded (blob/encode-str data {:version 5})
|
|
decoded (blob/decode-str encoded)]
|
|
(t/is (= data decoded))))
|
|
|
|
(t/deftest encode-str-with-version-1
|
|
(let [data {:simple "data"}
|
|
encoded (blob/encode-str data {:version 1})
|
|
decoded (blob/decode-str encoded)]
|
|
(t/is (= data decoded))))
|
|
|
|
(t/deftest encode-str-with-version-3
|
|
(let [data {:simple "data"}
|
|
encoded (blob/encode-str data {:version 3})
|
|
decoded (blob/decode-str encoded)]
|
|
(t/is (= data decoded))))
|