diff --git a/common/deps.edn b/common/deps.edn index e7ed617f90..ba6e28cc89 100644 --- a/common/deps.edn +++ b/common/deps.edn @@ -48,6 +48,11 @@ frankiesardo/linked {:mvn/version "1.3.0"} + ;; Resolves token values, reads and writes token files. + penpot/tokenscript-interpreter + {:git/sha "321061dbddaf78270319136eb41cfbff172fba02" + :git/url "https://github.com/penpot/tokenscript-interpreter.git"} + com.sun.mail/jakarta.mail {:mvn/version "2.0.2"} org.la4j/la4j {:mvn/version "0.6.0"} diff --git a/common/src/app/common/types/tokens_lib.cljc b/common/src/app/common/types/tokens_lib.cljc index 082da2d43e..d045e93e55 100644 --- a/common/src/app/common/types/tokens_lib.cljc +++ b/common/src/app/common/types/tokens_lib.cljc @@ -25,7 +25,10 @@ [clojure.pprint :as pp] [clojure.set :as set] [clojure.walk :as walk] - [cuerdas.core :as str])) + [cuerdas.core :as str] + [tokenscript.document.dtcg :as tsd] + [tokenscript.document.read :as tsr] + [tokenscript.document.write :as tsw])) ;; === Common @@ -1460,127 +1463,18 @@ active-tokens))) ;; === Import / Export from JSON format - -;; Supported formats: -;; - Legacy: for tokens files prior to DTCG second draft -;; - DTCG: for tokens files conforming to the DTCG second draft (current for now) -;; https://www.w3.org/community/design-tokens/2022/06/14/call-to-implement-the-second-editors-draft-and-share-feedback/ ;; -;; - Single set: for files that comply with the base DTCG format, that contain a single tree of tokens. -;; - Multi sets: for files with the Tokens Studio extension, that may contain several sets, and also themes and other $metadata. +;; Token files are read and written by `tokenscript.document.read` and +;; `tokenscript.document.write`, which know the formats: DTCG (the 2022 +;; drafts and the Format Module 2025.10), legacy and multi-set files. +;; This section maps its documents to a TokensLib and back: penpot's +;; token types, the keys of composite values, and the activation of +;; themes and sets. ;; ;; Small glossary: -;; * json data: a json-encoded string -;; * decode: convert a json string into a plain clojure nested map -;; * parse: build a TokensLib (or a fragment) from a decoded json data -;; * export: generate from a TokensLib a plain clojure nested map, suitable to be encoded as a json string - -(def ^:private legacy-node? - (sm/validator - [:or - [:map - ["value" :string] - ["type" :string]] - [:map - ["value" [:sequential [:map ["type" :string]]]] - ["type" :string]] - [:map - ["value" :map] - ["type" :string]]])) - -(def ^:private schema:dtcg-node - ;; Per the DTCG Community Group Final Report (W3C, 2025-10-28): - ;; "The presence of a $value property definitively identifies an object as a token." - ;; "A token's type can be specified by the optional $type property [...] - ;; Furthermore, the $type property on a group applies to all tokens nested - ;; within that group." - ;; So $value is the only required property; $type may be inherited from a - ;; parent group and is therefore optional on the token itself. - [:schema {:registry - {::simple-value - [:or :string :int :double ::sm/boolean] - ::value - [:or - [:ref ::simple-value] - [:vector ::simple-value] - [:vector [:map-of :string ::simple-value]] - [:map-of :string [:or - [:ref ::simple-value] - [:vector ::simple-value]]]]}} - [:map - ["$type" {:optional true} :string] - ["$value" [:ref ::value]]]]) - -(def ^:private dtcg-node? - (sm/validator schema:dtcg-node)) - -(defn- get-json-format - "Searches through decoded token file and returns: - - `:json-format/legacy` when first node satisfies `legacy-node?` predicate - - `:json-format/dtcg` when first node satisfies `dtcg-node?` predicate - - If neither combination is found, return dtcg format by default (we assume that - the file does not contain any token, so the format is irrelevan)." - ([decoded-json] - (get-json-format decoded-json legacy-node? dtcg-node?)) - ([decoded-json legacy-node? dtcg-node?] - (assert (map? decoded-json) "expected a plain clojure map for `decoded-json`") - (let [branch? map? - children (fn [node] (vals node)) - check-node (fn [node] - (cond - (legacy-node? node) :json-format/legacy - (dtcg-node? node) :json-format/dtcg - :else nil)) - walk (fn walk [node] - (lazy-seq - (cons - (check-node node) - (when (branch? node) - (mapcat walk (children node))))))] - (d/nilv (->> (walk decoded-json) - (filter some?) - first) - :json-format/dtcg)))) - -(defn- legacy-json->dtcg-json - "Converts a decoded json file in legacy format into DTCG format." - [decoded-json] - (assert (map? decoded-json) "expected a plain clojure map for `decoded-json`") - (walk/postwalk - (fn [node] - (cond-> node - ;; Handle sequential values that are objects with type - (and (map? node) - (contains? node "value") - (sequential? (get node "value")) - (map? (first (get node "value")))) - (update "value" - (fn [seq-value] - (map #(set/rename-keys % {"type" "$type"}) seq-value))) - - ;; Keep array of font families - (and (map? node) - (contains? node "type") - (= "fontFamilies" (get node "type")) - (contains? node "value") - (sequential? (get node "value")) - (not (map? (first (get node "value"))))) - identity - - ;; Rename keys for all token nodes - (and (map? node) - (and (contains? node "type") - (contains? node "value"))) - (set/rename-keys {"value" "$value" - "type" "$type"}))) - decoded-json)) - -(defn- single-set? - "Check if the decoded json file conforms to basic DTCG format with a single set." - [decoded-json] - (assert (map? decoded-json) "expected a plain clojure map for `decoded-json`") - (and (not (contains? decoded-json "$metadata")) - (not (contains? decoded-json "$themes")))) +;; * decoded json: a json file decoded into a plain map with string keys +;; * parse / import: build a TokensLib from decoded json +;; * export: build decoded json from a TokensLib (defn convert-dtcg-font-family "Convert font-family token value from DTCG format to internal format. @@ -1605,255 +1499,218 @@ ;; Reference value value)) -(defn convert-dtcg-shadow-composite - "Convert shadow token value from DTCG format to internal format." +(def ^:private dtcg-shadow-keys + {"offsetX" :offset-x + "offsetY" :offset-y + "blur" :blur + "spread" :spread + "color" :color + "inset" :inset}) + +(def ^:private dtcg-shadow-keys-inverted + (set/map-invert dtcg-shadow-keys)) + +(defn- shadow-with-dtcg-keys + [shadow] + (if (map? shadow) + (set/rename-keys shadow dtcg-shadow-keys-inverted) + shadow)) + +(defn- shadows-with-penpot-keys + "Shadows read by `tokenscript.document.dtcg/read-shadow` with penpot's + keys." [value] - (let [process-shadow (fn [shadow] - (if (map? shadow) - (let [legacy-shadow-type (get shadow "type")] - (-> shadow - (set/rename-keys {"x" :offset-x - "offsetX" :offset-x - "y" :offset-y - "offsetY" :offset-y - "blur" :blur - "spread" :spread - "color" :color - "inset" :inset}) - (update :inset #(cond - (boolean? %) % - (= "true" %) true - (= "false" %) false - (= legacy-shadow-type "innerShadow") true - :else false)) - (select-keys [:offset-x :offset-y :blur :spread :color :inset]))) - shadow))] - (cond - ;; Reference value - keep as string - (string? value) - value + (if (sequential? value) + (mapv (fn [shadow] + (if (map? shadow) + (-> (set/rename-keys shadow dtcg-shadow-keys) + (select-keys (vals dtcg-shadow-keys))) + shadow)) + value) + value)) - ;; Array of shadows - process each - (sequential? value) - (mapv process-shadow value) +(defn convert-dtcg-shadow-composite + "Convert shadow token value from DTCG format to internal format. The + value is read by `tokenscript.document.dtcg/read-shadow` first, which + takes single shadows and the legacy `x`, `y` and `innerShadow`. A value + already in the internal format, as plugins set it, is kept." + [value] + (shadows-with-penpot-keys + (tsd/read-shadow (if (sequential? value) + (mapv shadow-with-dtcg-keys value) + (shadow-with-dtcg-keys value))))) - ;; Single shadow object - wrap in vector - (map? value) - [(process-shadow value)] +(defn- file-value->token-value + "The token value of a value read from a file. The reader has already + read shadows with `read-shadow`, so only their keys change." + [token-type value] + (case token-type + :font-family (convert-dtcg-font-family value) + :typography (convert-dtcg-typography-composite value) + :shadow (shadows-with-penpot-keys value) + value)) - ;; Fallback - keep as is - :else - value))) +(defn- document-token + "A penpot token from the token data of a document, or nil when penpot + has no type for it." + [token-name {:keys [type value description]}] + (when-let [token-type (cto/dtcg-token-type->token-type type)] + (make-token :name token-name + :type token-type + :value (file-value->token-value token-type value) + :description description))) -(defn- flatten-nested-tokens-json - "Convert a tokens tree in the decoded json fragment into a flat map, - being the keys the token paths after joining the keys with '.'. +(defn- document-token-path + [{:keys [document]} set-name token-name] + (if (= :multi-set document) + (str set-name "." token-name) + token-name)) - Per the DTCG spec, a node is a token iff it carries a `$value` - property; every other map is a group. A group may declare a `$type` - that is inherited by every nested token that does not declare its - own `$type`, so this walker propagates the nearest enclosing group - `$type` down through the recursion." - ([decoded-json-tokens parent-path] - (flatten-nested-tokens-json decoded-json-tokens parent-path nil)) - ([decoded-json-tokens parent-path inherited-type] - (reduce-kv - (fn [tokens k v] - (let [child-path (if (empty? parent-path) - (name k) - (str parent-path "." k))] - (if (and (map? v) - (not (contains? v "$value"))) - ;; Group: recurse, letting the group's `$type` (if any) replace - ;; the inherited type for descendants that don't declare their - ;; own `$type`. - (merge tokens - (flatten-nested-tokens-json - v - child-path - (or (get v "$type") inherited-type))) - ;; Token: resolve the type from the token's own `$type`, - ;; falling back to the inherited group `$type`. - (let [type-key (or (get v "$type") inherited-type) - token-type (cto/dtcg-token-type->token-type type-key)] - (if token-type - (assoc tokens child-path (make-token - :name child-path - :type token-type - :value - (let [token-value (get v "$value")] - (case token-type - :font-family (convert-dtcg-font-family token-value) - :typography (convert-dtcg-typography-composite token-value) - :shadow (convert-dtcg-shadow-composite token-value) - token-value)) - :description (get v "$description"))) - ;; Discard unknown / un-typeable tokens - tokens))))) - {} - decoded-json-tokens))) +(defn- dropped-properties + "The properties of a token or group that penpot does not store; token + descriptions are stored, group descriptions are not. The reader only + keeps a token's `$extensions` when it is an object, so a malformed one + goes unreported on a token, unlike on a group." + [{:keys [description deprecated extensions properties]} group?] + (cond-> (vec (keys properties)) + (and group? (some? description)) (conj "$description") + (some? deprecated) (conj "$deprecated") + (d/not-empty? extensions) (conj "$extensions"))) -(defn- parse-single-set-dtcg-json - "Parse a decoded json file with a single set of tokens in DTCG format into a TokensLib." - [set-name decoded-json-tokens] - (assert (map? decoded-json-tokens) "expected a plain clojure map for `decoded-json-tokens`") - (assert (= (get-json-format decoded-json-tokens) :json-format/dtcg) "expected a dtcg format for `decoded-json-tokens`") +(defn- import-warnings + "What penpot drops from a document: `:unknown-types` maps the tokens of + types penpot does not have to their type, `:unsupported` lists the + properties penpot does not store (`$extensions`, `$deprecated`, group + descriptions, ...) and the tokens the reader dropped, as + `{:property :path}`." + [{:keys [sets issues] :as document}] + (let [path (partial document-token-path document)] + {:unknown-types + (not-empty + (into {} + (for [{set-name :name :keys [tokens]} sets + [token-name {:keys [type]}] tokens + :when (nil? (cto/dtcg-token-type->token-type type))] + [(path set-name token-name) type]))) - (let [set-name (normalize-set-name set-name) - tokens (flatten-nested-tokens-json decoded-json-tokens "")] + :unsupported + (not-empty + (vec (concat + (for [{set-name :name :keys [tokens groups]} sets + [node-name data group?] (concat (map (fn [[k v]] [k v false]) tokens) + (map (fn [[k v]] [k v true]) groups)) + property (dropped-properties data group?)] + {:property property :path (path set-name node-name)}) + (for [{:keys [code data] :as issue} issues + :let [[property node-name] + (case code + (:dtcg/invalid-value :dtcg/invalid-node) ["$value" (:token-name data)] + :dtcg/missing-type ["$type" (:token-name data)] + :dtcg/invalid-name ["$root" (:token-name data)] + (:dtcg/invalid-extends :dtcg/circular-extends) ["$extends" (:group data)] + nil)] + :when property] + {:property property :path (path (:set issue) node-name)}))))})) - (when (empty? tokens) +(defn- activate-theme-path + "Activates the theme at `theme-path` in `library`, deactivating the + other themes of its group. The hidden theme, which holds the active + sets of a file without themes, stays active." + [library theme-path] + (let [[group name] (split-theme-path theme-path)] + (if-let [theme (get-theme-by-name library group name)] + (let [group-paths (into #{} + (comp (filter #(= group (:group %))) + (remove hidden-theme?) + (map get-theme-path)) + (get-themes library))] + (TokensLib. (.-sets ^TokensLib library) + (.-themes ^TokensLib library) + (-> (set/difference (.-active-themes ^TokensLib library) group-paths) + (conj (get-theme-path theme))))) + library))) + +(defn- document->tokens-lib + "Builds a TokensLib from a document read by + `tokenscript.document.read/read`. Sets a theme marks `:disabled` are + not part of the theme." + [{doc-sets :sets :keys [document themes active-themes active-sets]}] + (let [xf-normalize-set-name (map normalize-set-name) + + sets + (reduce (fn [sets {:keys [name tokens]}] + (assoc sets + (normalize-set-name name) + (into {} + (keep (fn [[token-name data]] + (when-let [token (document-token token-name data)] + [token-name token]))) + tokens))) + (d/ordered-map) + doc-sets) + + themes + (->> themes + (map (fn [{:keys [id name group description is-source] theme-sets :sets}] + (make-token-theme + :id (or (uuid/parse* id) (uuid/next)) + :name name + :group group + :description description + :is-source is-source + :external-id id + :sets (into #{} + (comp (remove (comp #{:disabled} val)) + (map key) + xf-normalize-set-name + (filter #(contains? sets %))) + theme-sets)))) + (not-empty)) + + active-set-names + (into #{} xf-normalize-set-name active-sets) + + active-theme-names + (or (not-empty (vec (distinct active-themes))) + [hidden-theme-path])] + + (when (if (= :single-set document) + (every? empty? (vals sets)) + (and (empty? sets) (empty? themes))) (throw (ex-info "the file doesn't contain any tokens" {:error/code :error.import/invalid-json-data}))) - (-> (make-tokens-lib) - (add-set (make-token-set :name set-name - :tokens tokens))))) + (as-> (make-tokens-lib) library + (reduce-kv (fn [library name tokens] + (add-set library (make-token-set :name name :tokens tokens))) + library + sets) + (update-theme library hidden-theme-id #(assoc % :sets active-set-names)) + (reduce add-theme library themes) + (reduce activate-theme-path library active-theme-names)))) -(defn- parse-single-set-legacy-json - "Parse a decoded json file with a single set of tokens in legacy format into a TokensLib." - [set-name decoded-json-tokens] - (assert (map? decoded-json-tokens) "expected a plain clojure map for `decoded-json-tokens`") - (assert (= (get-json-format decoded-json-tokens) :json-format/legacy) "expected a legacy format for `decoded-json-tokens`") - (parse-single-set-dtcg-json set-name (legacy-json->dtcg-json decoded-json-tokens))) +(defn import-decoded-json + "Reads a decoded token file into a TokensLib, guessing its format. The + `file-name` is the set name when the file contains a single set. + Returns `{:tokens-lib lib :unknown-types .. :unsupported ..}`, see + `import-warnings`." + [decoded-json file-name] + (let [document (tsr/read decoded-json {:set-name file-name})] + (assoc (import-warnings document) + :tokens-lib (document->tokens-lib document)))) -(def ^:private schema:multi-set-dtcg - "Schema for penpot multi-set dtcg json decoded data/ - - Mainly used for validate the structure of the incoming data before - proceed to parse it to our internal data structures." - [:schema {:registry - {::node - [:or - [:map-of :string [:ref ::node]] - schema:dtcg-node]}} - [:map - ["$themes" {:optional true} - [:vector - [:map {:title "Theme"} - ["id" {:optional true} :string] - ["name" :string] - ["description" :string] - ["isSource" {:optional true} :boolean] - ["selectedTokenSets" - [:map-of :string [:enum "enabled" "disabled"]]]]]] - ["$metadata" {:optional true} - [:map {:title "Metadata"} - ["tokenSetOrder" {:optional true} [:vector :string]] - ["activeThemes" {:optional true} [:vector :string]] - ["activeSets" {:optional true} [:vector :string]]]] - - [:malli.core/default - [:map-of :string [:ref ::node]]]]]) - -(def ^:private check-multi-set-dtcg-data - (sm/check-fn schema:multi-set-dtcg)) - -(def ^:private decode-multi-set-dtcg-data - (sm/decoder schema:multi-set-dtcg - sm/json-transformer)) +;; FIXME: remove `-json` suffix +(defn parse-decoded-json + "Like `import-decoded-json`, the TokensLib only." + [decoded-json file-name] + (:tokens-lib (import-decoded-json decoded-json file-name))) ;; FIXME: remove `-json` suffix (defn parse-multi-set-dtcg-json - "Parse a decoded json file with multi sets in DTCG format into a TokensLib." + "Parse a decoded json file with multi sets into a TokensLib, also when + it has no `$themes` and no `$metadata`." [decoded-json] - (assert (map? decoded-json) "expected a plain clojure map for `decoded-json`") - (assert (= (get-json-format decoded-json) :json-format/dtcg) "expected a dtcg format for `decoded-json`") - - (let [metadata (get decoded-json "$metadata") - - xf-normalize-set-name - (map normalize-set-name) - - sets - (dissoc decoded-json "$themes" "$metadata") - - ordered-set-names - (-> (d/ordered-set) - (into xf-normalize-set-name (get metadata "tokenSetOrder")) - (into xf-normalize-set-name (keys sets))) - - active-set-names - (or (->> (get metadata "activeSets") - (into #{} xf-normalize-set-name) - (not-empty)) - #{}) - - active-theme-names - (or (->> (get metadata "activeThemes") - (into #{}) - (not-empty)) - #{hidden-theme-path}) - - themes - (->> (get decoded-json "$themes") - (map (fn [theme] - (make-token-theme - :id (or (uuid/parse* (get theme "id")) - (uuid/next)) - :name (get theme "name") - :group (get theme "group") - :is-source (or (get theme "isSource") - ;; NOTE: backward compatibility - (get theme "is-source")) - :external-id (get theme "id") - :sets (into #{} - (comp (map key) - xf-normalize-set-name - (filter #(contains? ordered-set-names %))) - (get theme "selectedTokenSets"))))) - (not-empty)) - - library - (make-tokens-lib) - - sets - (reduce-kv (fn [result name tokens] - (assoc result - (normalize-set-name name) - (flatten-nested-tokens-json tokens ""))) - {} - sets) - - library - (reduce (fn [library name] - (if-let [tokens (get sets name)] - (add-set library (make-token-set :name name :tokens tokens)) - library)) - library - ordered-set-names) - - library - (update-theme library hidden-theme-id - #(assoc % :sets active-set-names)) - - library - (reduce add-theme library themes) - - ;; Activate themes by directly manipulating the internal active-themes field - ;; (since activate-theme is being removed from the protocol) - library - (reduce (fn [library theme-path] - (let [[group name] (split-theme-path theme-path) - theme (get-theme-by-name library group name)] - (if theme - (let [group-themes (->> (get themes group) - (map (comp get-theme-path val)) - (into #{})) - active-themes' (-> (set/difference (.-active-themes ^TokensLib library) group-themes) - (conj (get-theme-path theme)))] - (TokensLib. (.-sets ^TokensLib library) - (.-themes ^TokensLib library) - active-themes')) - library))) - library - active-theme-names)] - - (when (and (empty? sets) (empty? themes)) - (throw (ex-info "the file doesn't contain any tokens" - {:error/code :error.import/invalid-json-data}))) - - library)) + (document->tokens-lib (tsr/read decoded-json {:multi-set? true}))) (defn read-multi-set-dtcg "Read penpot multi-set dctg tokens. Accepts string or JSON decoded @@ -1868,45 +1725,8 @@ data #?(:cljs (if (object? data) (json/->clj data :key-fn identity) data) - :clj data) - - data (decode-multi-set-dtcg-data data)] - (-> (check-multi-set-dtcg-data data) - (parse-multi-set-dtcg-json))))) - -(defn- parse-multi-set-legacy-json - "Parse a decoded json file with multi sets in legacy format into a TokensLib." - [decoded-json] - (assert (map? decoded-json) "expected a plain clojure map for `decoded-json`") - (assert (= (get-json-format decoded-json) :json-format/legacy) "expected a legacy format for `decoded-json`") - - (let [sets-data (dissoc decoded-json "$themes" "$metadata") - other-data (select-keys decoded-json ["$themes" "$metadata"]) - dtcg-sets-data (legacy-json->dtcg-json sets-data)] - (parse-multi-set-dtcg-json (merge other-data - dtcg-sets-data)))) - -;; FIXME: remove `-json` suffix -(defn parse-decoded-json - "Guess the format and content type of the decoded json file and parse it into a TokensLib. - The `file-name` is used to determine the set name when the json file contains a single set." - [decoded-json file-name] - (let [single-set? (single-set? decoded-json) - json-format (get-json-format decoded-json)] - (cond - (and single-set? - (= :json-format/legacy json-format)) - (parse-single-set-legacy-json file-name decoded-json) - - (and single-set? - (= :json-format/dtcg json-format)) - (parse-single-set-dtcg-json file-name decoded-json) - - (= :json-format/legacy json-format) - (parse-multi-set-legacy-json decoded-json) - - :else - (parse-multi-set-dtcg-json decoded-json)))) + :clj data)] + (parse-multi-set-dtcg-json data)))) (defn- typography-token->dtcg-token [value] @@ -1926,234 +1746,98 @@ (mapv (fn [shadow] (if (map? shadow) (-> shadow - (set/rename-keys {:offset-x "offsetX" - :offset-y "offsetY" - :blur "blur" - :spread "spread" - :color "color" - :inset "inset"}) - (select-keys ["offsetX" "offsetY" "blur" "spread" "color" "inset"])) + (set/rename-keys dtcg-shadow-keys-inverted) + (select-keys (keys dtcg-shadow-keys))) shadow)) value) value)) -(defn- token->dtcg-token [token] - (cond-> {"$value" (cond-> (:value token) - ;; Transform typography token values - (= :typography (:type token)) - typography-token->dtcg-token - ;; Transform shadow token values - (= :shadow (:type token)) - shadow-token->dtcg-token) - "$type" (cto/token-type->dtcg-token-type (:type token))} - (:description token) (assoc "$description" (:description token)))) +(defn- token->document-token + [token] + {:value (cond-> (:value token) + (= :typography (:type token)) typography-token->dtcg-token + (= :shadow (:type token)) shadow-token->dtcg-token) + :type (cto/token-type->dtcg-token-type (:type token)) + :description (:description token)}) -(defn- dtcg-export-themes - "Extract themes for a dtcg json export." - ([tokens-lib] - ;; Backward-compatible version for print/serialization without tokens-status - (let [themes-xform - (comp - (filter #(and (instance? TokenTheme %) - (not (hidden-theme? %)))) - (map (fn [token-theme] - ;; NOTE: this probaly can be implemented as type method - (d/without-nils - {"id" (:external-id token-theme) - "name" (:name token-theme) - "group" (:group token-theme) - "description" (:description token-theme) - "isSource" (:is-source token-theme) - "selectedTokenSets" (reduce #(assoc %1 %2 "enabled") {} (:sets token-theme))})))) +(defn- token-theme->document-theme + [token-theme] + (d/without-nils + {:id (:external-id token-theme) + :name (:name token-theme) + :group (:group token-theme) + :description (:description token-theme) + :is-source (:is-source token-theme) + :sets (zipmap (:sets token-theme) (repeat :enabled))})) - themes - (->> (get-theme-tree tokens-lib) - (tree-seq d/ordered-map? vals) - (into [] themes-xform)) +(defn- legacy-activation + "Active theme paths and set names from the lib's own active themes, + for printing and serialization without a TokensStatus." + [tokens-lib] + (let [active-themes (.-active-themes ^TokensLib tokens-lib)] + [(disj active-themes hidden-theme-path) + (into #{} + (comp (filter #(contains? active-themes (get-theme-path %))) + (mapcat :sets)) + (get-themes tokens-lib))])) - ;; Active themes without exposing hidden penpot theme - active-themes - (-> (.-active-themes ^TokensLib tokens-lib) - (disj hidden-theme-path))] - [themes active-themes])) - ([tokens-lib active-theme-ids] - ;; Version with explicit active-theme-ids from TokensStatus - (let [themes-xform - (comp - (filter #(and (instance? TokenTheme %) - (not (hidden-theme? %)))) - (map (fn [token-theme] - (d/without-nils - {"id" (:external-id token-theme) - "name" (:name token-theme) - "group" (:group token-theme) - "description" (:description token-theme) - "isSource" (:is-source token-theme) - "selectedTokenSets" (reduce #(assoc %1 %2 "enabled") {} (:sets token-theme))})))) +(defn- status-activation + "Active theme paths and set names from a TokensStatus." + [tokens-lib tokens-status] + (let [active-set-ids (ctos/get-active-set-ids tokens-status)] + [(into #{} + (comp (map #(get-theme tokens-lib %)) + (filter some?) + (map get-theme-path) + (remove #(= % hidden-theme-path))) + (ctos/get-active-theme-ids tokens-status)) + (into #{} + (comp (filter #(contains? active-set-ids (get-id %))) + (map get-name)) + (get-sets tokens-lib))])) - themes - (->> (get-theme-tree tokens-lib) - (tree-seq d/ordered-map? vals) - (into [] themes-xform)) - - ;; Convert theme ids to theme paths, excluding hidden theme - active-themes - (into #{} - (comp (map #(get-theme tokens-lib %)) - (filter some?) - (map get-theme-path) - (filter #(not= % hidden-theme-path))) - active-theme-ids)] - [themes active-themes]))) +(defn- tokens-lib->document + [tokens-lib [active-themes active-sets]] + {:sets (mapv (fn [token-set] + {:name (get-name token-set) + :tokens (d/update-vals (get-tokens- token-set) token->document-token)}) + (get-sets tokens-lib)) + :themes (into [] + (comp (filter #(and (instance? TokenTheme %) (not (hidden-theme? %)))) + (map token-theme->document-theme)) + (tree-seq d/ordered-map? vals (get-theme-tree tokens-lib))) + :active-themes active-themes + :active-sets active-sets}) (defn export-dtcg-multi-file - "Convert a TokensLib into a plain clojure map, suitable to be encoded as a multi json files each encoded in DTCG format." + "Convert a TokensLib into a plain clojure map, suitable to be encoded + as multiple json files, one per set plus `$themes.json` and + `$metadata.json`. Options: `:profile`, `:default` or `:dtcg` for + DTCG 2025.10 values and types, see `tokenscript.document.write/write`. + A `:dtcg` file is not meant to import back unchanged: tokens come back + with their DTCG type and value, a spacing token as a dimension, `4` as + `4px` and `rgb(255, 0, 0)` as `#ff0000`." ([tokens-lib] - ;; Backward-compatible version for print/serialization without tokens-status - (let [[themes active-themes] - (dtcg-export-themes tokens-lib) - - sets - (->> (get-sets tokens-lib) - (map (fn [token-set] - (let [name (get-name token-set) - tokens (get-tokens- token-set)] - [(str name ".json") (tokens-tree tokens :update-token-fn token->dtcg-token)]))) - (into {}))] - - (-> sets - (assoc "$themes.json" themes) - (assoc "$metadata.json" - {"tokenSetOrder" (get-set-names tokens-lib) - "activeThemes" active-themes - ;; For backward compat, use internal active-themes field - "activeSets" (into #{} - (comp (filter #(and (instance? TokenTheme %) - (contains? (.-active-themes ^TokensLib tokens-lib) - (get-theme-path %)))) - (mapcat :sets)) - (get-themes tokens-lib))})))) + (tsw/write-files (tokens-lib->document tokens-lib (legacy-activation tokens-lib)))) ([tokens-lib tokens-status] - ;; Version with tokens-status for real exports - (let [active-theme-ids (ctos/get-active-theme-ids tokens-status) - [themes active-themes] - (dtcg-export-themes tokens-lib active-theme-ids) - - sets - (->> (get-sets tokens-lib) - (map (fn [token-set] - (let [name (get-name token-set) - tokens (get-tokens- token-set)] - [(str name ".json") (tokens-tree tokens :update-token-fn token->dtcg-token)]))) - (into {})) - - active-set-ids (ctos/get-active-set-ids tokens-status) - active-set-names (into #{} - (comp (filter #(contains? active-set-ids (get-id %))) - (map get-name)) - (get-sets tokens-lib))] - - (-> sets - (assoc "$themes.json" themes) - (assoc "$metadata.json" - {"tokenSetOrder" (get-set-names tokens-lib) - "activeThemes" active-themes - "activeSets" active-set-names}))))) + (export-dtcg-multi-file tokens-lib tokens-status nil)) + ([tokens-lib tokens-status opts] + (tsw/write-files (tokens-lib->document tokens-lib (status-activation tokens-lib tokens-status)) + opts))) (defn export-dtcg-json - "Convert a TokensLib into a plain clojure map, suitable to be encoded as a multi sets json string in DTCG format." + "Convert a TokensLib into a plain clojure map, suitable to be encoded + as a multi sets json string, or nil for an empty lib. Options as in + `export-dtcg-multi-file`." ([tokens-lib] - ;; Backward-compatible version for print/serialization without tokens-status - (let [[themes active-themes] - (dtcg-export-themes tokens-lib) - - name-set-tuples - (->> (get-set-tree tokens-lib) - (tree-seq d/ordered-map? vals) - (filter (partial instance? TokenSet)) - (map (fn [set] - [(get-name set) - (tokens-tree (get-tokens- set) :update-token-fn token->dtcg-token)]))) - - ordered-set-names - (mapv first name-set-tuples) - - sets - (into {} name-set-tuples) - - ;; For backward compat, use internal active-themes field - active-set-names (into #{} - (comp (filter #(and (instance? TokenTheme %) - (contains? (.-active-themes ^TokensLib tokens-lib) - (get-theme-path %)))) - (mapcat :sets)) - (get-themes tokens-lib))] - - (when-not (empty-lib? tokens-lib) - (-> sets - (assoc "$themes" themes) - (assoc "$metadata" {"tokenSetOrder" ordered-set-names - "activeThemes" active-themes - "activeSets" active-set-names}))))) + (when-not (empty-lib? tokens-lib) + (tsw/write (tokens-lib->document tokens-lib (legacy-activation tokens-lib))))) ([tokens-lib tokens-status] - ;; Version with tokens-status for real exports - (let [active-theme-ids (ctos/get-active-theme-ids tokens-status) - [themes active-themes] - (dtcg-export-themes tokens-lib active-theme-ids) - - name-set-tuples - (->> (get-set-tree tokens-lib) - (tree-seq d/ordered-map? vals) - (filter (partial instance? TokenSet)) - (map (fn [set] - [(get-name set) - (tokens-tree (get-tokens- set) :update-token-fn token->dtcg-token)]))) - - ordered-set-names - (mapv first name-set-tuples) - - sets - (into {} name-set-tuples) - - active-set-ids (ctos/get-active-set-ids tokens-status) - active-set-names (into #{} - (comp (filter #(contains? active-set-ids (get-id %))) - (map get-name)) - (get-sets tokens-lib))] - - (when-not (empty-lib? tokens-lib) - (-> sets - (assoc "$themes" themes) - (assoc "$metadata" {"tokenSetOrder" ordered-set-names - "activeThemes" active-themes - "activeSets" active-set-names})))))) - -(defn get-tokens-of-unknown-type - "Search for all tokens in the decoded json file that have a type that is not currently - supported by Penpot. Returns a map token-path -> token type." - [decoded-json {:keys [json-format parent-path process-token-type] - :or {json-format (get-json-format decoded-json) - parent-path "" - process-token-type identity} - :as opts}] - (let [type-key (if (= json-format :json-format/dtcg) "$type" "type")] - (reduce-kv - (fn [unknown-tokens k v] - (let [child-path (if (empty? parent-path) - (name k) - (str parent-path "." k))] - (if (and (map? v) - (not (contains? v type-key))) - (let [nested-unknown-tokens (get-tokens-of-unknown-type v (assoc opts :parent-path child-path))] - (merge unknown-tokens nested-unknown-tokens)) - (let [token-type-str (get v type-key) - token-type (-> (cto/dtcg-token-type->token-type token-type-str) - (process-token-type))] - (if (and (not (some? token-type)) (some? token-type-str)) - (assoc unknown-tokens child-path token-type-str) - unknown-tokens))))) - nil - decoded-json))) + (export-dtcg-json tokens-lib tokens-status nil)) + ([tokens-lib tokens-status opts] + (when-not (empty-lib? tokens-lib) + (tsw/write (tokens-lib->document tokens-lib (status-activation tokens-lib tokens-status)) + opts)))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; SERIALIZATION (TRANSIT) diff --git a/common/test/common_tests/types/tokens_lib_test.cljc b/common/test/common_tests/types/tokens_lib_test.cljc index 0c06f7d9df..80d8119a1a 100644 --- a/common/test/common_tests/types/tokens_lib_test.cljc +++ b/common/test/common_tests/types/tokens_lib_test.cljc @@ -1893,6 +1893,226 @@ (t/is (= (:type original-ref) (:type imported-ref))) (t/is (= (:value imported-ref) (:value original-ref)))))))) +(t/deftest convert-dtcg-shadow-composite + (let [shadow {:offset-x "1" :offset-y "2" :blur "4" :spread "0" :color "#000000" :inset false}] + (t/testing "DTCG keys" + (t/is (= [shadow] + (ctob/convert-dtcg-shadow-composite + {"offsetX" "1" "offsetY" "2" "blur" "4" "spread" "0" "color" "#000000" "inset" "false"})))) + + (t/testing "legacy keys" + (t/is (= [(assoc shadow :inset true)] + (ctob/convert-dtcg-shadow-composite + [{"x" "1" "y" "2" "blur" "4" "spread" "0" "color" "#000000" "type" "innerShadow"}])))) + + (t/testing "a value already in penpot's format is kept" + (t/is (= [shadow] (ctob/convert-dtcg-shadow-composite [shadow]))) + (t/is (= [shadow] (ctob/convert-dtcg-shadow-composite shadow)))) + + (t/testing "references are kept" + (t/is (= "{shadow.sm}" (ctob/convert-dtcg-shadow-composite "{shadow.sm}")))))) + +(t/deftest import-warnings-for-dropped-data + (let [json {"core" {"color" {"$type" "color" + "$description" "Colors" + "old" {"$value" "#000" + "$deprecated" true + "$extensions" {"com.figma" {"scopes" ["ALL"]}}} + "lab" {"$value" {"colorSpace" "lab" "components" [50 20 30]}} + "oklch" {"$value" {"colorSpace" "oklch" "components" [0.628 0.2577 29.23]}}} + "motion" {"fast" {"$type" "duration" "$value" {"value" 100 "unit" "ms"}}} + "broken" {"$extends" "{missing}"}} + "$metadata" {"tokenSetOrder" ["core"]}} + {:keys [tokens-lib unknown-types unsupported]} (ctob/import-decoded-json json "")] + (t/testing "the tokens are imported without the dropped data" + (t/is (= "#000" (:value (ctob/get-token-by-name tokens-lib "core" "color.old"))))) + (t/testing "colors keep their color space" + (t/is (= "lab(50, 20, 30)" (:value (ctob/get-token-by-name tokens-lib "core" "color.lab")))) + (t/is (= "oklch(0.628, 0.2577, 29.23)" (:value (ctob/get-token-by-name tokens-lib "core" "color.oklch"))))) + (t/is (= {"core.motion.fast" "duration"} unknown-types)) + (t/is (= #{{:property "$deprecated" :path "core.color.old"} + {:property "$extensions" :path "core.color.old"} + {:property "$description" :path "core.color"} + {:property "$extends" :path "core.broken"}} + (set unsupported))))) + +(t/deftest import-warnings-for-dropped-tokens + (let [json {"core" {"$root" {"$value" "1" "$type" "number"} + "color" {"$type" "color" + "ok" {"$value" "#fff"} + "odd" {"$value" {"colorSpace" "nope" "components" [1 0 0]}} + "stray" 1} + "untyped" {"$value" "4"} + "a" {"$extends" "{b}"} + "b" {"$extends" "{a}"}} + "$metadata" {"tokenSetOrder" ["core"]}} + {:keys [tokens-lib unsupported]} (ctob/import-decoded-json json "")] + (t/testing "the tokens that can be read are imported" + (t/is (= #{"color.ok"} (set (keys (ctob/get-all-tokens-map tokens-lib)))))) + (t/is (= #{{:property "$root" :path "core.$root"} + {:property "$value" :path "core.color.odd"} + {:property "$value" :path "core.color.stray"} + {:property "$type" :path "core.untyped"} + {:property "$extends" :path "core.a"} + {:property "$extends" :path "core.b"}} + (set unsupported))))) + +(t/deftest import-warnings-for-group-extensions + (let [json {"core" {"color" {"$type" "color" + "$extensions" true + "primary" {"$value" "#fff" "$extensions" "figma"}} + "size" {"$type" "number" + "$extensions" {} + "sm" {"$value" "4"}}} + "$metadata" {"tokenSetOrder" ["core"]}} + {:keys [tokens-lib unsupported]} (ctob/import-decoded-json json "")] + (t/is (= "#fff" (:value (ctob/get-token-by-name tokens-lib "core" "color.primary")))) + (t/testing "a malformed group `$extensions` is reported, an empty one is not" + (t/is (= [{:property "$extensions" :path "core.color"}] unsupported))) + (t/testing "the reader drops a malformed token `$extensions` before penpot sees it" + (t/is (not-any? #(= "core.color.primary" (:path %)) unsupported))))) + +(t/deftest import-theme-set-statuses + (let [json {"core" {"a" {"$value" "1" "$type" "number"}} + "light" {"b" {"$value" "2" "$type" "number"}} + "dark" {"b" {"$value" "3" "$type" "number"}} + "$themes" [{"name" "light" "group" "mode" + "selectedTokenSets" {"core" "source" "light" "enabled" "dark" "disabled"}} + {"name" "dark" "group" "mode" + "selectedTokenSets" {"core" "source" "dark" "enabled"}}] + "$metadata" {"tokenSetOrder" ["core" "light" "dark"] + "activeThemes" ["mode/light" "mode/dark"]}} + lib (ctob/parse-decoded-json json "")] + (t/testing "source and enabled sets are part of the theme, disabled ones are not" + (t/is (= #{"core" "light"} (:sets (ctob/get-theme-by-name lib "mode" "light"))))) + (t/testing "activating a theme deactivates the other themes of its group" + (t/is (= #{"mode/dark"} (disj (ctob/get-legacy-active-themes lib) ctob/hidden-theme-path)))))) + +(t/deftest import-active-top-level-theme + (let [json {"core" {"a" {"$value" "1" "$type" "number"}} + "extra" {"b" {"$value" "2" "$type" "number"}} + "$themes" [{"name" "solo" "selectedTokenSets" {"core" "enabled"}}] + "$metadata" {"activeThemes" ["solo"] "activeSets" ["extra"]}} + lib (ctob/parse-decoded-json json "")] + (t/testing "a theme without group is active by its path, next to the hidden theme" + (t/is (= #{"/solo" ctob/hidden-theme-path} (ctob/get-legacy-active-themes lib))) + (t/is (= #{"extra"} (:sets (ctob/get-theme lib ctob/hidden-theme-id))))) + (t/testing "an export read back without encoding keeps what is active" + (let [lib' (ctob/parse-multi-set-dtcg-json (ctob/export-dtcg-json lib))] + (t/is (= (ctob/get-legacy-active-themes lib) (ctob/get-legacy-active-themes lib'))) + (t/is (= #{"core" "extra"} (:sets (ctob/get-theme lib' ctob/hidden-theme-id)))))))) + +(t/deftest export-default-profile-keeps-values-and-type-names + (let [tokens-lib (-> (ctob/make-tokens-lib) + (ctob/add-set (ctob/make-token-set + :name "core" + :tokens {"radius.sm" (ctob/make-token {:name "radius.sm" :type :border-radius :value "4"}) + "size.rem" (ctob/make-token {:name "size.rem" :type :sizing :value "1.5rem"}) + "weight.bold" (ctob/make-token {:name "weight.bold" :type :font-weight :value "700"}) + "color.brand" (ctob/make-token {:name "color.brand" :type :color :value "oklch(0.628, 0.2577, 29.23)"}) + "typo.body" (ctob/make-token {:name "typo.body" :type :typography + :value {:font-weight "700" :font-size "16px" :line-height "1.5"}})}))) + tokens-status (ctos/make-tokens-status) + result (ctob/export-dtcg-json tokens-lib tokens-status)] + (t/testing "the :default profile is the export without options" + (t/is (= result (ctob/export-dtcg-json tokens-lib tokens-status {:profile :default}))) + (t/is (= (ctob/export-dtcg-multi-file tokens-lib tokens-status) + (ctob/export-dtcg-multi-file tokens-lib tokens-status {:profile :default})))) + (t/testing "values stay strings and types keep their names" + (t/is (= {"radius" {"sm" {"$value" "4" "$type" "borderRadius" "$description" ""}} + "size" {"rem" {"$value" "1.5rem" "$type" "sizing" "$description" ""}} + "weight" {"bold" {"$value" "700" "$type" "fontWeights" "$description" ""}} + "color" {"brand" {"$value" "oklch(0.628, 0.2577, 29.23)" "$type" "color" "$description" ""}} + "typo" {"body" {"$value" {"fontWeights" "700" "fontSizes" "16px" "lineHeights" "1.5"} + "$type" "typography" + "$description" ""}}} + (get result "core")))))) + +(t/deftest export-dtcg-2025-values + (let [tokens-lib (-> (ctob/make-tokens-lib) + (ctob/add-set (ctob/make-token-set + :name "core" + :tokens {"radius.sm" (ctob/make-token {:name "radius.sm" :type :border-radius :value "4"}) + "size.rem" (ctob/make-token {:name "size.rem" :type :sizing :value "1.5rem"}) + "space.md" (ctob/make-token {:name "space.md" :type :spacing :value "{radius.sm} * 2"}) + "weight.bold" (ctob/make-token {:name "weight.bold" :type :font-weight :value "700"}) + "color.brand" (ctob/make-token {:name "color.brand" :type :color :value "oklch(0.628, 0.2577, 29.23)"}) + "shadow.sm" (ctob/make-token {:name "shadow.sm" :type :shadow + :value [{:offset-x "0" :offset-y "2px" :blur "4px" :spread "0" :color "#000000" :inset false}]})}))) + tokens-status (ctos/make-tokens-status) + result (ctob/export-dtcg-json tokens-lib tokens-status {:profile :dtcg})] + (t/testing "literals become DTCG values and types" + (t/is (= {"radius" {"sm" {"$value" {"value" 4 "unit" "px"} + "$type" "dimension" + "$description" ""}} + "size" {"rem" {"$value" {"value" 1.5 "unit" "rem"} + "$type" "dimension" + "$description" ""}} + "space" {"md" {"$value" "{radius.sm} * 2" + "$type" "dimension" + "$description" ""}} + "weight" {"bold" {"$value" 700 + "$type" "fontWeight" + "$description" ""}} + "color" {"brand" {"$value" {"colorSpace" "oklch" + "components" [0.628 0.2577 29.23] + "hex" "#ff0000"} + "$type" "color" + "$description" ""}} + "shadow" {"sm" {"$value" [{"offsetX" {"value" 0 "unit" "px"} + "offsetY" {"value" 2 "unit" "px"} + "blur" {"value" 4 "unit" "px"} + "spread" {"value" 0 "unit" "px"} + "color" {"colorSpace" "srgb" "components" [0 0 0] "hex" "#000000"} + "inset" false}] + "$type" "shadow" + "$description" ""}}} + (get result "core")))) + (t/testing "a multi-file export writes the same set" + (t/is (= (get result "core") + (get (ctob/export-dtcg-multi-file tokens-lib tokens-status {:profile :dtcg}) "core.json")))))) + +(t/deftest export-dtcg-2025-round-trip + (let [tokens-lib (-> (ctob/make-tokens-lib) + (ctob/add-set (ctob/make-token-set + :name "core" + :tokens {"space.sm" (ctob/make-token {:name "space.sm" :type :spacing :value "4px"}) + "space.md" (ctob/make-token {:name "space.md" :type :spacing :value "{space.sm} * 2" + :description "Twice the small space"}) + "size.rem" (ctob/make-token {:name "size.rem" :type :sizing :value "1.5rem"}) + "color.brand" (ctob/make-token {:name "color.brand" :type :color :value "#ff0000"}) + "color.oklch" (ctob/make-token {:name "color.oklch" :type :color :value "oklch(0.628, 0.2577, 29.23)"}) + "color.alpha" (ctob/make-token {:name "color.alpha" :type :color :value "#0000001a"}) + "family.body" (ctob/make-token {:name "family.body" :type :font-family :value ["Inter" "sans-serif"]}) + "typo.body" (ctob/make-token {:name "typo.body" :type :typography + :value {:font-weight "bold" :font-size "16px" + :letter-spacing "0.05em" :font-family ["Inter"]}}) + "typo.ref" (ctob/make-token {:name "typo.ref" :type :typography :value "{typo.body}"}) + "shadow.sm" (ctob/make-token {:name "shadow.sm" :type :shadow + :value [{:offset-x "0px" :offset-y "2px" :blur "4px" :spread "0px" + :color "#0000001a" :inset false}]})})) + (ctob/add-set (ctob/make-token-set :name "dark")) + (ctob/add-theme (ctob/make-token-theme :id (thi/new-id! :mode-light) + :name "light" + :group "mode" + :external-id "mode-light" + :sets #{"core"}))) + tokens-status (cfo/toggle-theme-active (ctos/make-tokens-status) tokens-lib (thi/id :mode-light)) + exported (ctob/export-dtcg-json tokens-lib tokens-status {:profile :dtcg}) + imported (ctob/parse-decoded-json exported "")] + (t/testing "penpot reads its own export back, spacing and sizing as dimensions" + (doseq [token (vals (ctob/get-tokens- (ctob/get-set-by-name tokens-lib "core")))] + (let [token' (ctob/get-token-by-name imported "core" (:name token))] + (t/is (= (get {:spacing :dimensions :sizing :dimensions} (:type token) (:type token)) + (:type token')) + (:name token)) + (t/is (= (:value token) (:value token')) (:name token)) + (t/is (= (:description token) (:description token')) (:name token))))) + (t/testing "sets, themes and what is active read back" + (t/is (= ["core" "dark"] (ctob/get-set-names imported))) + (t/is (= #{"core"} (:sets (ctob/get-theme-by-name imported "mode" "light")))) + (t/is (= #{"mode/light"} (disj (ctob/get-legacy-active-themes imported) ctob/hidden-theme-path)))))) + (t/deftest token-name-path-exists?-test (t/is (true? (ctob/token-name-path-exists? "border-radius" {"border-radius" {"sm" {:name "sm"}}}))) (t/is (true? (ctob/token-name-path-exists? "border-radius" {"border-radius" {:name "sm"}}))) diff --git a/frontend/deps.edn b/frontend/deps.edn index 3de812cb70..9ddd3104c5 100644 --- a/frontend/deps.edn +++ b/frontend/deps.edn @@ -30,11 +30,6 @@ :git/sha "d726717" :git/url "https://github.com/funcool/rumext.git"} - penpot/tokenscript-interpreter - {:git/tag "v0.2.0" - :git/sha "20db9b1" - :git/url "https://github.com/penpot/tokenscript-interpreter.git"} - instaparse/instaparse {:mvn/version "1.5.0"} garden/garden {:git/url "https://github.com/noprompt/garden" :git/sha "05590ecb5f6fa670856f3d1ab400aa4961047480"}} diff --git a/frontend/src/app/main/data/workspace/tokens/import_export.cljs b/frontend/src/app/main/data/workspace/tokens/import_export.cljs index 05b221f1c7..6b8e64e430 100644 --- a/frontend/src/app/main/data/workspace/tokens/import_export.cljs +++ b/frontend/src/app/main/data/workspace/tokens/import_export.cljs @@ -50,25 +50,44 @@ (reduce (fn [acc [k v]] (update acc v conj k)) {} m)) -(defn- show-unknown-types-warning [unknown-tokens] - (let [type->tokens (group-by-value unknown-tokens)] - (l/wrn :hint "unsupported token types found during import" - :tokens (str/join ", " (map (fn [[path type]] (str path " (" type ")")) unknown-tokens))) - (ntf/show {:content (tr "workspace.tokens.unknown-token-type-message") - :is-html true - :detail (->> (for [[token-type token-paths] type->tokens] - (str (tr "workspace.tokens.unknown-token-type-section" - (dom/escape-html token-type) - (tr "labels.warning-count" (i18n/c (count token-paths)))) - "")) - (str/join "")) - :type :toast - :level :info}))) +(defn- skipped-detail + "HTML lists of the skipped token paths, one section per label, with + `section-fn` called with the label and the count text." + [section-fn label->paths] + (->> (for [[label paths] (sort-by key label->paths)] + (str (section-fn (dom/escape-html label) + (tr "labels.warning-count" (i18n/c (count paths)))) + "")) + (str/join ""))) + +(defn- import-warnings-notification + "One toast for what the import skipped: tokens of types penpot does not + have, and token data it does not store (`$extensions`, `$deprecated`, + group descriptions, values it cannot read). Its details list both, + grouped by type and by property." + [unknown-types unsupported] + {:content (if (seq unsupported) + (tr "workspace.tokens.unsupported-token-data-message") + (tr "workspace.tokens.unknown-token-type-message")) + :is-html true + :detail (str (skipped-detail #(tr "workspace.tokens.unknown-token-type-section" %1 %2) + (group-by-value unknown-types)) + (skipped-detail #(tr "workspace.tokens.unsupported-token-data-section" %1 %2) + (update-vals (group-by :property unsupported) #(mapv :path %)))) + :type :toast + :level :info}) + +(defn- show-import-warnings + [unknown-types unsupported] + (l/wrn :hint "token data skipped during import" + :types (str/join ", " (map (fn [[path type]] (str path " (" type ")")) unknown-types)) + :data (str/join ", " (map (fn [{:keys [path property]}] (str path " (" property ")")) unsupported))) + (ntf/show (import-warnings-notification unknown-types unsupported))) (defn- decode-json [json-string] @@ -80,8 +99,7 @@ (defn- parse-decoded-json [decoded-json file-name] (try - {:tokens-lib (ctob/parse-decoded-json decoded-json file-name) - :unknown-tokens (ctob/get-tokens-of-unknown-type decoded-json {})} + (ctob/import-decoded-json decoded-json file-name) (catch js/Error e (let [err (or (extract-error-with-code e) (extract-name-error e) @@ -91,9 +109,9 @@ (defn- validate-library "Resolve tokens in the library and search for errors. Reference errors are ignored, since it can be resolved by the user in the UI. All the other errors are thrown as exceptions." - [{:keys [tokens-lib unknown-tokens]}] - (when unknown-tokens - (st/emit! (show-unknown-types-warning unknown-tokens))) + [{:keys [tokens-lib unknown-types unsupported]}] + (when (or unknown-types unsupported) + (st/emit! (show-import-warnings unknown-types unsupported))) (try (let [tokens-tree (ctob/get-all-tokens-map tokens-lib) resolved-tokens (if (contains? cf/flags :tokenscript) diff --git a/frontend/test/frontend_tests/plugins/tokens_test.cljs b/frontend/test/frontend_tests/plugins/tokens_test.cljs index 34c81dde12..2f0fc21855 100644 --- a/frontend/test/frontend_tests/plugins/tokens_test.cljs +++ b/frontend/test/frontend_tests/plugins/tokens_test.cljs @@ -897,6 +897,38 @@ (t/is (= token-id (:token-id @captured))) (t/is (= ["Inter" "Arial"] (get-in @captured [:attrs :value]))))))) +(t/deftest shadow-token-value-round-trips + (let [set-id (cthi/new-id! :token-set) + token-id (cthi/new-id! :shadow-token) + shadow [{:offset-x "1" :offset-y "2" :blur "4" :spread "0" :color "#000000" :inset false}] + file (-> (cthf/sample-file :file1 :page-label :page1) + (ctht/add-tokens-lib) + (ctht/update-tokens-lib + #(-> % + (ctob/add-set + (ctob/make-token-set :id set-id + :name "shadows")) + (ctob/add-token + set-id + (ctob/make-token :id token-id + :name "shadow.card" + :type :shadow + :value shadow))))) + store (ths/setup-store file) + _ (set! st/state store) + _ (set! st/stream (ptk/input-stream store)) + file-id (:id file) + captured (atom nil)] + (with-redefs [r/check-permission (constantly true) + u/check-editable-tokens (constantly nil) + dwtl/update-token (mock/stub (fn [_ _ attrs] + (reset! captured attrs) + :update-token)) + st/emit! mock/noop] + (let [token (ptok/token-proxy "plugin-id" file-id set-id token-id)] + (set! (.-value token) (.-value token)) + (t/is (= shadow (:value @captured))))))) + (t/deftest typography-token-resolved-value-is-plugin-array-shape (let [token (ctob/make-token {:name "type.body" diff --git a/frontend/test/frontend_tests/tokens/import_export_test.cljs b/frontend/test/frontend_tests/tokens/import_export_test.cljs index 982c6a7cae..315cd1c851 100644 --- a/frontend/test/frontend_tests/tokens/import_export_test.cljs +++ b/frontend/test/frontend_tests/tokens/import_export_test.cljs @@ -9,8 +9,59 @@ [app.common.json :as json] [app.common.types.tokens-lib :as ctob] [app.main.data.workspace.tokens.import-export :as dwti] + [app.util.i18n :as i18n] [beicon.v2.core :as rx] - [cljs.test :as t :include-macros true])) + [cljs.test :as t :include-macros true] + [clojure.string :as str])) + +(def ^:private import-warnings-notification @#'dwti/import-warnings-notification) + +;; Echoes the key and its arguments, so the test sees what went into each +;; string. Both arities, like `tr`: a call site compiled for one of them +;; looks up that arity on the replacement. +(defn- tr-stub + ([key] key) + ([key & args] + (str key "(" (str/join ", " (map #(if (i18n/c? %) @% %) args)) ")"))) + +(t/deftest import-warnings-notification-test + (with-redefs [i18n/tr tr-stub] + (let [unknown-types {"core.motion.fast" "duration" + "core.motion.slow" "duration" + "core.odd<1>" "x"} + unsupported [{:property "$extensions" :path "core.color.old"} + {:property "$extensions" :path "core.color"} + {:property "$value" :path "core.color."}]] + + (t/testing "only unknown types" + (let [{:keys [content detail]} (import-warnings-notification unknown-types nil)] + (t/is (= "workspace.tokens.unknown-token-type-message" content)) + (t/is (str/includes? detail "workspace.tokens.unknown-token-type-section(duration, labels.warning-count(2))")) + (t/is (str/includes? detail "core.motion.fast")) + (t/is (str/includes? detail "core.motion.slow")) + (t/is (not (str/includes? detail "workspace.tokens.unsupported-token-data-section"))))) + + (t/testing "only unsupported data" + (let [{:keys [content detail]} (import-warnings-notification nil unsupported)] + (t/is (= "workspace.tokens.unsupported-token-data-message" content)) + (t/is (str/includes? detail "workspace.tokens.unsupported-token-data-section($extensions, labels.warning-count(2))")) + (t/is (str/includes? detail "workspace.tokens.unsupported-token-data-section($value, labels.warning-count(1))")) + (t/is (str/includes? detail "core.color.old")) + (t/is (not (str/includes? detail "workspace.tokens.unknown-token-type-section"))))) + + (t/testing "both list their sections, and the unsupported data picks the message" + (let [{:keys [content detail]} (import-warnings-notification unknown-types unsupported)] + (t/is (= "workspace.tokens.unsupported-token-data-message" content)) + (t/is (str/includes? detail "workspace.tokens.unknown-token-type-section(duration, labels.warning-count(2))")) + (t/is (str/includes? detail "workspace.tokens.unsupported-token-data-section($extensions, labels.warning-count(2))")))) + + (t/testing "types and paths from the file are escaped" + (let [{:keys [detail]} (import-warnings-notification unknown-types unsupported)] + (t/is (str/includes? detail "workspace.tokens.unknown-token-type-section(x<y>, labels.warning-count(1))")) + (t/is (str/includes? detail "core.odd<1>")) + (t/is (str/includes? detail "core.color.<odd>")) + (t/is (not (str/includes? detail ""))) + (t/is (not (str/includes? detail "x")))))))) (t/deftest import-file-stream-test (t/async @@ -104,3 +155,24 @@ color.value tries to reference missing, which is not defined."))) (t/is (instance? ctob/TokensLib tokens-lib)) (t/is (= "{missing}" (:value (ctob/get-token-by-name tokens-lib "core" "color")))) (done)))))))) + +(t/deftest import-dtcg-2025-json-test + (t/async + done + (t/testing "imports DTCG 2025.10 object values" + (let [json (-> {"core" {"space" {"$type" "dimension" + "sm" {"$value" {"value" 4 "unit" "px"}}} + "color" {"$type" "color" + "brand" {"$value" {"colorSpace" "srgb" "components" [1 0 0]} + "$extensions" {"com.figma" {"scopes" ["ALL"]}}}}} + "$metadata" {"tokenSetOrder" ["core"]}} + (json/encode {:type :json-verbose}))] + (->> (rx/of json) + (dwti/import-file-stream "") + (rx/subs! (fn [tokens-lib] + (t/is (= "4px" (:value (ctob/get-token-by-name tokens-lib "core" "space.sm")))) + (t/is (= "srgb(1, 0, 0)" (:value (ctob/get-token-by-name tokens-lib "core" "color.brand")))) + (done)) + (fn [err] + (t/do-report {:type :error :message "Stream error" :actual err}) + (done)))))))) diff --git a/frontend/translations/en.po b/frontend/translations/en.po index b25f177276..75000006d3 100644 --- a/frontend/translations/en.po +++ b/frontend/translations/en.po @@ -9709,6 +9709,16 @@ msgstr "" msgid "workspace.tokens.unknown-token-type-section" msgstr "Type '%s' is not supported (%s):" +#: src/app/main/data/workspace/tokens/import_export.cljs:80 +msgid "workspace.tokens.unsupported-token-data-message" +msgstr "" +"Import was successful, but some token data was skipped because Penpot " +"does not support it. Expand details to see what was skipped." + +#: src/app/main/data/workspace/tokens/import_export.cljs:83 +msgid "workspace.tokens.unsupported-token-data-section" +msgstr "'%s' is not supported (%s):" + #: src/app/main/ui/workspace/tokens/management/forms/shadow.cljs:316, src/app/main/ui/workspace/tokens/management/forms/typography.cljs:270 msgid "workspace.tokens.use-reference" msgstr "Use a reference" diff --git a/frontend/translations/es.po b/frontend/translations/es.po index 9aed20a322..86ae3007c3 100644 --- a/frontend/translations/es.po +++ b/frontend/translations/es.po @@ -9811,6 +9811,17 @@ msgstr "" msgid "workspace.tokens.unknown-token-type-section" msgstr "El tipo '%s' no está soportado (%s):" +#: src/app/main/data/workspace/tokens/import_export.cljs:80 +msgid "workspace.tokens.unsupported-token-data-message" +msgstr "" +"La importación se ha realizado correctamente, pero algunos datos de los " +"tokens fueron omitidos porque Penpot no los soporta. Expande los detalles " +"para ver qué se omitió." + +#: src/app/main/data/workspace/tokens/import_export.cljs:83 +msgid "workspace.tokens.unsupported-token-data-section" +msgstr "'%s' no está soportado (%s):" + #: src/app/main/ui/workspace/tokens/management/forms/shadow.cljs:316, src/app/main/ui/workspace/tokens/management/forms/typography.cljs:270 msgid "workspace.tokens.use-reference" msgstr "Usa una referencia"