diff --git a/common/deps.edn b/common/deps.edn index 92b4188ce5..0120d04357 100644 --- a/common/deps.edn +++ b/common/deps.edn @@ -50,7 +50,8 @@ ;; Resolves token values, reads and writes token files. penpot/tokenscript-interpreter - {:git/sha "321061dbddaf78270319136eb41cfbff172fba02" + {:git/tag "v0.3.0" + :git/sha "040583ad348e215b78ad1e4075759a9b7ec52750" :git/url "https://github.com/penpot/tokenscript-interpreter.git"} com.sun.mail/jakarta.mail {:mvn/version "2.0.2"} diff --git a/common/src/app/common/files/tokens.cljc b/common/src/app/common/files/tokens.cljc index f8fb4e32aa..423d89ee64 100644 --- a/common/src/app/common/files/tokens.cljc +++ b/common/src/app/common/files/tokens.cljc @@ -785,6 +785,57 @@ (d/ordered-map) ordered-active))) +(defn- token-set-entries + "The tokens of `sets` as tokenscript takes them: `[[set-id tokens] ..]` + in the given order, later sets taking precedence." + [sets] + (mapv (fn [set] [(ctob/get-id set) (ctob/get-tokens- set)]) sets)) + +(defn get-active-token-sets + "The active sets unmerged (see `token-set-entries`), the sets + `get-tokens-in-active-sets` merges." + [tokens-status tokens-lib] + (assert (ctos/tokens-status? tokens-status) "expected valid tokens-status") + (assert (ctob/tokens-lib? tokens-lib) "expected valid tokens-lib") + (let [active-set-ids (ctos/get-active-set-ids tokens-status)] + (-> (filter #(contains? active-set-ids (ctob/get-id %)) (ctob/get-sets tokens-lib)) + (token-set-entries)))) + +(defn get-active-token-sets-force + "Same as `get-active-token-sets` but force-including a set by id. A set + that is not active comes first, with the lowest precedence, as in + `get-tokens-in-active-sets-force`." + [tokens-status tokens-lib force-set-id] + (assert (uuid? force-set-id) "expected valid set id") + (let [forced (when-not (contains? (ctos/get-active-set-ids tokens-status) force-set-id) + (ctob/get-set tokens-lib force-set-id))] + (cond->> (get-active-token-sets tokens-status tokens-lib) + forced (into (token-set-entries [forced]))))) + +(defn get-token-sets-for-edit + "Every set of the library unmerged (see `token-set-entries`), with + `set-id` moved last so its tokens take precedence: the sets + `get-all-tokens-map` merges with the set of a token being edited on + top." + [tokens-lib set-id] + (assert (ctob/tokens-lib? tokens-lib) "expected valid tokens-lib") + (let [sets (ctob/get-sets tokens-lib) + target (when set-id (d/seek #(= set-id (ctob/get-id %)) sets))] + (-> (cond-> (into [] (remove #(identical? % target)) sets) + target (conj target)) + (token-set-entries)))) + +(defn without-token + "`sets` (see `token-set-entries`) with the token `token-name` left out of + every set. A set without it stays the same object, so tokenscript skips + it as unchanged." + [sets token-name] + (mapv (fn [[set-id tokens :as entry]] + (if (contains? tokens token-name) + [set-id (dissoc tokens token-name)] + entry)) + sets)) + (defn sync-tokens-status-with-lib "Synchronizes tokens status with the current tokens lib: - Delete any theme or set that no longer exists in the lib. diff --git a/common/test/common_tests/files/tokens_test.cljc b/common/test/common_tests/files/tokens_test.cljc index 540a7b9625..4f2a00aae8 100644 --- a/common/test/common_tests/files/tokens_test.cljc +++ b/common/test/common_tests/files/tokens_test.cljc @@ -1243,6 +1243,77 @@ (t/is (= 1 (count tokens))) (t/is (= "8px" (:value (get tokens "spacing"))))))) +(defn- three-sets-lib + "A library with the sets `set-a`, `set-b` and `set-c`, in that order, + each defining `spacing`; `set-b` also defines `radius`." + [] + (-> (ctob/make-tokens-lib) + (ctob/add-set (ctob/make-token-set :id (thi/new-id! :set-a) :name "set-a")) + (ctob/add-set (ctob/make-token-set :id (thi/new-id! :set-b) :name "set-b")) + (ctob/add-set (ctob/make-token-set :id (thi/new-id! :set-c) :name "set-c")) + (ctob/add-token (thi/id :set-a) (ctob/make-token :name "spacing" :type :spacing :value "8px")) + (ctob/add-token (thi/id :set-b) (ctob/make-token :name "spacing" :type :spacing :value "16px")) + (ctob/add-token (thi/id :set-b) (ctob/make-token :name "radius" :type :border-radius :value "4px")) + (ctob/add-token (thi/id :set-c) (ctob/make-token :name "spacing" :type :spacing :value "32px")))) + +(t/deftest test-get-active-token-sets + (t/testing "returns the active sets unmerged, in set order" + (let [tokens-lib (three-sets-lib) + tokens-status (ctos/make-tokens-status :active-set-ids #{(thi/id :set-c) (thi/id :set-a)}) + sets (cfo/get-active-token-sets tokens-status tokens-lib)] + (t/is (= [(thi/id :set-a) (thi/id :set-c)] (mapv first sets))) + (t/is (identical? (ctob/get-tokens tokens-lib (thi/id :set-a)) (second (first sets))) + "the tokens are the set's own map, so an unchanged set is the same object"))) + + (t/testing "returns no sets when no sets are active" + (t/is (= [] (cfo/get-active-token-sets (ctos/make-tokens-status) (three-sets-lib)))))) + +(t/deftest test-get-active-token-sets-force + (t/testing "an inactive forced set comes first, with the lowest precedence" + (let [tokens-lib (three-sets-lib) + tokens-status (ctos/make-tokens-status :active-set-ids #{(thi/id :set-a)}) + sets (cfo/get-active-token-sets-force tokens-status tokens-lib (thi/id :set-b))] + (t/is (= [(thi/id :set-b) (thi/id :set-a)] (mapv first sets))))) + + (t/testing "an active forced set keeps its place and is not added twice" + (let [tokens-lib (three-sets-lib) + tokens-status (ctos/make-tokens-status :active-set-ids #{(thi/id :set-a) (thi/id :set-b)}) + sets (cfo/get-active-token-sets-force tokens-status tokens-lib (thi/id :set-b))] + (t/is (= [(thi/id :set-a) (thi/id :set-b)] (mapv first sets))))) + + (t/testing "an unknown forced set id adds nothing" + (let [tokens-lib (three-sets-lib) + tokens-status (ctos/make-tokens-status :active-set-ids #{(thi/id :set-a)}) + sets (cfo/get-active-token-sets-force tokens-status tokens-lib (uuid/next))] + (t/is (= [(thi/id :set-a)] (mapv first sets)))))) + +(t/deftest test-get-token-sets-for-edit + (t/testing "returns every set, with the edited set moved last" + (let [sets (cfo/get-token-sets-for-edit (three-sets-lib) (thi/id :set-a))] + (t/is (= [(thi/id :set-b) (thi/id :set-c) (thi/id :set-a)] (mapv first sets))))) + + (t/testing "keeps the set order without a set id" + (let [sets (cfo/get-token-sets-for-edit (three-sets-lib) nil)] + (t/is (= [(thi/id :set-a) (thi/id :set-b) (thi/id :set-c)] (mapv first sets))))) + + (t/testing "keeps the set order for an unknown set id" + (let [sets (cfo/get-token-sets-for-edit (three-sets-lib) (uuid/next))] + (t/is (= [(thi/id :set-a) (thi/id :set-b) (thi/id :set-c)] (mapv first sets)))))) + +(t/deftest test-without-token + (let [sets (cfo/get-token-sets-for-edit (three-sets-lib) nil)] + (t/testing "leaves the token out of every set that defines it" + (let [sets' (cfo/without-token sets "spacing")] + (t/is (= (mapv first sets) (mapv first sets'))) + (t/is (not-any? #(contains? (second %) "spacing") sets')) + (t/is (contains? (second (nth sets' 1)) "radius")))) + + (t/testing "keeps the sets without the token as the same objects" + (t/is (= [true false true] (mapv identical? sets (cfo/without-token sets "radius"))))) + + (t/testing "keeps every set when no set defines the token" + (t/is (every? true? (map identical? sets (cfo/without-token sets "missing"))))))) + (t/deftest test-sync-tokens-status-with-lib (t/testing "removes theme ids that no longer exist in the lib" (let [theme-1-id (thi/new-id! :theme-1) diff --git a/frontend/src/app/main/data/tokenscript.cljs b/frontend/src/app/main/data/tokenscript.cljs index 9a5a3a6b74..50cfaa32b9 100644 --- a/frontend/src/app/main/data/tokenscript.cljs +++ b/frontend/src/app/main/data/tokenscript.cljs @@ -13,7 +13,12 @@ numbers in pixels, colors as hex strings, composites as maps), `:resolved-symbol` (the tokenscript value, see `tokenscript.symbols`) and `:errors` (Penpot error maps, see - `app.main.data.workspace.tokens.errors`) added to each token." + `app.main.data.workspace.tokens.errors`) added to each token. + `resolve-sets` takes ordered token sets instead of a merged map. + + Resolution is incremental: the interpreter state of the last calls is + kept, and a call re-resolves and converts only the tokens that changed + since the previous one and the tokens depending on them." (:require [app.common.data :as d] [app.common.logging :as l] @@ -23,6 +28,7 @@ [cuerdas.core :as str] [tokenscript.core :as tsc] [tokenscript.errors :as tse] + [tokenscript.processor :as tsp] [tokenscript.symbols :as sym])) ;; Value conversion ----------------------------------------------------------- @@ -171,13 +177,13 @@ (defn- missing-reference-of "Missing token name behind an issue, if the root cause is a missing - token." + token or a token without a value." [{:keys [code data]}] (case code :processor/token-not-found (:token-name data) :interpreter/unknown-reference (:reference data) :processor/dependency-error (let [cause (:cause data)] - (when (= :processor/token-not-found (:code cause)) + (when (contains? #{:processor/token-not-found :penpot/empty-value} (:code cause)) (get-in cause [:data :token-name]))) nil)) @@ -192,6 +198,9 @@ message of their own are an invalid value." [{:keys [code data] :as issue} value] (cond + (= :penpot/empty-value code) + (wte/get-error-code :error.token/empty-input) + (circular? issue) (wte/error-with-value :error.token/circular-reference nil) @@ -275,26 +284,27 @@ (mapv quote-font-family value) value)) -(defn- token->input - "Token data for the interpreter. Types are passed by their DTCG names, - which the interpreter's bundled token types validate." - [{:keys [type value]}] - (cond-> {:value (case type - :font-family (quote-font-families value) - :typography (cond-> value - (map? value) (d/update-when :font-family quote-font-families)) - value)} - (some? type) (assoc :type (get cto/token-type->dtcg-token-type type (name type))))) +(defn- read-token + "Token reader for the interpreter: the value and the type of a Penpot + token. Types are passed by their DTCG names, which the interpreter's + bundled token types validate. A token without a value fails with + `:penpot/empty-value`, so the tokens referencing it fail too." + [{:keys [type value] token-name :name}] + (if (nil? value) + (tse/error-map :penpot/empty-value {:data {:token-name token-name}}) + (cond-> {:value (case type + :font-family (quote-font-families value) + :typography (cond-> value + (map? value) (d/update-when :font-family quote-font-families)) + value)} + (some? type) (assoc :type (get cto/token-type->dtcg-token-type type (name type)))))) -(defn- empty-value? - [token] - (nil? (:value token))) - -(defn- resolved-token - "Adds the result of the interpreter to `token`: `:resolved-value` and - `:resolved-symbol` for the resolved symbol `s`, or `:errors` for its - `issues`, a failing conversion or a value Penpot doesn't accept." - [token issues s] +(defn- write-token + "Token writer for the interpreter: `token` with `:resolved-value` and + `:resolved-symbol`, or `:errors`, from its resolved symbol `s` and its + `issues`. The interpreter keeps these in `:output` and runs the writer + only for the tokens an operation changed." + [token s issues] (if-let [errors (token-errors issues token)] (assoc token :errors (vec errors)) (let [{:keys [value error]} (convert-token-value token s)] @@ -304,57 +314,111 @@ :resolved-value value :resolved-symbol s))))) -(defn- resolve-tokens* - [tokens] - (let [tpoint (ct/tpoint-ms) - inputs (into {} - (comp (remove (fn [[_ token]] (empty-value? token))) - (map (fn [[name token]] [name (token->input token)]))) - tokens) - result (tsc/build inputs) - resolved (:tokens result) - issues (:issues result) - output (reduce-kv - (fn [acc name token] - (assoc acc name - (if (empty-value? token) - (assoc token :errors [(wte/get-error-code :error.token/empty-input)]) - (resolved-token token (get issues name) (get resolved name))))) - {} - tokens)] - (l/dbg :hint "tokenscript/resolve-tokens" :elapsed (tpoint)) - output)) +(defn- resolve-sets* + "Resolves `sets` starting from `prev`, the previous resolution (nil for + the first). The sets go to the interpreter as they are, with + `read-token` and `write-token`; it merges them, and from `prev` it + looks only into the sets that are not the same objects, re-resolves + only what changed and writes only those tokens into its `:output`." + [sets prev] + (let [tpoint (ct/tpoint-ms) + result (if prev + (tsp/set-sets (:state prev) sets) + (tsc/build sets {:token-reader read-token :token-writer write-token}))] + (l/dbg :hint "tokenscript/resolve-sets" :elapsed (tpoint) :changed (count (:changed result))) + {:sets sets + :state (:state result) + :output (:output result)})) -;; Several components resolve the same token maps on every change (the -;; sidebar, the forms and the propagation each compute the active tokens -;; separately), so the results of the last few maps are kept. Entries are -;; matched by identity first and by equality second: comparing two maps -;; of a few hundred tokens is far cheaper than resolving them again. +;; Several components resolve the same sets on every change (the sidebar, +;; the forms and the propagation each compute the active tokens +;; separately), and a form resolves a new map on every keystroke. The last +;; few resolutions are kept with their interpreter state: sets that are the +;; same objects as an entry's return its output, others are resolved from +;; the most recent entry, which re-resolves only what changed. The entries +;; are values, so a preview resolved from an entry leaves it as it is. (def ^:private cache-size 8) -(defonce ^:private results-cache (volatile! [])) +(defonce ^:private cache (volatile! [])) -(defn- cached-result - [tokens] - (let [entries @results-cache] - (or (some (fn [[t result]] (when (identical? t tokens) result)) entries) - (some (fn [[t result]] (when (= t tokens) result)) entries)))) +(defn- same-sets? + "Whether `a` and `b` hold the same set keys and the same tokens maps by + identity, the common case of a repeated call." + [a b] + (or (identical? a b) + (and (= (count a) (count b)) + (every? true? (map (fn [[ka ta] [kb tb]] (and (= ka kb) (identical? ta tb))) a b))))) -(defn- remember-result - [tokens result] - (vswap! results-cache (fn [entries] (vec (take cache-size (cons [tokens result] entries)))))) +(defn- cached-entry + [sets] + (some #(when (same-sets? (:sets %) sets) %) @cache)) + +(defn- remember + "Puts `entry` first, dropping the oldest beyond `cache-size`." + [entry] + (vswap! cache (fn [entries] + (into [entry] + (comp (remove #(identical? % entry)) (take (dec cache-size))) + entries))) + entry) + +(defn- base-entry + "Entry to resolve `sets` from: the most recent one with the same set ids + in the same order, so that only the changed sets are looked into, else + the most recent one sharing a set id. Nil when none does: resolving + from an unrelated state costs more than building from scratch." + [sets] + (let [entries @cache + ids (mapv first sets) + id-set (set ids)] + (or (some #(when (= ids (mapv first (:sets %))) %) entries) + (some #(when (some (fn [[k]] (contains? id-set k)) (:sets %)) %) entries)))) + +(defn resolve-sets + "Resolves the ordered token `sets`, `[[set-id tokens] ..]`, where `tokens` + maps token names to tokens and a name defined by several sets is taken + from the last one (see `cfo/get-active-token-sets`). Returns a map of the + names to their tokens with `:resolved-value` and `:resolved-symbol`, or + `:errors`, added. + + Resolutions are incremental: the interpreter state of the last few + calls is kept. A call looks only into the sets that are not the same + objects as before, re-resolves the tokens in them that are not the same + objects and the tokens referencing them, and converts only those. Every + other entry of the output is the previous object." + [sets] + (:output (remember (or (cached-entry sets) + (resolve-sets* sets (base-entry sets)))))) (defn resolve-tokens - "Resolves `tokens` (a map of token name to token). Returns the tokens - with `:resolved-value` or `:errors` added." + "Resolves `tokens` (a map of token name to token), as one set. Returns + the tokens with `:resolved-value` or `:errors` added. See `resolve-sets`." [tokens] - (or (cached-result tokens) - (let [result (resolve-tokens* tokens)] - (remember-result tokens result) - result))) + (if (empty? tokens) + {} + (resolve-sets [[nil tokens]]))) -(defn resolve-token - "Resolves `token` against `tokens` (the token replaces any token with - the same name). Returns the resolved token." - [tokens token] - (get (resolve-tokens (assoc tokens (:name token) token)) (:name token))) +;; A form resolves a draft on every keystroke. The last draft resolution is +;; kept apart from the cache above, so that typing does not push out the +;; resolutions the sidebar and the propagation reuse. +(defonce ^:private draft-cache (volatile! nil)) + +(defn resolve-draft + "Resolves `draft`, a token being edited in a form, on top of `sets` (see + `cfo/get-token-sets-for-edit`): it goes in a set of its own after them. + Called on every keystroke with the same `sets`, it re-resolves only the + draft and the tokens referencing it. Returns the resolved draft." + [sets draft] + (let [sets (conj (vec sets) [::draft {(:name draft) draft}]) + prev @draft-cache + entry (cond + (and prev (same-sets? (:sets prev) sets)) + prev + + (and prev (= (mapv first (:sets prev)) (mapv first sets))) + (resolve-sets* sets prev) + + :else + (resolve-sets* sets (or (cached-entry (pop sets)) (base-entry sets))))] + (vreset! draft-cache entry) + (get (:output entry) (:name draft)))) diff --git a/frontend/src/app/main/data/workspace/tokens/application.cljs b/frontend/src/app/main/data/workspace/tokens/application.cljs index cc76192a18..9a4f1dd932 100644 --- a/frontend/src/app/main/data/workspace/tokens/application.cljs +++ b/frontend/src/app/main/data/workspace/tokens/application.cljs @@ -710,14 +710,14 @@ (cond (ctt/typography-token-keys (:type token)) (set/union attributes-to-remove ctt/typography-keys) (ctt/typography-keys (:type token)) (set/union attributes-to-remove ctt/typography-token-keys) - :else attributes-to-remove)] - (when-let [tokens (let [tokens-lib (dsh/lookup-tokens-lib state) - tokens-status (dsh/lookup-tokens-status state)] - (when (and tokens-lib tokens-status) - (cfo/get-tokens-in-active-sets tokens-status tokens-lib)))] + :else attributes-to-remove) + + tokens-lib (dsh/lookup-tokens-lib state) + tokens-status (dsh/lookup-tokens-status state)] + (when (and tokens-lib tokens-status) (->> (if (contains? cf/flags :tokenscript) - (rx/of (ts/resolve-tokens tokens)) - (sd/resolve-tokens tokens)) + (rx/of (ts/resolve-sets (cfo/get-active-token-sets tokens-status tokens-lib))) + (sd/resolve-tokens (cfo/get-tokens-in-active-sets tokens-status tokens-lib))) (rx/mapcat (fn [resolved-tokens] (let [undo-id (js/Symbol) diff --git a/frontend/src/app/main/data/workspace/tokens/propagation.cljs b/frontend/src/app/main/data/workspace/tokens/propagation.cljs index aae7f17945..e41201905d 100644 --- a/frontend/src/app/main/data/workspace/tokens/propagation.cljs +++ b/frontend/src/app/main/data/workspace/tokens/propagation.cljs @@ -245,24 +245,23 @@ (let [tokens-lib (dsh/lookup-tokens-lib state) tokens-status (dsh/lookup-tokens-status state)] (when (and tokens-lib tokens-status) - (when-let [tokens-tree (cfo/get-tokens-in-active-sets tokens-status tokens-lib)] - (->> (if (contains? cf/flags :tokenscript) - (rx/of (ts/resolve-tokens tokens-tree)) - (sd/resolve-tokens tokens-tree)) - (rx/mapcat - (fn [sd-tokens] - (let [undo-id (js/Symbol)] - (rx/concat - (rx/of (dwu/start-undo-transaction undo-id :timeout false)) + (->> (if (contains? cf/flags :tokenscript) + (rx/of (ts/resolve-sets (cfo/get-active-token-sets tokens-status tokens-lib))) + (sd/resolve-tokens (cfo/get-tokens-in-active-sets tokens-status tokens-lib))) + (rx/mapcat + (fn [sd-tokens] + (let [undo-id (js/Symbol)] + (rx/concat + (rx/of (dwu/start-undo-transaction undo-id :timeout false)) - ;; FIXME: now the tokens propagations is done by accumulating the update-shapes - ;; into a single commit-changes. This is not really the best way, the token application - ;; should be done with a changes_builder and sending only one `commit-changes` instead - ;; of creating lots of `update-shapes`. - (rx/of (dwsh/update-shapes-buffer-start)) - (->> (propagate-tokens state sd-tokens scope) - (rx/catch #(rx/concat - (rx/of (dwsh/update-shapes-buffer-stop)) - (rx/throw %)))) - (rx/of (dwsh/update-shapes-buffer-stop)) - (rx/of (dwu/commit-undo-transaction undo-id)))))))))))))) + ;; FIXME: now the tokens propagations is done by accumulating the update-shapes + ;; into a single commit-changes. This is not really the best way, the token application + ;; should be done with a changes_builder and sending only one `commit-changes` instead + ;; of creating lots of `update-shapes`. + (rx/of (dwsh/update-shapes-buffer-start)) + (->> (propagate-tokens state sd-tokens scope) + (rx/catch #(rx/concat + (rx/of (dwsh/update-shapes-buffer-stop)) + (rx/throw %)))) + (rx/of (dwsh/update-shapes-buffer-stop)) + (rx/of (dwu/commit-undo-transaction undo-id))))))))))))) diff --git a/frontend/src/app/main/ui/workspace/sidebar.cljs b/frontend/src/app/main/ui/workspace/sidebar.cljs index 63d4aa53b6..3d05b4fbef 100644 --- a/frontend/src/app/main/ui/workspace/sidebar.cljs +++ b/frontend/src/app/main/ui/workspace/sidebar.cljs @@ -393,18 +393,25 @@ tokenscript? (contains? cf/flags :tokenscript) resolved-active-tokens - (sd/use-resolved-tokens* active-tokens) + (sd/use-resolved-tokens* (when-not tokenscript? active-tokens)) + ;; Tokenscript takes the sets unmerged and resolves what changed tokenscript-resolved-active-tokens - (mf/with-memo [active-tokens tokenscript?] - (when tokenscript? (ts/resolve-tokens active-tokens))) + (mf/with-memo [tokens-status tokens-lib tokenscript?] + (when tokenscript? + (if (and tokens-status tokens-lib) + (ts/resolve-sets (cfo/get-active-token-sets tokens-status tokens-lib)) + {}))) tokenscript-resolved-active-tokens-force-set - (mf/with-memo [active-tokens-force-set tokenscript?] - (when tokenscript? (ts/resolve-tokens active-tokens-force-set))) + (mf/with-memo [tokens-status tokens-lib selected-token-set-id tokenscript?] + (when tokenscript? + (if (and tokens-status tokens-lib selected-token-set-id) + (ts/resolve-sets (cfo/get-active-token-sets-force tokens-status tokens-lib selected-token-set-id)) + {}))) resolved-active-tokens-force-set - (sd/use-resolved-tokens* active-tokens-force-set)] + (sd/use-resolved-tokens* (when-not tokenscript? active-tokens-force-set))] [:> (mf/provider ctx/tokens-lib) {:value tokens-lib} [:> (mf/provider ctx/tokens-status) {:value tokens-status} diff --git a/frontend/src/app/main/ui/workspace/tokens/management/forms/controls/color_input.cljs b/frontend/src/app/main/ui/workspace/tokens/management/forms/controls/color_input.cljs index 115fbddd45..133ab45848 100644 --- a/frontend/src/app/main/ui/workspace/tokens/management/forms/controls/color_input.cljs +++ b/frontend/src/app/main/ui/workspace/tokens/management/forms/controls/color_input.cljs @@ -11,7 +11,6 @@ [app.common.data.macros :as dm] [app.common.files.tokens :as cfo] [app.common.types.color :as cl] - [app.common.types.token :as cto] [app.common.types.tokens-lib :as ctob] [app.config :as cf] [app.main.data.style-dictionary :as sd] @@ -57,34 +56,22 @@ ;; differ in how they persist the value within the form’s nested structure. (defn- resolve-value - [tokens prev-token token-name value] - (let [valid-token-name? - (and (string? token-name) - (re-matches cto/token-name-validation-regex token-name)) - - token - {:value value - :name (if (or (not valid-token-name?) (str/blank? token-name)) - "__PENPOT__TOKEN__NAME__PLACEHOLDER__" - token-name)} - - tokens - (-> tokens - ;; Remove previous token when renaming a token - (dissoc (:name prev-token)) - (update (:name token) #(ctob/make-token (merge % prev-token token))))] - ;; TODO: Review this when tokenscript is fully integrated. - (if (cfo/token-circular-reference? tokens (:name token)) - (rx/of {:error (wte/error-with-value :error.token/circular-reference nil)}) - (->> (if (contains? cf/flags :tokenscript) - (rx/of (ts/resolve-tokens tokens)) - (sd/resolve-tokens-interactive tokens)) - (rx/mapcat - (fn [resolved-tokens] - (let [{:keys [errors resolved-value]} (get resolved-tokens (:name token))] - (if resolved-value - (rx/of {:value resolved-value}) - (rx/of {:error (first errors)}))))))))) + "Resolves the value typed for the token being edited. With tokenscript + the draft is resolved against `token-sets` (see + `csu/use-edit-token-sets`), otherwise against the merged `tokens`." + [tokens token-sets prev-token token-name value] + (let [draft (csu/draft-token prev-token token-name value)] + (if (contains? cf/flags :tokenscript) + (rx/of (csu/resolved->result (ts/resolve-draft token-sets draft) value)) + (let [tokens + (-> tokens + ;; Remove previous token when renaming a token + (dissoc (:name prev-token)) + (update (:name draft) #(ctob/make-token (merge % draft))))] + (if (cfo/token-circular-reference? tokens (:name draft)) + (rx/of {:error (wte/error-with-value :error.token/circular-reference nil)}) + (->> (sd/resolve-tokens-interactive tokens) + (rx/map #(csu/resolved->result (get % (:name draft)) value)))))))) (defn- hex->color-obj [hex] @@ -145,6 +132,7 @@ (let [form (mf/use-ctx fc/context) input-name name token-name (get-in @form [:data :name] nil) + token-sets (csu/use-edit-token-sets (:name token)) touched? @@ -273,10 +261,10 @@ :else props)] - (mf/with-effect [resolve-stream tokens token input-name token-name] + (mf/with-effect [resolve-stream tokens token-sets token input-name token-name] (let [subs (->> resolve-stream (rx/debounce 300) - (rx/mapcat (partial resolve-value tokens token token-name)) + (rx/mapcat (partial resolve-value tokens token-sets token token-name)) (rx/map (fn [result] (d/update-when result :error wte/resolve-error-message))) (rx/subs! (fn [{:keys [error value]}] @@ -338,6 +326,7 @@ (let [form (mf/use-ctx fc/context) input-name name token-name (get-in @form [:data :name] nil) + token-sets (csu/use-edit-token-sets (:name token)) touched? (get-in @form [:touched :value value-subfield index input-name]) @@ -464,10 +453,10 @@ :hint-message (:message error)}) props)] - (mf/with-effect [resolve-stream tokens token input-name index value-subfield token-name] + (mf/with-effect [resolve-stream tokens token-sets token input-name index value-subfield token-name] (let [subs (->> resolve-stream (rx/debounce 300) - (rx/mapcat (partial resolve-value tokens token token-name)) + (rx/mapcat (partial resolve-value tokens token-sets token token-name)) (rx/map (fn [result] (d/update-when result :error wte/resolve-error-assoc-message))) diff --git a/frontend/src/app/main/ui/workspace/tokens/management/forms/controls/combobox.cljs b/frontend/src/app/main/ui/workspace/tokens/management/forms/controls/combobox.cljs index 730ced7b99..acce602470 100644 --- a/frontend/src/app/main/ui/workspace/tokens/management/forms/controls/combobox.cljs +++ b/frontend/src/app/main/ui/workspace/tokens/management/forms/controls/combobox.cljs @@ -9,7 +9,6 @@ (:require [app.common.data :as d] [app.common.files.tokens :as cfo] - [app.common.types.token :as cto] [app.common.types.tokens-lib :as ctob] [app.config :as cf] [app.main.data.style-dictionary :as sd] @@ -31,39 +30,25 @@ [app.util.i18n :refer [tr]] [app.util.object :as obj] [beicon.v2.core :as rx] - [cuerdas.core :as str] [rumext.v2 :as mf])) (defn- resolve-value - [tokens prev-token token-name value] - (let [valid-token-name? - (and (string? token-name) - (re-matches cto/token-name-validation-regex token-name)) - - token - {:value value - :name (if (or (not valid-token-name?) (str/blank? token-name)) - "__PENPOT__TOKEN__NAME__PLACEHOLDER__" - token-name)} - tokens - (-> tokens - ;; Remove previous token when renaming a token - (dissoc (:name prev-token)) - (update (:name token) #(ctob/make-token (merge % prev-token token))))] - ;; TODO: Review this when tokenscript is fully integrated. - (if (cfo/token-circular-reference? tokens (:name token)) - (rx/of {:error (wte/error-with-value :error.token/circular-reference nil)}) - (->> (if (contains? cf/flags :tokenscript) - (rx/of (ts/resolve-tokens tokens)) - (sd/resolve-tokens-interactive tokens)) - (rx/mapcat - (fn [resolved-tokens] - (let [{:keys [errors resolved-value]} (get resolved-tokens (:name token))] - (if resolved-value - (rx/of {:value resolved-value}) - (rx/of {:error (if errors - (first errors) - (wte/error-with-value :error/unknown value))}))))))))) + "Resolves the value typed for the token being edited. With tokenscript + the draft is resolved against `token-sets` (see + `csu/use-edit-token-sets`), otherwise against the merged `tokens`." + [tokens token-sets prev-token token-name value] + (let [draft (csu/draft-token prev-token token-name value)] + (if (contains? cf/flags :tokenscript) + (rx/of (csu/resolved->result (ts/resolve-draft token-sets draft) value)) + (let [tokens + (-> tokens + ;; Remove previous token when renaming a token + (dissoc (:name prev-token)) + (update (:name draft) #(ctob/make-token (merge % draft))))] + (if (cfo/token-circular-reference? tokens (:name draft)) + (rx/of {:error (wte/error-with-value :error.token/circular-reference nil)}) + (->> (sd/resolve-tokens-interactive tokens) + (rx/map #(csu/resolved->result (get % (:name draft)) value)))))))) (mf/defc value-combobox* [{:keys [name tokens token token-type empty-to-end ref] :rest props}] @@ -71,6 +56,7 @@ (let [form (mf/use-ctx fc/context) token-name (get-in @form [:data :name] nil) + token-sets (csu/use-edit-token-sets (:name token)) touched? (and (contains? (:data @form) name) (get-in @form [:touched name])) @@ -275,10 +261,10 @@ {:keys [style ready?]} (use-floating-dropdown is-open input-wrapper-ref wrapper-ref dropdown-ref)] - (mf/with-effect [resolve-stream tokens token name token-name] + (mf/with-effect [resolve-stream tokens token-sets token name token-name] (let [subs (->> resolve-stream (rx/debounce 300) - (rx/mapcat (partial resolve-value tokens token token-name)) + (rx/mapcat (partial resolve-value tokens token-sets token token-name)) (rx/map (fn [result] (d/update-when result :error (fn [error] diff --git a/frontend/src/app/main/ui/workspace/tokens/management/forms/controls/fonts_combobox.cljs b/frontend/src/app/main/ui/workspace/tokens/management/forms/controls/fonts_combobox.cljs index 8838adc844..938c6af69a 100644 --- a/frontend/src/app/main/ui/workspace/tokens/management/forms/controls/fonts_combobox.cljs +++ b/frontend/src/app/main/ui/workspace/tokens/management/forms/controls/fonts_combobox.cljs @@ -54,40 +54,42 @@ ;; validate data within the form state. (defn- resolve-value - [tokens prev-token _token-name value] + "Resolves the typed font families as a temporary font family token. + With tokenscript it is resolved against `token-sets` (see + `csu/use-edit-token-sets`), otherwise against the merged `tokens`." + [tokens token-sets prev-token _token-name value] (let [tmp-value (cto/split-font-family value) - tmp-name "__PENPOT__FONT_FAMILY__PLACEHOLDER__" - ;; Create a temporary font-family token to validate the value - token - {:name tmp-name - :type :font-family - :value (if (= (:type prev-token) :typography) - (assoc (:value prev-token) :font-family tmp-value) - tmp-value)} + ;; A temporary font-family token to validate the value + draft + (assoc prev-token + :name "__PENPOT__FONT_FAMILY__PLACEHOLDER__" + :type :font-family + :value (if (= (:type prev-token) :typography) + (assoc (:value prev-token) :font-family tmp-value) + tmp-value))] + (if (contains? cf/flags :tokenscript) + (rx/of (csu/resolved->result (ts/resolve-draft token-sets draft) value)) + (let [tokens (update tokens (:name draft) #(ctob/make-token (merge % draft)))] + (if (cfo/token-circular-reference? tokens (:name draft)) + (rx/of {:error (wte/error-with-value :error.token/circular-reference nil)}) + (->> (sd/resolve-tokens-interactive tokens) + (rx/map #(csu/resolved->result (get % (:name draft)) value)))))))) - tokens - (update tokens (:name token) #(ctob/make-token (merge % prev-token token)))] - ;; TODO: Review this when tokenscript is fully integrated. - (if (cfo/token-circular-reference? tokens (:name token)) - (rx/of {:error (wte/error-with-value :error.token/circular-reference nil)}) - (->> (if (contains? cf/flags :tokenscript) - (rx/of (ts/resolve-tokens tokens)) - (sd/resolve-tokens-interactive tokens)) - (rx/mapcat - (fn [resolved-tokens] - (let [{:keys [errors resolved-value]} (get resolved-tokens (:name token))] - (if resolved-value - (rx/of {:value resolved-value}) - (rx/of {:error (if errors - (first errors) - (wte/error-with-value :error/unknown value))}))))))))) +(defn- font-family->display + "Resolved font families as one list of names. Tokenscript resolves + them to a vector, StyleDictionary to a seq." + [value] + (if (sequential? value) + (cto/join-font-family value) + value)) (mf/defc fonts-combobox* [{:keys [token tokens name] :rest props}] (let [form (mf/use-ctx fc/context) input-name name token-name (get-in @form [:data :name] nil) + token-sets (csu/use-edit-token-sets nil) touched? (and (contains? (:data @form) input-name) @@ -167,10 +169,10 @@ :hint-message (:message error)}) props)] - (mf/with-effect [resolve-stream tokens token input-name token-name] + (mf/with-effect [resolve-stream tokens token-sets token input-name token-name] (let [subs (->> resolve-stream (rx/debounce 300) - (rx/mapcat (partial resolve-value tokens token token-name)) + (rx/mapcat (partial resolve-value tokens token-sets token token-name)) (rx/map (fn [result] (d/update-when result :error wte/resolve-error-assoc-message))) (rx/subs! (fn [{:keys [error value]}] @@ -180,7 +182,7 @@ (not touched?) (when (and (not error) (csu/show-hint-on-open? (get-in @form [:data input-name]) value)) - (reset! hint* {:message (tr "workspace.tokens.resolved-value" value) + (reset! hint* {:message (tr "workspace.tokens.resolved-value" (font-family->display value)) :type "hint"})) error @@ -192,7 +194,7 @@ (reset! hint* {:message error' :type "error"})))) :else - (let [message (tr "workspace.tokens.resolved-value" value)] + (let [message (tr "workspace.tokens.resolved-value" (font-family->display value))] (swap! form update :extra-errors dissoc input-name) (reset! hint* {:message message :type "hint"})))))))] (fn [] @@ -226,6 +228,7 @@ (let [form (mf/use-ctx fc/context) input-name name token-name (get-in @form [:data :name] nil) + token-sets (csu/use-edit-token-sets nil) error (get-in @form [:errors :value input-name]) @@ -301,10 +304,10 @@ :hint-message (:message error)}) props)] - (mf/with-effect [resolve-stream tokens token input-name token-name] + (mf/with-effect [resolve-stream tokens token-sets token input-name token-name] (let [subs (->> resolve-stream (rx/debounce 300) - (rx/mapcat (partial resolve-value tokens token token-name)) + (rx/mapcat (partial resolve-value tokens token-sets token token-name)) (rx/map (fn [result] (d/update-when result :error wte/resolve-error-message))) (rx/subs! @@ -327,7 +330,7 @@ (reset! hint* {:message error' :type "error"})))) :else - (let [message (tr "workspace.tokens.resolved-value" value) + (let [message (tr "workspace.tokens.resolved-value" (font-family->display value)) input-value (get-in @form [:data :value input-name] "")] (swap! form update :errors dissoc :value) (swap! form update :extra-errors dissoc :value) diff --git a/frontend/src/app/main/ui/workspace/tokens/management/forms/controls/input.cljs b/frontend/src/app/main/ui/workspace/tokens/management/forms/controls/input.cljs index 4a057d72bd..57393b5a9a 100644 --- a/frontend/src/app/main/ui/workspace/tokens/management/forms/controls/input.cljs +++ b/frontend/src/app/main/ui/workspace/tokens/management/forms/controls/input.cljs @@ -144,21 +144,11 @@ (defn- resolve-value-tokenscript - [tokens prev-token token-name value] - (let [valid-token-name? - (and (string? token-name) - (re-matches cto/token-name-validation-regex token-name)) - - token - (assoc prev-token - :value value - :name (if (or (not valid-token-name?) (str/blank? token-name)) - "__PENPOT__TOKEN__NAME__PLACEHOLDER__" - token-name)) - - {:keys [errors resolved-value]} - ;; Remove previous token when renaming a token - (ts/resolve-token (dissoc tokens (:name prev-token)) token)] + "Resolves the draft against `token-sets`, the library's sets without + the token being edited (see `csu/use-edit-token-sets`)." + [token-sets prev-token token-name value] + (let [draft (csu/draft-token prev-token token-name value) + {:keys [errors resolved-value]} (ts/resolve-draft token-sets draft)] (rx/of (if (seq errors) {:error (first errors)} @@ -217,6 +207,9 @@ (rx/behavior-subject (:value token)) (rx/subject))) + token-sets + (csu/use-edit-token-sets (:name token)) + hint* (mf/use-state {}) @@ -243,14 +236,14 @@ :hint-message (:message error)}) props)] - (mf/with-effect [resolve-stream tokens token input-name token-name] + (mf/with-effect [resolve-stream tokens token-sets token input-name token-name] (let [resolve-value (if (contains? cf/flags :tokenscript) - resolve-value-tokenscript - resolve-value) + (partial resolve-value-tokenscript token-sets token token-name) + (partial resolve-value tokens token token-name)) subs (->> resolve-stream (rx/debounce 300) - (rx/mapcat (partial resolve-value tokens token token-name)) + (rx/mapcat resolve-value) (rx/map (fn [result] (d/update-when result :error wte/resolve-error-message))) (rx/subs! (fn [{:keys [error value]}] diff --git a/frontend/src/app/main/ui/workspace/tokens/management/forms/controls/utils.cljs b/frontend/src/app/main/ui/workspace/tokens/management/forms/controls/utils.cljs index 4213b5d5d2..059e0a378d 100644 --- a/frontend/src/app/main/ui/workspace/tokens/management/forms/controls/utils.cljs +++ b/frontend/src/app/main/ui/workspace/tokens/management/forms/controls/utils.cljs @@ -1,10 +1,56 @@ (ns app.main.ui.workspace.tokens.management.forms.controls.utils (:require [app.common.data.macros :as dm] + [app.common.files.tokens :as cfo] [app.common.types.token :as cto] + [app.common.types.tokens-lib :as ctob] + [app.config :as cf] + [app.main.data.workspace.tokens.errors :as wte] [app.main.data.workspace.tokens.format :as dwtf] + [app.main.refs :as refs] [app.util.i18n :refer [tr]] - [cuerdas.core :as str])) + [cuerdas.core :as str] + [rumext.v2 :as mf])) + +(defn use-edit-token-sets + "Token sets a token being edited in the selected set resolves against + with tokenscript, unmerged (see `cfo/get-token-sets-for-edit`). + `excluded-name`, the name of the token being edited, is left out, so a + renamed draft doesn't leave the old name resolving and references to + the old name break as they will once it is saved. The sets stay the + same objects while the library does not change, so each keystroke + re-resolves only the draft. Nil without the tokenscript flag." + [excluded-name] + (let [tokens-lib (mf/deref refs/tokens-lib) + set-id (mf/deref refs/selected-token-set-id)] + (mf/with-memo [tokens-lib set-id excluded-name] + (when (and tokens-lib (contains? cf/flags :tokenscript)) + (cond-> (cfo/get-token-sets-for-edit tokens-lib + (or set-id (some-> (ctob/get-sets tokens-lib) first ctob/get-id))) + excluded-name (cfo/without-token excluded-name)))))) + +(defn resolved->result + "The `{:value ..}` or `{:error ..}` a form control shows for the + resolved draft, an unknown error when it has neither a value nor + errors." + [{:keys [errors resolved-value]} value] + (if resolved-value + {:value resolved-value} + {:error (if errors + (first errors) + (wte/error-with-value :error/unknown value))})) + +(defn draft-token + "The token being edited as the form has it: `prev-token` with `value`, + named `token-name`, or a placeholder name while that is not a valid + token name." + [prev-token token-name value] + (let [valid-name? (and (string? token-name) + (not (str/blank? token-name)) + (re-matches cto/token-name-validation-regex token-name))] + (assoc prev-token + :name (if valid-name? token-name "__PENPOT__TOKEN__NAME__PLACEHOLDER__") + :value value))) (defn- token->dropdown-option [token] diff --git a/frontend/src/app/main/ui/workspace/tokens/management/forms/generic_form.cljs b/frontend/src/app/main/ui/workspace/tokens/management/forms/generic_form.cljs index 66b0bf93b7..90ca16d9ad 100644 --- a/frontend/src/app/main/ui/workspace/tokens/management/forms/generic_form.cljs +++ b/frontend/src/app/main/ui/workspace/tokens/management/forms/generic_form.cljs @@ -119,12 +119,18 @@ ;; A map name-> token with resolved-values, resolved with style dictionary resolved-active-tokens - (sd/use-resolved-tokens* tokens-tree) + (sd/use-resolved-tokens* (when-not tokenscript? tokens-tree)) + + tokens-lib (mf/deref refs/tokens-lib) ;; A map name-> token with resolved-values, resolved with tokescript + ;; from the library's sets unmerged tokenscript-resolved-active-tokens - (mf/with-memo [tokens-tree tokenscript?] - (when tokenscript? (ts/resolve-tokens tokens-tree))) + (mf/with-memo [tokens-lib tokenscript?] + (when tokenscript? + (if tokens-lib + (ts/resolve-sets (cfo/get-token-sets-for-edit tokens-lib nil)) + {}))) ;; A map which keys are token types and values are vectors of tokens of that type, with resolved values. active-tokens-by-type diff --git a/frontend/src/app/plugins/tokens.cljs b/frontend/src/app/plugins/tokens.cljs index 62573c232f..dc471e04f0 100644 --- a/frontend/src/app/plugins/tokens.cljs +++ b/frontend/src/app/plugins/tokens.cljs @@ -148,43 +148,33 @@ :shadow (ctob/convert-dtcg-shadow-composite value) value)) -(defn- resolution-tokens - "Tokens used to resolve references for a token in `set-id`: every token in - the library, with the tokens of `set-id` taking precedence." - [tokens-lib set-id] - (merge (ctob/get-all-tokens-map tokens-lib) - (ctob/get-tokens tokens-lib set-id))) - (defn- resolved-without-errors? [resolved] (and (contains? resolved :resolved-value) (empty? (:errors resolved)))) (defn- valid-token-candidate? + "Whether `token` of `set-id` with `attrs` is valid. The candidate is + resolved as a draft against every set of the library with `set-id` on + top, without the token's current name, like the token forms do." [file-id set-id token attrs] (let [tokens-lib (u/locate-tokens-lib file-id) candidate (merge (datafy token) attrs) - tokens (-> (resolution-tokens tokens-lib set-id) - (dissoc (:name token)) - (assoc (:name candidate) candidate)) - resolved (get (ts/resolve-tokens tokens) (:name candidate))] + sets (-> (cfo/get-token-sets-for-edit tokens-lib set-id) + (cfo/without-token (:name token))) + resolved (ts/resolve-draft sets candidate)] (and (sm/validate (token-name-schema file-id set-id token) (:name candidate)) (sm/validate (cfo/make-token-value-schema (:type candidate)) (:value candidate)) (or (nil? (:description candidate)) (sm/validate cfo/schema:token-description (:description candidate))) (resolved-without-errors? resolved)))) -;; Last resolution result, reused while the tokens library and set are unchanged -(def ^:private resolution-cache (atom nil)) - (defn- resolve-set-tokens + "Tokens of the library resolved for `set-id`: every set of the library + with `set-id` on top. Repeated calls with an unchanged library are + answered from the resolver's cache." [tokens-lib set-id] - (let [{:keys [lib set resolved]} @resolution-cache] - (if (and (identical? lib tokens-lib) (= set set-id)) - resolved - (let [resolved (ts/resolve-tokens (resolution-tokens tokens-lib set-id))] - (reset! resolution-cache {:lib tokens-lib :set set-id :resolved resolved}) - resolved)))) + (ts/resolve-sets (cfo/get-token-sets-for-edit tokens-lib set-id))) (defn valid-token-resolution? "Checks that an existing token resolves without errors, which is the only @@ -280,8 +270,8 @@ (into-array (map shadow-entry->js entries)))) (defn- get-resolved-value - [token tokens-tree] - (let [resolved-tokens (ts/resolve-tokens tokens-tree) + [token sets] + (let [resolved-tokens (ts/resolve-sets sets) resolved-token (get resolved-tokens (:name token)) resolved-value (:resolved-value resolved-token)] (cond @@ -385,9 +375,8 @@ (fn [_] (let [token (u/locate-token file-id set-id id) tokens-lib (u/locate-tokens-lib file-id) - tokens-status (u/locate-tokens-status file-id) - tokens-tree (cfo/get-tokens-in-active-sets tokens-status tokens-lib)] - (get-resolved-value token tokens-tree)))} + tokens-status (u/locate-tokens-status file-id)] + (get-resolved-value token (cfo/get-active-token-sets tokens-status tokens-lib))))} :resolvedValueString {:this true @@ -396,9 +385,8 @@ (fn [_] (let [token (u/locate-token file-id set-id id) tokens-lib (u/locate-tokens-lib file-id) - tokens-status (u/locate-tokens-status file-id) - tokens-tree (cfo/get-tokens-in-active-sets tokens-status tokens-lib)] - (str (get-resolved-value token tokens-tree))))} + tokens-status (u/locate-tokens-status file-id)] + (str (get-resolved-value token (cfo/get-active-token-sets tokens-status tokens-lib)))))} :description {:this true diff --git a/frontend/test/frontend_tests/plugins/tokens_test.cljs b/frontend/test/frontend_tests/plugins/tokens_test.cljs index 2f0fc21855..bf87a304a5 100644 --- a/frontend/test/frontend_tests/plugins/tokens_test.cljs +++ b/frontend/test/frontend_tests/plugins/tokens_test.cljs @@ -929,6 +929,67 @@ (set! (.-value token) (.-value token)) (t/is (= shadow (:value @captured))))))) +(t/deftest token-setters-validate-the-candidate-resolution + ;; The candidate resolves against the library without the token's + ;; current name, so references to it can't validate it. + (let [set-id (cthi/new-id! :token-set) + size-id (cthi/new-id! :size-token) + double-id (cthi/new-id! :double-token) + file (-> (cthf/sample-file :file1 :page-label :page1) + (ctht/add-tokens-lib) + (ctht/update-tokens-lib + #(-> % + (ctob/add-set (ctob/make-token-set :id set-id :name "core")) + (ctob/add-token set-id (ctob/make-token :id size-id :name "size" :type :sizing :value "4")) + (ctob/add-token set-id (ctob/make-token :id double-id :name "double" :type :sizing :value "{size} * 2"))))) + store (ths/setup-store file) + _ (set! st/state store) + _ (set! st/stream (ptk/input-stream store)) + file-id (:id file) + updates (atom []) + invalid (atom [])] + (with-redefs [r/check-permission (constantly true) + u/check-editable-tokens (constantly nil) + u/not-valid (fn [_ code value] (swap! invalid conj [code value])) + dwtl/update-token (mock/stub (fn [_ _ attrs] (swap! updates conj attrs) {:attrs attrs})) + st/emit! mock/noop] + (let [size (ptok/token-proxy "plugin-id" file-id set-id size-id) + double (ptok/token-proxy "plugin-id" file-id set-id double-id)] + (set! (.-value double) "{size} * 3") + (set! (.-value double) "{missing} * 2") + (set! (.-value double) "{double} * 2") + (set! (.-name size) "space") + (t/is (= [{:value "{size} * 3"} {:name "space"}] @updates)) + (t/is (= [[:value "{missing} * 2"] [:value "{double} * 2"]] @invalid)))))) + +(t/deftest token-name-setter-accepts-a-name-below-the-old-name + ;; The candidate resolves without the token's current name: with the + ;; old `size` left in, `size.small` would be a token below another + ;; token, a name collision. + (let [set-id (cthi/new-id! :token-set) + size-id (cthi/new-id! :size-token) + file (-> (cthf/sample-file :file1 :page-label :page1) + (ctht/add-tokens-lib) + (ctht/update-tokens-lib + #(-> % + (ctob/add-set (ctob/make-token-set :id set-id :name "core")) + (ctob/add-token set-id (ctob/make-token :id size-id :name "size" :type :sizing :value "4"))))) + store (ths/setup-store file) + _ (set! st/state store) + _ (set! st/stream (ptk/input-stream store)) + file-id (:id file) + updates (atom []) + invalid (atom [])] + (with-redefs [r/check-permission (constantly true) + u/check-editable-tokens (constantly nil) + u/not-valid (fn [_ code value] (swap! invalid conj [code value])) + dwtl/update-token (mock/stub (fn [_ _ attrs] (swap! updates conj attrs) {:attrs attrs})) + st/emit! mock/noop] + (let [size (ptok/token-proxy "plugin-id" file-id set-id size-id)] + (set! (.-name size) "size.small") + (t/is (= [{:name "size.small"}] @updates)) + (t/is (= [] @invalid)))))) + (t/deftest typography-token-resolved-value-is-plugin-array-shape (let [token (ctob/make-token {:name "type.body" @@ -940,7 +1001,7 @@ :letter-spacing "1" :text-case "uppercase" :text-decoration "underline"}}) - result (get-resolved-value token {(:name token) token}) + result (get-resolved-value token [[nil {(:name token) token}]]) entry (aget result 0)] (t/is (array? result)) (t/is (= ["Inter" "Arial"] (vec (aget entry "fontFamilies")))) @@ -960,7 +1021,7 @@ :spread "4px" :color "#000000" :inset false}]}) - result (get-resolved-value token {(:name token) token}) + result (get-resolved-value token [[nil {(:name token) token}]]) entry (aget result 0)] (t/is (array? result)) (t/is (= 1 (aget entry "offsetX"))) @@ -973,7 +1034,7 @@ {:name "font.primary" :type :font-family :value ["Inter" "Arial"]}) - result (get-resolved-value token {(:name token) token})] + result (get-resolved-value token [[nil {(:name token) token}]])] (t/is (array? result)) (t/is (= ["Inter" "Arial"] (vec result))))) @@ -984,7 +1045,7 @@ {:name "font.body" :type :font-family :value ["Hanken Grotesk" "IBM Plex Mono"]}) - result (get-resolved-value token {(:name token) token})] + result (get-resolved-value token [[nil {(:name token) token}]])] (t/is (array? result)) (t/is (= ["Hanken Grotesk" "IBM Plex Mono"] (vec result))))) @@ -994,7 +1055,7 @@ :type :typography :value {:font-family ["Hanken Grotesk" "Arial"] :font-size "16px"}}) - result (get-resolved-value token {(:name token) token}) + result (get-resolved-value token [[nil {(:name token) token}]]) entry (aget result 0)] (t/is (= ["Hanken Grotesk" "Arial"] (vec (aget entry "fontFamilies")))))) @@ -1007,7 +1068,7 @@ {:name "font.display" :type :font-family :value families}) - result (get-resolved-value token {(:name token) token})] + result (get-resolved-value token [[nil {(:name token) token}]])] (t/is (= families (vec result))))) (t/deftest font-family-token-resolved-value-mixes-references-and-names @@ -1019,8 +1080,8 @@ {:name "font.stack" :type :font-family :value ["{font.base}" "Rock 3D"]}) - result (get-resolved-value token {(:name base) base - (:name token) token})] + result (get-resolved-value token [[nil {(:name base) base + (:name token) token}]])] (t/is (= ["Red Hat Text" "Rock 3D"] (vec result))))) (t/deftest font-family-token-resolved-value-keeps-quoted-names @@ -1028,7 +1089,7 @@ {:name "font.quoted" :type :font-family :value ["\"Red Hat Mono\"" "'Black Ops One'"]}) - result (get-resolved-value token {(:name token) token})] + result (get-resolved-value token [[nil {(:name token) token}]])] (t/is (= ["Red Hat Mono" "Black Ops One"] (vec result))))) (t/deftest typography-token-resolved-value-keeps-names-with-expression-words @@ -1037,7 +1098,7 @@ :type :typography :value {:font-family ["Red Hat Display" "Press Start 2P"] :font-size "16px"}}) - result (get-resolved-value token {(:name token) token}) + result (get-resolved-value token [[nil {(:name token) token}]]) entry (aget result 0)] (t/is (= ["Red Hat Display" "Press Start 2P"] (vec (aget entry "fontFamilies")))))) diff --git a/frontend/test/frontend_tests/tokens/tokenscript_test.cljs b/frontend/test/frontend_tests/tokens/tokenscript_test.cljs index 22ee6bf712..410739d783 100644 --- a/frontend/test/frontend_tests/tokens/tokenscript_test.cljs +++ b/frontend/test/frontend_tests/tokens/tokenscript_test.cljs @@ -6,9 +6,14 @@ (ns frontend-tests.tokens.tokenscript-test (:require + [app.common.files.tokens :as cfo] [app.common.test-helpers.ids-map :as cthi] + [app.common.types.token :as cto] [app.common.types.tokens-lib :as ctob] + [app.common.types.tokens-status :as ctos] + [app.main.data.style-dictionary :as sd] [app.main.data.tokenscript :as ts] + [beicon.v2.core :as rx] [cljs.test :as t :include-macros true])) (defn- tokens-map @@ -101,8 +106,8 @@ (t/is (= {:error/code :error.token/name-collision :error/value "a.b.c"} (error "a.b.c"))) (t/is (= {:error/code :error.style-dictionary/missing-reference :error/value ["a.b"]} (error "c")))) (t/testing "a new token below an existing one collides" - (let [tokens (tokens-map {:name "rad1" :value "8px" :type :border-radius}) - resolved (ts/resolve-token tokens {:name "rad1.bad" :value "10" :type :border-radius})] + (let [sets [[:core (tokens-map {:name "rad1" :value "8px" :type :border-radius})]] + resolved (ts/resolve-draft sets {:name "rad1.bad" :value "10" :type :border-radius})] (t/is (= :error.token/name-collision (get-in resolved [:errors 0 :error/code])))))) (t/deftest resolve-tokens-composites-test @@ -187,7 +192,244 @@ (t/is (= :error.token/empty-input (get-in resolved ["typography.empty" :errors 0 :error/code]))) (t/is (= 12 (get-in resolved ["borderRadius.sm" :resolved-value]))))) -(t/deftest resolve-token-test - (let [tokens (tokens-map {:name "a" :value "8px" :type :border-radius}) - resolved (ts/resolve-token tokens {:name "b" :value "{a} * 2" :type :border-radius})] +(t/deftest resolve-draft-test + (let [sets [[:core (tokens-map {:name "a" :value "8px" :type :border-radius})]] + resolved (ts/resolve-draft sets {:name "b" :value "{a} * 2" :type :border-radius})] (t/is (= 16 (:resolved-value resolved))))) + +;; Incremental resolution ------------------------------------------------------ +;; +;; The adapter keeps the interpreter state of the last resolutions. A new +;; map re-resolves only the tokens that changed and their dependents; the +;; output entries of every other token are the previous objects. + +(defn- token + [attrs] + (ctob/make-token attrs)) + +(defn- by-name + [& tokens] + (into {} (map (fn [t] [(:name t) t])) tokens)) + +(t/deftest resolve-tokens-incremental-test + (let [base (token {:name "base" :value "8" :type :sizing}) + double (token {:name "double" :value "{base} * 2" :type :sizing}) + other (token {:name "other" :value "1" :type :sizing}) + first-run (ts/resolve-tokens (by-name base double other)) + edited (assoc base :value "10") + edited-tokens (by-name edited double other) + second-run (ts/resolve-tokens edited-tokens)] + (t/is (= 16 (get-in first-run ["double" :resolved-value]))) + (t/is (= 10 (get-in second-run ["base" :resolved-value]))) + (t/is (= 20 (get-in second-run ["double" :resolved-value])) "dependents are re-resolved") + (t/is (identical? (get first-run "other") (get second-run "other")) + "tokens the edit did not reach keep their output object") + (t/is (not (identical? (get first-run "double") (get second-run "double")))) + (t/is (identical? second-run (ts/resolve-tokens edited-tokens)) + "the same input returns the cached output"))) + +(t/deftest resolve-tokens-incremental-errors-test + (t/testing "a token fixed by an edit and the tokens waiting for it recover" + (let [ref (token {:name "ref" :value "{missing}" :type :sizing}) + first-run (ts/resolve-tokens (by-name ref)) + missing (token {:name "missing" :value "4" :type :sizing}) + second-run (ts/resolve-tokens (by-name ref missing)) + third-run (ts/resolve-tokens (by-name ref))] + (t/is (= :error.style-dictionary/missing-reference (get-in first-run ["ref" :errors 0 :error/code]))) + (t/is (= 4 (get-in second-run ["ref" :resolved-value]))) + (t/is (= :error.style-dictionary/missing-reference (get-in third-run ["ref" :errors 0 :error/code])) + "deleting the token breaks the reference again"))) + (t/testing "a token that loses its value is an empty input and its references break" + (let [a (token {:name "a" :value "8" :type :sizing}) + b (token {:name "b" :value "{a}" :type :sizing}) + first-run (ts/resolve-tokens (by-name a b)) + second-run (ts/resolve-tokens (by-name (assoc a :value nil) b))] + (t/is (= 8 (get-in first-run ["b" :resolved-value]))) + (t/is (= :error.token/empty-input (get-in second-run ["a" :errors 0 :error/code]))) + (t/is (= :error.style-dictionary/missing-reference (get-in second-run ["b" :errors 0 :error/code])))))) + +(t/deftest resolve-sets-test + (let [core-base (token {:name "base" :value "8" :type :sizing}) + double (token {:name "double" :value "{base} * 2" :type :sizing}) + other (token {:name "other" :value "1" :type :sizing}) + theme-base (token {:name "base" :value "10" :type :sizing}) + core (by-name core-base double other) + theme (by-name theme-base) + resolved (ts/resolve-sets [[:core core] [:theme theme]])] + (t/testing "a later set wins and the result holds the winning tokens" + (t/is (= 10 (get-in resolved ["base" :resolved-value]))) + (t/is (= (:id theme-base) (get-in resolved ["base" :id])) "the output token is the winning record") + (t/is (= 20 (get-in resolved ["double" :resolved-value])))) + (t/testing "editing a token a later set overrides changes nothing visible" + (let [resolved' (ts/resolve-sets [[:core (by-name (assoc core-base :value "99") double other)] [:theme theme]])] + (t/is (= 10 (get-in resolved' ["base" :resolved-value]))) + (t/is (identical? (get resolved "double") (get resolved' "double"))))) + (t/testing "activating a set in the middle re-resolves the names it wins and their dependents" + (let [mid (by-name (token {:name "other" :value "5" :type :sizing}) + (token {:name "new" :value "{other} + 1" :type :sizing})) + resolved' (ts/resolve-sets [[:core core] [:mid mid] [:theme theme]])] + (t/is (= 5 (get-in resolved' ["other" :resolved-value]))) + (t/is (= 6 (get-in resolved' ["new" :resolved-value]))) + (t/is (identical? (get resolved "base") (get resolved' "base"))) + (t/is (identical? (get resolved "double") (get resolved' "double"))))) + (t/testing "an empty value in a later set hides the earlier definition" + (let [resolved' (ts/resolve-sets [[:core core] [:theme (by-name (assoc theme-base :value nil))]])] + (t/is (= :error.token/empty-input (get-in resolved' ["base" :errors 0 :error/code]))) + (t/is (= :error.style-dictionary/missing-reference (get-in resolved' ["double" :errors 0 :error/code]))))) + (t/testing "a preview against the same sets does not change their cached result" + (let [preview (ts/resolve-sets [[:core core] [:theme theme] [:draft (by-name (assoc core-base :value "1"))]])] + (t/is (= 2 (get-in preview ["double" :resolved-value]))) + (t/is (identical? resolved (ts/resolve-sets [[:core core] [:theme theme]]))))))) + +;; The active sets go to the resolver unmerged; the result is the one of +;; the merged active tokens. +(t/deftest resolve-active-sets-test + (let [lib (-> (ctob/make-tokens-lib) + (ctob/add-set (ctob/make-token-set :id (cthi/new-id! :core) :name "core")) + (ctob/add-set (ctob/make-token-set :id (cthi/new-id! :dark) :name "dark")) + (ctob/add-set (ctob/make-token-set :id (cthi/new-id! :extra) :name "extra")) + (ctob/add-token (cthi/id :core) (ctob/make-token {:name "base" :value "8" :type :sizing})) + (ctob/add-token (cthi/id :core) (ctob/make-token {:name "double" :value "{base} * 2" :type :sizing})) + (ctob/add-token (cthi/id :dark) (ctob/make-token {:name "base" :value "10" :type :sizing})) + (ctob/add-token (cthi/id :extra) (ctob/make-token {:name "base" :value "1" :type :sizing}))) + status (ctos/make-tokens-status :active-set-ids #{(cthi/id :core) (cthi/id :dark)}) + values (fn [resolved] (update-vals resolved :resolved-value)) + by-sets (ts/resolve-sets (cfo/get-active-token-sets status lib))] + (t/is (= {"base" 10 "double" 20} (values by-sets))) + (t/is (= (values (ts/resolve-tokens (cfo/get-tokens-in-active-sets status lib))) (values by-sets))) + (t/is (identical? by-sets (ts/resolve-sets (cfo/get-active-token-sets status lib))) + "the same sets from the library return the cached output") + (t/testing "a forced inactive set comes first, as in the merged variant" + (let [forced (ts/resolve-sets (cfo/get-active-token-sets-force status lib (cthi/id :extra)))] + (t/is (= (values (ts/resolve-tokens (cfo/get-tokens-in-active-sets-force status lib (cthi/id :extra)))) + (values forced))) + (t/is (= 10 (get-in forced ["base" :resolved-value]))))))) + +;; A token edited in a form resolves against every set of the library with +;; the selected set on top, and the draft on top of that. +(t/deftest resolve-edited-token-test + (let [lib (-> (ctob/make-tokens-lib) + (ctob/add-set (ctob/make-token-set :id (cthi/new-id! :base) :name "base")) + (ctob/add-set (ctob/make-token-set :id (cthi/new-id! :brand) :name "brand")) + (ctob/add-set (ctob/make-token-set :id (cthi/new-id! :other) :name "other")) + (ctob/add-token (cthi/id :base) (ctob/make-token {:name "size" :value "4" :type :sizing})) + (ctob/add-token (cthi/id :base) (ctob/make-token {:name "double" :value "{size} * 2" :type :sizing})) + (ctob/add-token (cthi/id :brand) (ctob/make-token {:name "size" :value "8" :type :sizing})) + (ctob/add-token (cthi/id :other) (ctob/make-token {:name "size" :value "16" :type :sizing}))) + sets (cfo/get-token-sets-for-edit lib (cthi/id :brand)) + values (fn [resolved] (update-vals resolved :resolved-value))] + (t/testing "the selected set is moved last and wins, as when merged over all tokens" + (t/is (= [(cthi/id :base) (cthi/id :other) (cthi/id :brand)] (mapv first sets))) + (t/is (= (values (ts/resolve-tokens (merge (ctob/get-all-tokens-map lib) (ctob/get-tokens lib (cthi/id :brand))))) + (values (ts/resolve-sets sets)))) + (t/is (= 8 (get-in (ts/resolve-sets sets) ["size" :resolved-value])))) + (t/testing "the draft wins over every set and its dependents see it" + (t/is (= 3 (:resolved-value (ts/resolve-draft sets {:name "size" :value "3" :type :sizing})))) + (t/is (= 14 (:resolved-value (ts/resolve-draft sets {:name "tmp" :value "{double} - 2" :type :sizing}))) + "each draft replaces the previous one")) + (t/testing "a renamed token is left out of every set, so references to it break" + (let [sets' (cfo/without-token sets "size")] + (t/is (not-any? #(contains? (second %) "size") sets')) + (t/is (= :error.style-dictionary/missing-reference + (get-in (ts/resolve-draft sets' {:name "tmp" :value "{size}" :type :sizing}) [:errors 0 :error/code])))) + (t/is (= [false true true] (mapv identical? sets (cfo/without-token sets "double"))) + "sets without the token stay the same objects")) + (t/testing "typing in a form does not push resolutions out of the cache" + (let [active (cfo/get-active-token-sets (ctos/make-tokens-status :active-set-ids #{(cthi/id :base)}) lib) + before (ts/resolve-sets active)] + (doseq [v (range 20)] + (ts/resolve-draft sets {:name "size" :value (str v) :type :sizing})) + (t/is (identical? before (ts/resolve-sets active))))))) + +(t/deftest resolve-sets-empty-test + (t/is (= {} (ts/resolve-sets []))) + (t/is (= {} (ts/resolve-tokens {})))) + +;; Renaming a token, with or without updating the references to it (see +;; `remap/build-remap-changes`), resolves as with StyleDictionary. The +;; renamed library is resolved from the interpreter state of the library +;; before the rename, as in the workspace. + +(defn- rename-token + [lib set-name old-name new-name] + (let [set-id (ctob/get-id (ctob/get-set-by-name lib set-name)) + token (ctob/get-token-by-name lib set-name old-name)] + (ctob/update-token lib set-id (:id token) #(assoc % :name new-name)))) + +(defn- update-references + [lib old-name new-name] + (reduce (fn [lib token-set] + (let [set-id (ctob/get-id token-set)] + (reduce (fn [lib token] + (ctob/update-token lib set-id (:id token) + #(update % :value cto/update-token-value-references old-name new-name))) + lib + (vals (ctob/get-tokens lib set-id))))) + lib + (ctob/get-sets lib))) + +(defn- resolved-values + [resolved] + (update-vals resolved (fn [token] + (if-let [errors (seq (:errors token))] + (mapv :error/code errors) + (:resolved-value token))))) + +(t/deftest rename-token-as-style-dictionary-test + (t/async + done + (let [lib (-> (ctob/make-tokens-lib) + (ctob/add-set (ctob/make-token-set :id (cthi/new-id! :core) :name "core")) + (ctob/add-set (ctob/make-token-set :id (cthi/new-id! :theme) :name "theme")) + (ctob/add-token (cthi/id :core) (ctob/make-token {:name "size" :value "4" :type :sizing})) + (ctob/add-token (cthi/id :core) (ctob/make-token {:name "double" :value "{size} * 2" :type :sizing})) + (ctob/add-token (cthi/id :core) (ctob/make-token {:name "brand.color" :value "#ff0000" :type :color})) + (ctob/add-token (cthi/id :core) (ctob/make-token {:name "accent" :value "{brand.color}" :type :color})) + (ctob/add-token (cthi/id :core) (ctob/make-token {:name "radius" :value "{size} + 1" :type :border-radius})) + (ctob/add-token (cthi/id :theme) (ctob/make-token {:name "size" :value "8" :type :sizing}))) + status (ctos/make-tokens-status :active-set-ids #{(cthi/id :core) (cthi/id :theme)}) + cases [["the visible definition, references updated" + #(-> % (rename-token "theme" "size" "space") (update-references "size" "space"))] + ["the visible definition, references kept: the earlier one shows through" + #(rename-token % "theme" "size" "space")] + ["a shadowed definition, references updated" + #(-> % (rename-token "core" "size" "space") (update-references "size" "space"))] + ["a shadowed definition, references kept" + #(rename-token % "core" "size" "space")] + ["a referenced token, references updated" + #(-> % (rename-token "core" "brand.color" "brand.primary") (update-references "brand.color" "brand.primary"))] + ["a referenced token, references kept: they break" + #(rename-token % "core" "brand.color" "brand.primary")] + ["to a name an earlier set defines" + #(rename-token % "theme" "size" "double")] + ["and back" + #(-> % (rename-token "theme" "size" "space") (rename-token "theme" "space" "size"))]] + ;; each case resolves the library first, so the renamed one is + ;; resolved incrementally from its state + results (mapv (fn [[label rename]] + (let [lib' (rename lib)] + (ts/resolve-sets (cfo/get-active-token-sets status lib)) + {:label label + :tokens (cfo/get-tokens-in-active-sets status lib') + :tokenscript (resolved-values (ts/resolve-sets (cfo/get-active-token-sets status lib')))})) + cases)] + (->> (rx/from results) + (rx/mapcat (fn [{:keys [tokens] :as result}] + (rx/map #(assoc result :style-dictionary (resolved-values %)) + (sd/resolve-tokens tokens)))) + (rx/reduce conj []) + (rx/subs! + (fn [results] + (doseq [{:keys [label tokenscript style-dictionary]} results] + (t/is (= style-dictionary tokenscript) (str "renaming " label))) + (let [[updated kept _ _ _ broken] (mapv :tokenscript results)] + (t/is (= {"size" 4 "space" 8 "double" 16 "radius" 9} + (select-keys updated ["size" "space" "double" "radius"]))) + (t/is (= {"size" 4 "space" 8 "double" 8 "radius" 5} + (select-keys kept ["size" "space" "double" "radius"]))) + (t/is (= [:error.style-dictionary/missing-reference] (get broken "accent")))) + (t/is (= 8 (count results))) + (done)) + (fn [cause] + (t/is (nil? cause)) + (done)))))))