From 3714f956668eefd7838f9827f5b843098c573bb1 Mon Sep 17 00:00:00 2001 From: Alejandro Alonso Date: Mon, 13 Jul 2026 10:58:49 +0200 Subject: [PATCH] :sparkles: Add Penpot-to-Ladybug graph ingest vertical slice --- backend/src/app/graph/ingest.clj | 42 +++-- backend/src/app/graph/ladybug.clj | 183 +++++++++++++++---- backend/src/app/graph/project/document.clj | 115 ++++++++++++ backend/src/app/graph/project/transforms.clj | 15 ++ backend/src/app/graph/report.clj | 60 ++++++ backend/src/app/graph/schema.clj | 81 ++++++++ backend/src/app/graph/stats.clj | 34 ++++ backend/src/app/srepl/main.clj | 28 ++- 8 files changed, 503 insertions(+), 55 deletions(-) create mode 100644 backend/src/app/graph/project/document.clj create mode 100644 backend/src/app/graph/project/transforms.clj create mode 100644 backend/src/app/graph/report.clj create mode 100644 backend/src/app/graph/schema.clj create mode 100644 backend/src/app/graph/stats.clj diff --git a/backend/src/app/graph/ingest.clj b/backend/src/app/graph/ingest.clj index 569bb82008..1762a1f4e2 100644 --- a/backend/src/app/graph/ingest.clj +++ b/backend/src/app/graph/ingest.clj @@ -5,21 +5,22 @@ ;; Copyright (c) KALEIDOS INC Sucursal en España SL (ns app.graph.ingest - "Penpot file -> Ladybug graph projection. - - Skeleton stage: loads the canonical file from the backend and exercises - Ladybug. Document projection will replace the smoke test step." + "Penpot file -> Ladybug graph projection." (:require [app.binfile.common :as bfc] [app.common.exceptions :as ex] [app.common.logging :as l] [app.db :as db] [app.graph.ladybug :as ladybug] + [app.graph.project.document :as project.document] + [app.graph.project.transforms :as project.transforms] + [app.graph.schema :as schema] + [app.graph.stats :as stats] [app.srepl.helpers :as h])) (defn ingest-file! - [system file-id & {:keys [db-path smoke-test?] - :or {smoke-test? true}}] + [system file-id & {:keys [db-path reset-db?] + :or {reset-db? true}}] (let [file-id (h/parse-uuid file-id) file (db/run! system #(bfc/get-file % file-id :realize? true)) db-path (or db-path (ladybug/db-path-for-file file-id))] @@ -27,15 +28,24 @@ (ex/raise :type :not-found :code :file-not-found :file-id (str file-id))) - (l/inf :hint "graph ingest skeleton" + (when reset-db? + (ladybug/reset-db-path! db-path)) + (l/inf :hint "graph ingest" :file-id (str file-id) :revn (:revn file) - :db-path db-path) - ;; TODO: project (:data file) into Ladybug node/rel tables. - (let [ladybug-result (when smoke-test? - (ladybug/smoke-test! system :db-path db-path))] - {:file-id file-id - :revn (:revn file) - :name (get-in file [:data :name]) - :db-path db-path - :ladybug ladybug-result}))) + :db-path db-path + :schema schema/schema-version) + (let [data (:data file) + ddl (schema/ddl-statements) + {:keys [statements stats]} + (project.document/projection-statements data file) + ingest-statements (conj (into ddl statements) "CHECKPOINT;")] + (ladybug/exec! system db-path ingest-statements) + {:file-id file-id + :revn (:revn file) + :name (or (:name data) (:name file)) + :db-path db-path + :schema-version schema/schema-version + :projection {:stats stats} + :transforms (project.transforms/apply-transforms! system db-path data file) + :stats (stats/summarize system db-path)}))) diff --git a/backend/src/app/graph/ladybug.clj b/backend/src/app/graph/ladybug.clj index 4b309db524..3aeb6b52c8 100644 --- a/backend/src/app/graph/ladybug.clj +++ b/backend/src/app/graph/ladybug.clj @@ -5,24 +5,24 @@ ;; Copyright (c) KALEIDOS INC Sucursal en España SL (ns app.graph.ladybug - "Thin Ladybug access layer for graph-backed Penpot. - - Uses the `lbug` CLI for now. A JNI-backed implementation can replace - `exec!` later without changing callers." + "Thin Ladybug CLI access layer for graph-backed Penpot." (:require [app.common.exceptions :as ex] [app.util.shell :as shell] [clojure.string :as str] [datoteka.fs :as fs]) (:import - java.io.File - java.nio.file.Files)) + java.util.concurrent.TimeUnit + org.apache.commons.io.IOUtils)) -(set! *warn-on-reflection* true) - -(defn- lbug-bin +(defn lbug-bin + "Resolved Ladybug CLI path (`PENPOT_LBUG_BIN`, `./lbug`, or `lbug`)." [] - (or (System/getenv "PENPOT_LBUG_BIN") "lbug")) + (or (System/getenv "PENPOT_LBUG_BIN") + (let [local (java.io.File. "lbug")] + (when (.exists local) + (.getAbsolutePath local))) + "lbug")) (defn default-graph-dir [] @@ -32,37 +32,158 @@ [file-id] (str (fs/path (default-graph-dir) (str file-id ".lbug")))) -(defn- write-temp-script! +(defn reset-db-path! + [db-path] + (when-not (= db-path ":memory:") + (when (fs/exists? db-path) + (fs/delete db-path)))) + +(defn escape-cypher-string + [s] + (-> (str s) + (str/replace "\\" "\\\\") + (str/replace "'" "\\'"))) + +(defn format-uuid + [id] + (str "uuid('" (str id) "')")) + +(defn format-string + [s] + (str "'" (escape-cypher-string s) "'")) + +(defn format-int + [n] + (str (long n))) + +(defn format-value + [v] + (cond + (nil? v) "NULL" + (uuid? v) (format-uuid v) + (string? v) (format-string v) + (number? v) (format-int v) + (boolean? v) (if v "true" "false") + :else (format-string (str v)))) + +(defn- ensure-semicolon + [statement] + (let [s (str/trim (str statement))] + (if (str/ends-with? s ";") s (str s ";")))) + +(defn- script-content [statements] - (let [^File file (File/createTempFile "penpot-graph-" ".cypher") - content (str/join "\n" (concat statements [":quit"]))] - (spit file content) - (.getAbsolutePath file))) + (str (str/join "\n" (map ensure-semicolon statements)) "\n")) + +(defn- shell-quote + [s] + (str "\"" (str/replace s "\"" "\\\"") "\"")) + +(defn- shell-single-quote + [s] + (str "'" (str/replace s "'" "'\\''") "'")) + +(defn- exec-sh-sync! + "Run `sh -c` synchronously on the calling thread. + + `shell/exec!` can return empty stdout for very fast pipelines when used from + the REPL executor; this path reads the merged stream before `waitFor` returns." + [shell-cmd & {:keys [timeout] :or {timeout 120}}] + (let [^Process proc (.start (doto (ProcessBuilder. (into-array String ["sh" "-c" shell-cmd])) + (.redirectErrorStream true))) + out (IOUtils/toString (.getInputStream proc) "UTF-8")] + (when-not (.waitFor proc (long timeout) TimeUnit/SECONDS) + (.destroyForcibly proc) + (ex/raise :type :internal + :code :ladybug-timeout + :hint "Ladybug query timed out" + :shell-cmd shell-cmd + :timeout timeout)) + {:exit (.exitValue proc) + :out out + :err ""})) + +(defn- run-script! + "Run `lbug` with Cypher on stdin. + + Writes use a heredoc via `shell/exec!`. Queries use `printf ... | lbug` + via a synchronous shell invocation (matches the working manual command)." + [system db-path flags script & {:keys [timeout] :or {timeout 120}}] + (let [cmd (into [(lbug-bin) db-path] flags) + query? (some #{"line"} flags) + shell-cmd (if query? + (str "printf " (shell-single-quote script) + " | " (str/join " " (map shell-quote cmd))) + (str (str/join " " (map shell-quote cmd)) + " <<'LBUG_EOF'\n" script "\nLBUG_EOF"))] + (if query? + (exec-sh-sync! shell-cmd :timeout timeout) + (shell/exec! system {:cmd ["sh" "-c" shell-cmd] :timeout timeout})))) (defn exec! - "Execute one or more Cypher statements against a Ladybug database. - - `db-path` is either `:memory:` or a filesystem path to a `.lbug` database." - [system db-path statements & {:keys [timeout] :or {timeout 120}}] + "Execute Cypher statements. `:mode` is `:write` (default) or `:query`." + [system db-path statements & {:keys [timeout mode] :or {timeout 120 mode :write}}] (assert (sequential? statements) "statements should be a sequential collection") (when-not (= db-path ":memory:") (fs/create-dir (fs/parent db-path))) - (let [script-path (write-temp-script! statements) - result (shell/exec! - system - {:cmd [(lbug-bin) db-path "-i" script-path "-m" "csv" "-s" "-b"] - :timeout timeout})] - (Files/deleteIfExists (fs/path script-path)) + (let [flags (if (= mode :query) + ["-m" "line" "-s"] + ["-m" "trash" "-s" "-b"]) + script (if (= mode :query) + (str ":singleline\n" (script-content statements)) + (script-content statements)) + result (run-script! system db-path flags script :timeout timeout)] (when (not= 0 (:exit result)) (ex/raise :type :internal :code :ladybug-exec-failed - :hint "Ladybug query execution failed" + :hint "Ladybug execution failed" :db-path db-path :exit (:exit result) - :err (:err result) - :out (:out result))) + :out (:out result) + :err (:err result))) result)) +(defn- lbug-noise-line? + [line] + (or (str/blank? line) + (str/starts-with? line "--") + (str/starts-with? line ":singleline") + (str/includes? line "Single line mode") + (str/includes? line "usage hints") + (str/includes? line "Processing:") + (str/includes? line "Pipeline") + (str/includes? line "Progress:"))) + +(defn- data-lines + [out] + (->> (str/split-lines (str out)) + (map str/trim) + (remove lbug-noise-line?))) + +(defn- parse-scalar-line + [line] + (let [line (str/trim line)] + (cond + (re-matches #"-?\d+" line) (Long/parseLong line) + :else (some->> (re-seq #"-?\d+" line) last Long/parseLong)))) + +(defn- parse-equality-value + "Last `label = 123` tuple in output (results come after pipeline noise)." + [out] + (some->> (re-seq #"([A-Za-z][A-Za-z0-9_]*)\s*=\s*(-?\d+)" (str out)) + (remove (fn [[_ label _]] + (or (str/includes? label "Pipeline") + (str/includes? label "Progress")))) + last + (nth 2) + Long/parseLong)) + +(defn query-scalar! + [system db-path statement & {:keys [timeout] :or {timeout 120}}] + (let [out (:out (exec! system db-path [statement] :timeout timeout :mode :query))] + (or (some parse-scalar-line (data-lines out)) + (parse-equality-value out)))) + (defn smoke-test-statements [] ["CREATE NODE TABLE Person(name STRING, age INT64, PRIMARY KEY(name));" @@ -71,8 +192,6 @@ "MATCH (a:Person) RETURN a.name AS NAME, a.age AS AGE ORDER BY NAME;"]) (defn smoke-test! - "Run a minimal CREATE + MATCH against Ladybug." [system & {:keys [db-path] :or {db-path ":memory:"}}] - (let [result (exec! system db-path (smoke-test-statements))] - {:db-path db-path - :out (:out result)})) + {:db-path db-path + :out (:out (exec! system db-path (smoke-test-statements)))}) diff --git a/backend/src/app/graph/project/document.clj b/backend/src/app/graph/project/document.clj new file mode 100644 index 0000000000..09b900c79a --- /dev/null +++ b/backend/src/app/graph/project/document.clj @@ -0,0 +1,115 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS INC Sucursal en España SL + +(ns app.graph.project.document + "Project a Penpot file-data map into Ladybug nodes and structural edges. + + Vertical slice: Document, Page, top-level shapes, and `IsChildOf` edges + mirroring beadpot's `add_document` first pass." + (:require + [app.common.logging :as l] + [app.common.uuid :as uuid] + [app.graph.ladybug :as ladybug] + [clojure.string :as str])) + +(def root-frame-id + uuid/zero) + +(def ^:private shape-type->table + {:frame "Frame" + :rect "Rectangle" + :group "Group" + :circle "Circle" + :path "Path" + :text "Text" + :bool "Boolean" + :image "Image" + :svg-raw "SVGRaw"}) + +(defn- create-node-statement + [table attrs] + (let [props (str/join ", " + (map (fn [[k v]] + (str "`" (name k) "`: " (ladybug/format-value v))) + attrs))] + (str "CREATE (n:`" table "` {" props "});"))) + +(defn- merge-edge-statement + [from-table from-id to-table to-id position] + (str "MATCH (c:`" from-table "` {`id`: " (ladybug/format-uuid from-id) "}), " + "(p:`" to-table "` {`id`: " (ladybug/format-uuid to-id) "}) " + "MERGE (c)-[:`IsChildOf` {`position`: " (ladybug/format-int position) "}]->(p);")) + +(defn- document-attrs + [file data] + {:id (or (:id data) (:id file)) + :name (or (:name data) (:name file) "Untitled") + :version (long (or (:version data) 67)) + :revision (long (or (:revn file) 0))}) + +(defn- page-attrs + [page index] + (cond-> {:id (:id page) + :name (:name page)} + (some? index) (assoc :index (long index)))) + +(defn- shape-table + [shape] + (get shape-type->table (keyword (:type shape)))) + +(defn- project-page + [statements stats doc-id page position] + (let [page-id (:id page) + objects (:objects page) + root (get objects root-frame-id) + top-level-ids (when root (vec (reverse (:shapes root)))) + statements' (conj statements + (create-node-statement "Page" (page-attrs page position)) + (merge-edge-statement "Page" page-id "Document" doc-id position)) + stats' (update stats :pages inc)] + (if (seq top-level-ids) + (reduce + (fn [[stmts st] [shape-pos shape-id]] + (if-let [shape (get objects shape-id)] + (if-let [table (shape-table shape)] + [(-> stmts + (conj (create-node-statement table {:id (:id shape) + :name (:name shape)}) + (merge-edge-statement table (:id shape) + "Page" page-id shape-pos))) + (update st :shapes inc)] + (do + (l/wrn :hint "unsupported shape type for graph slice" + :shape-id (str shape-id) + :type (:type shape)) + [stmts st])) + [stmts st])) + [statements' stats'] + (map-indexed vector top-level-ids)) + [statements' stats']))) + +(defn projection-statements + "Build Cypher statements for projecting `data` into Ladybug. + + Returns `{:statements [...] :stats {...}}` without executing them." + [data file] + (let [doc-id (or (:id data) (:id file)) + pages (seq (reverse (:pages data))) + initial [(create-node-statement "Document" (document-attrs file data))] + [statements stats] + (if (empty? pages) + [initial {:documents 1 :pages 0 :shapes 0}] + (reduce (fn [[stmts st] [position page-id]] + (if-let [page (get-in data [:pages-index page-id])] + (project-page stmts st doc-id page position) + (do + (l/wrn :hint "missing page in pages-index" + :page-id (str page-id)) + [stmts st]))) + [initial {:documents 1 :pages 0 :shapes 0}] + (map-indexed vector pages)))] + {:statements statements + :stats stats})) diff --git a/backend/src/app/graph/project/transforms.clj b/backend/src/app/graph/project/transforms.clj new file mode 100644 index 0000000000..9d64f1b224 --- /dev/null +++ b/backend/src/app/graph/project/transforms.clj @@ -0,0 +1,15 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS INC Sucursal en España SL + +(ns app.graph.project.transforms + "Derived graph links (instances, tokens, nested containment, etc.). + + Stub for now: beadpot's `apply_transformations` will be ported here.") + +(defn apply-transforms! + "Apply derived transformations to an already projected graph." + [_system _db-path _data _file] + {:transforms 0}) diff --git a/backend/src/app/graph/report.clj b/backend/src/app/graph/report.clj new file mode 100644 index 0000000000..d615a26447 --- /dev/null +++ b/backend/src/app/graph/report.clj @@ -0,0 +1,60 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS INC Sucursal en España SL + +(ns app.graph.report + (:require + [clojure.core :as c])) + +(defn- println! + [& lines] + (doseq [line lines] + (println line))) + +(defn- section-title + [title] + (println! (str "\n" title) + (str (apply str (repeat (count title) "─"))))) + +(defn- kv-line + [k v] + (format " %-14s %s" (str k ":") v)) + +(defn- print-node-counts + [nodes] + (doseq [[table count] (sort-by first nodes) + :when (pos? (long count))] + (println! (kv-line table count)))) + +(defn print-ingest! + "Pretty-print the result map returned by `app.graph.ingest/ingest-file!`." + [{:keys [file-id revn name db-path schema-version projection transforms stats]}] + (section-title "Graph ingest") + (println! (kv-line "File" (str name " (" file-id ")")) + (kv-line "Revision" revn) + (kv-line "Schema" schema-version) + (kv-line "Database" db-path)) + + (when-let [pstats (:stats projection)] + (section-title "Projection") + (doseq [[k v] (sort-by key pstats)] + (println! (kv-line (c/name k) v)))) + + (section-title "Transforms") + (println! (kv-line "Applied" (or (:transforms transforms) 0))) + + (when stats + (section-title "Graph counts") + (when-let [nodes (:nodes stats)] + (println! " Nodes") + (print-node-counts nodes)) + (when-let [edges (:edges stats)] + (println! " Edges") + (doseq [[rel count] (sort-by key edges) + :when (pos? (long count))] + (println! (kv-line (c/name rel) count))))) + + (println!) + nil) diff --git a/backend/src/app/graph/schema.clj b/backend/src/app/graph/schema.clj new file mode 100644 index 0000000000..b5c4b1b90b --- /dev/null +++ b/backend/src/app/graph/schema.clj @@ -0,0 +1,81 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS INC Sucursal en España SL + +(ns app.graph.schema + "Ladybug DDL for the graph-backed Penpot vertical slice. + + Table names follow beadpot conventions. This is an intentionally small + subset of the full schema and will grow incrementally." + (:require + [clojure.string :as str])) + +(def schema-version + "penpot-graph-slice-1") + +(def node-tables + [{:name "Document" + :columns [["id" "UUID"] + ["name" "STRING"] + ["version" "INT64"] + ["revision" "INT64"]]} + {:name "Page" + :columns [["id" "UUID"] + ["name" "STRING"] + ["index" "INT64"]]} + {:name "Frame" + :columns [["id" "UUID"] + ["name" "STRING"]]} + {:name "Rectangle" + :columns [["id" "UUID"] + ["name" "STRING"]]} + {:name "Group" + :columns [["id" "UUID"] + ["name" "STRING"]]} + {:name "Circle" + :columns [["id" "UUID"] + ["name" "STRING"]]} + {:name "Path" + :columns [["id" "UUID"] + ["name" "STRING"]]} + {:name "Text" + :columns [["id" "UUID"] + ["name" "STRING"]]} + {:name "Boolean" + :columns [["id" "UUID"] + ["name" "STRING"]]} + {:name "Image" + :columns [["id" "UUID"] + ["name" "STRING"]]} + {:name "SVGRaw" + :columns [["id" "UUID"] + ["name" "STRING"]]}]) + +(def shape-node-tables + (mapv :name (drop 2 node-tables))) + +(defn- create-node-table-ddl + [{:keys [name columns]}] + (let [cols (str/join ", " + (concat + (map (fn [[col type]] + (str "`" col "` " type)) + columns) + ["PRIMARY KEY (`id`)"]))] + (str "CREATE NODE TABLE `" name "` (" cols ");"))) + +(def is-child-of-ddl + (str "CREATE REL TABLE `IsChildOf` (" + "FROM `Page` TO `Document`, " + (str/join ", " + (map (fn [shape] + (str "FROM `" shape "` TO `Page`")) + shape-node-tables)) + ", `position` INT64);")) + +(defn ddl-statements + [] + (conj (vec (map create-node-table-ddl node-tables)) + is-child-of-ddl)) diff --git a/backend/src/app/graph/stats.clj b/backend/src/app/graph/stats.clj new file mode 100644 index 0000000000..00ad5ef182 --- /dev/null +++ b/backend/src/app/graph/stats.clj @@ -0,0 +1,34 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS INC Sucursal en España SL + +(ns app.graph.stats + (:require + [app.graph.ladybug :as ladybug] + [app.graph.schema :as schema])) + +(defn- node-label-for-match + [table] + (if (#{"Group" "Boolean"} table) + (str "`" table "`") + table)) + +(defn- count-query + [system db-path statement] + (or (ladybug/query-scalar! system db-path statement) 0)) + +(defn summarize + "Return node/edge counts from the graph database." + [system db-path] + {:nodes (into {} + (map (fn [{:keys [name]}] + [name (count-query + system db-path + (str "MATCH (n:" (node-label-for-match name) ") " + "RETURN count(n) AS " name "_c;"))]) + schema/node-tables)) + :edges {:IsChildOf (count-query + system db-path + "MATCH ()-[e:IsChildOf]->() RETURN count(e) AS IsChildOf_c;")}}) diff --git a/backend/src/app/srepl/main.clj b/backend/src/app/srepl/main.clj index 3e5c85fb2a..dcefefccb6 100644 --- a/backend/src/app/srepl/main.clj +++ b/backend/src/app/srepl/main.clj @@ -27,6 +27,7 @@ [app.features.file-snapshots :as fsnap] [app.graph.ingest :as graph.ingest] [app.graph.ladybug :as graph.ladybug] + [app.graph.report :as graph.report] [app.http.session :as session] [app.loggers.audit :as audit] [app.main :as main] @@ -412,14 +413,27 @@ [& {:keys [db-path] :or {db-path ":memory:"}}] (graph.ladybug/smoke-test! main/system :db-path db-path)) -(defn ingest-file-to-graph! - "Skeleton graph ingest for a Penpot file. +(defn graph-query-test! + "Query Document count for a file's graph db (REPL diagnostic)." + [file-id & {:keys [db-path]}] + (let [file-id (h/parse-uuid file-id) + db-path (or db-path (graph.ladybug/db-path-for-file file-id)) + stmt "MATCH (n:Document) RETURN count(n) AS Document_c;"] + (graph.ladybug/query-scalar! main/system db-path stmt))) - Loads and realizes the file from the database, prepares the per-file - Ladybug database path, and (for now) runs the Ladybug smoke test. - Full document projection is not implemented yet." - [file-id & {:as opts}] - (graph.ingest/ingest-file! main/system file-id opts)) +(defn ingest-file-to-graph! + "Project a Penpot file into a per-file Ladybug database. + + Loads and realizes the file from the database, ensures the slice schema, + projects Document/Page/shape nodes, and returns graph stats. + + Options: + - `:db-path` path or `:memory:` + - `:reset-db?` delete any existing db first (default true)" + [file-id & opts] + (let [result (graph.ingest/ingest-file! main/system file-id opts)] + (graph.report/print-ingest! result) + result)) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; PROCESSING