mirror of
https://github.com/penpot/penpot.git
synced 2026-08-08 05:48:50 +00:00
✨ Add Penpot-to-Ladybug graph ingest vertical slice
This commit is contained in:
Alejandro Alonso
parent
e796512ddf
commit
3714f95666
@ -5,21 +5,22 @@
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.graph.ingest
|
||||
"Penpot file -> Ladybug graph projection.
|
||||
|
||||
Skeleton stage: loads the canonical file from the backend and exercises
|
||||
Ladybug. Document projection will replace the smoke test step."
|
||||
"Penpot file -> Ladybug graph projection."
|
||||
(:require
|
||||
[app.binfile.common :as bfc]
|
||||
[app.common.exceptions :as ex]
|
||||
[app.common.logging :as l]
|
||||
[app.db :as db]
|
||||
[app.graph.ladybug :as ladybug]
|
||||
[app.graph.project.document :as project.document]
|
||||
[app.graph.project.transforms :as project.transforms]
|
||||
[app.graph.schema :as schema]
|
||||
[app.graph.stats :as stats]
|
||||
[app.srepl.helpers :as h]))
|
||||
|
||||
(defn ingest-file!
|
||||
[system file-id & {:keys [db-path smoke-test?]
|
||||
:or {smoke-test? true}}]
|
||||
[system file-id & {:keys [db-path reset-db?]
|
||||
:or {reset-db? true}}]
|
||||
(let [file-id (h/parse-uuid file-id)
|
||||
file (db/run! system #(bfc/get-file % file-id :realize? true))
|
||||
db-path (or db-path (ladybug/db-path-for-file file-id))]
|
||||
@ -27,15 +28,24 @@
|
||||
(ex/raise :type :not-found
|
||||
:code :file-not-found
|
||||
:file-id (str file-id)))
|
||||
(l/inf :hint "graph ingest skeleton"
|
||||
(when reset-db?
|
||||
(ladybug/reset-db-path! db-path))
|
||||
(l/inf :hint "graph ingest"
|
||||
:file-id (str file-id)
|
||||
:revn (:revn file)
|
||||
:db-path db-path)
|
||||
;; TODO: project (:data file) into Ladybug node/rel tables.
|
||||
(let [ladybug-result (when smoke-test?
|
||||
(ladybug/smoke-test! system :db-path db-path))]
|
||||
{:file-id file-id
|
||||
:revn (:revn file)
|
||||
:name (get-in file [:data :name])
|
||||
:db-path db-path
|
||||
:ladybug ladybug-result})))
|
||||
:db-path db-path
|
||||
:schema schema/schema-version)
|
||||
(let [data (:data file)
|
||||
ddl (schema/ddl-statements)
|
||||
{:keys [statements stats]}
|
||||
(project.document/projection-statements data file)
|
||||
ingest-statements (conj (into ddl statements) "CHECKPOINT;")]
|
||||
(ladybug/exec! system db-path ingest-statements)
|
||||
{:file-id file-id
|
||||
:revn (:revn file)
|
||||
:name (or (:name data) (:name file))
|
||||
:db-path db-path
|
||||
:schema-version schema/schema-version
|
||||
:projection {:stats stats}
|
||||
:transforms (project.transforms/apply-transforms! system db-path data file)
|
||||
:stats (stats/summarize system db-path)})))
|
||||
|
||||
@ -5,24 +5,24 @@
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.graph.ladybug
|
||||
"Thin Ladybug access layer for graph-backed Penpot.
|
||||
|
||||
Uses the `lbug` CLI for now. A JNI-backed implementation can replace
|
||||
`exec!` later without changing callers."
|
||||
"Thin Ladybug CLI access layer for graph-backed Penpot."
|
||||
(:require
|
||||
[app.common.exceptions :as ex]
|
||||
[app.util.shell :as shell]
|
||||
[clojure.string :as str]
|
||||
[datoteka.fs :as fs])
|
||||
(:import
|
||||
java.io.File
|
||||
java.nio.file.Files))
|
||||
java.util.concurrent.TimeUnit
|
||||
org.apache.commons.io.IOUtils))
|
||||
|
||||
(set! *warn-on-reflection* true)
|
||||
|
||||
(defn- lbug-bin
|
||||
(defn lbug-bin
|
||||
"Resolved Ladybug CLI path (`PENPOT_LBUG_BIN`, `./lbug`, or `lbug`)."
|
||||
[]
|
||||
(or (System/getenv "PENPOT_LBUG_BIN") "lbug"))
|
||||
(or (System/getenv "PENPOT_LBUG_BIN")
|
||||
(let [local (java.io.File. "lbug")]
|
||||
(when (.exists local)
|
||||
(.getAbsolutePath local)))
|
||||
"lbug"))
|
||||
|
||||
(defn default-graph-dir
|
||||
[]
|
||||
@ -32,37 +32,158 @@
|
||||
[file-id]
|
||||
(str (fs/path (default-graph-dir) (str file-id ".lbug"))))
|
||||
|
||||
(defn- write-temp-script!
|
||||
(defn reset-db-path!
|
||||
[db-path]
|
||||
(when-not (= db-path ":memory:")
|
||||
(when (fs/exists? db-path)
|
||||
(fs/delete db-path))))
|
||||
|
||||
(defn escape-cypher-string
|
||||
[s]
|
||||
(-> (str s)
|
||||
(str/replace "\\" "\\\\")
|
||||
(str/replace "'" "\\'")))
|
||||
|
||||
(defn format-uuid
|
||||
[id]
|
||||
(str "uuid('" (str id) "')"))
|
||||
|
||||
(defn format-string
|
||||
[s]
|
||||
(str "'" (escape-cypher-string s) "'"))
|
||||
|
||||
(defn format-int
|
||||
[n]
|
||||
(str (long n)))
|
||||
|
||||
(defn format-value
|
||||
[v]
|
||||
(cond
|
||||
(nil? v) "NULL"
|
||||
(uuid? v) (format-uuid v)
|
||||
(string? v) (format-string v)
|
||||
(number? v) (format-int v)
|
||||
(boolean? v) (if v "true" "false")
|
||||
:else (format-string (str v))))
|
||||
|
||||
(defn- ensure-semicolon
|
||||
[statement]
|
||||
(let [s (str/trim (str statement))]
|
||||
(if (str/ends-with? s ";") s (str s ";"))))
|
||||
|
||||
(defn- script-content
|
||||
[statements]
|
||||
(let [^File file (File/createTempFile "penpot-graph-" ".cypher")
|
||||
content (str/join "\n" (concat statements [":quit"]))]
|
||||
(spit file content)
|
||||
(.getAbsolutePath file)))
|
||||
(str (str/join "\n" (map ensure-semicolon statements)) "\n"))
|
||||
|
||||
(defn- shell-quote
|
||||
[s]
|
||||
(str "\"" (str/replace s "\"" "\\\"") "\""))
|
||||
|
||||
(defn- shell-single-quote
|
||||
[s]
|
||||
(str "'" (str/replace s "'" "'\\''") "'"))
|
||||
|
||||
(defn- exec-sh-sync!
|
||||
"Run `sh -c` synchronously on the calling thread.
|
||||
|
||||
`shell/exec!` can return empty stdout for very fast pipelines when used from
|
||||
the REPL executor; this path reads the merged stream before `waitFor` returns."
|
||||
[shell-cmd & {:keys [timeout] :or {timeout 120}}]
|
||||
(let [^Process proc (.start (doto (ProcessBuilder. (into-array String ["sh" "-c" shell-cmd]))
|
||||
(.redirectErrorStream true)))
|
||||
out (IOUtils/toString (.getInputStream proc) "UTF-8")]
|
||||
(when-not (.waitFor proc (long timeout) TimeUnit/SECONDS)
|
||||
(.destroyForcibly proc)
|
||||
(ex/raise :type :internal
|
||||
:code :ladybug-timeout
|
||||
:hint "Ladybug query timed out"
|
||||
:shell-cmd shell-cmd
|
||||
:timeout timeout))
|
||||
{:exit (.exitValue proc)
|
||||
:out out
|
||||
:err ""}))
|
||||
|
||||
(defn- run-script!
|
||||
"Run `lbug` with Cypher on stdin.
|
||||
|
||||
Writes use a heredoc via `shell/exec!`. Queries use `printf ... | lbug`
|
||||
via a synchronous shell invocation (matches the working manual command)."
|
||||
[system db-path flags script & {:keys [timeout] :or {timeout 120}}]
|
||||
(let [cmd (into [(lbug-bin) db-path] flags)
|
||||
query? (some #{"line"} flags)
|
||||
shell-cmd (if query?
|
||||
(str "printf " (shell-single-quote script)
|
||||
" | " (str/join " " (map shell-quote cmd)))
|
||||
(str (str/join " " (map shell-quote cmd))
|
||||
" <<'LBUG_EOF'\n" script "\nLBUG_EOF"))]
|
||||
(if query?
|
||||
(exec-sh-sync! shell-cmd :timeout timeout)
|
||||
(shell/exec! system {:cmd ["sh" "-c" shell-cmd] :timeout timeout}))))
|
||||
|
||||
(defn exec!
|
||||
"Execute one or more Cypher statements against a Ladybug database.
|
||||
|
||||
`db-path` is either `:memory:` or a filesystem path to a `.lbug` database."
|
||||
[system db-path statements & {:keys [timeout] :or {timeout 120}}]
|
||||
"Execute Cypher statements. `:mode` is `:write` (default) or `:query`."
|
||||
[system db-path statements & {:keys [timeout mode] :or {timeout 120 mode :write}}]
|
||||
(assert (sequential? statements) "statements should be a sequential collection")
|
||||
(when-not (= db-path ":memory:")
|
||||
(fs/create-dir (fs/parent db-path)))
|
||||
(let [script-path (write-temp-script! statements)
|
||||
result (shell/exec!
|
||||
system
|
||||
{:cmd [(lbug-bin) db-path "-i" script-path "-m" "csv" "-s" "-b"]
|
||||
:timeout timeout})]
|
||||
(Files/deleteIfExists (fs/path script-path))
|
||||
(let [flags (if (= mode :query)
|
||||
["-m" "line" "-s"]
|
||||
["-m" "trash" "-s" "-b"])
|
||||
script (if (= mode :query)
|
||||
(str ":singleline\n" (script-content statements))
|
||||
(script-content statements))
|
||||
result (run-script! system db-path flags script :timeout timeout)]
|
||||
(when (not= 0 (:exit result))
|
||||
(ex/raise :type :internal
|
||||
:code :ladybug-exec-failed
|
||||
:hint "Ladybug query execution failed"
|
||||
:hint "Ladybug execution failed"
|
||||
:db-path db-path
|
||||
:exit (:exit result)
|
||||
:err (:err result)
|
||||
:out (:out result)))
|
||||
:out (:out result)
|
||||
:err (:err result)))
|
||||
result))
|
||||
|
||||
(defn- lbug-noise-line?
|
||||
[line]
|
||||
(or (str/blank? line)
|
||||
(str/starts-with? line "--")
|
||||
(str/starts-with? line ":singleline")
|
||||
(str/includes? line "Single line mode")
|
||||
(str/includes? line "usage hints")
|
||||
(str/includes? line "Processing:")
|
||||
(str/includes? line "Pipeline")
|
||||
(str/includes? line "Progress:")))
|
||||
|
||||
(defn- data-lines
|
||||
[out]
|
||||
(->> (str/split-lines (str out))
|
||||
(map str/trim)
|
||||
(remove lbug-noise-line?)))
|
||||
|
||||
(defn- parse-scalar-line
|
||||
[line]
|
||||
(let [line (str/trim line)]
|
||||
(cond
|
||||
(re-matches #"-?\d+" line) (Long/parseLong line)
|
||||
:else (some->> (re-seq #"-?\d+" line) last Long/parseLong))))
|
||||
|
||||
(defn- parse-equality-value
|
||||
"Last `label = 123` tuple in output (results come after pipeline noise)."
|
||||
[out]
|
||||
(some->> (re-seq #"([A-Za-z][A-Za-z0-9_]*)\s*=\s*(-?\d+)" (str out))
|
||||
(remove (fn [[_ label _]]
|
||||
(or (str/includes? label "Pipeline")
|
||||
(str/includes? label "Progress"))))
|
||||
last
|
||||
(nth 2)
|
||||
Long/parseLong))
|
||||
|
||||
(defn query-scalar!
|
||||
[system db-path statement & {:keys [timeout] :or {timeout 120}}]
|
||||
(let [out (:out (exec! system db-path [statement] :timeout timeout :mode :query))]
|
||||
(or (some parse-scalar-line (data-lines out))
|
||||
(parse-equality-value out))))
|
||||
|
||||
(defn smoke-test-statements
|
||||
[]
|
||||
["CREATE NODE TABLE Person(name STRING, age INT64, PRIMARY KEY(name));"
|
||||
@ -71,8 +192,6 @@
|
||||
"MATCH (a:Person) RETURN a.name AS NAME, a.age AS AGE ORDER BY NAME;"])
|
||||
|
||||
(defn smoke-test!
|
||||
"Run a minimal CREATE + MATCH against Ladybug."
|
||||
[system & {:keys [db-path] :or {db-path ":memory:"}}]
|
||||
(let [result (exec! system db-path (smoke-test-statements))]
|
||||
{:db-path db-path
|
||||
:out (:out result)}))
|
||||
{:db-path db-path
|
||||
:out (:out (exec! system db-path (smoke-test-statements)))})
|
||||
|
||||
115
backend/src/app/graph/project/document.clj
Normal file
115
backend/src/app/graph/project/document.clj
Normal file
@ -0,0 +1,115 @@
|
||||
;; This Source Code Form is subject to the terms of the Mozilla Public
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.graph.project.document
|
||||
"Project a Penpot file-data map into Ladybug nodes and structural edges.
|
||||
|
||||
Vertical slice: Document, Page, top-level shapes, and `IsChildOf` edges
|
||||
mirroring beadpot's `add_document` first pass."
|
||||
(:require
|
||||
[app.common.logging :as l]
|
||||
[app.common.uuid :as uuid]
|
||||
[app.graph.ladybug :as ladybug]
|
||||
[clojure.string :as str]))
|
||||
|
||||
(def root-frame-id
|
||||
uuid/zero)
|
||||
|
||||
(def ^:private shape-type->table
|
||||
{:frame "Frame"
|
||||
:rect "Rectangle"
|
||||
:group "Group"
|
||||
:circle "Circle"
|
||||
:path "Path"
|
||||
:text "Text"
|
||||
:bool "Boolean"
|
||||
:image "Image"
|
||||
:svg-raw "SVGRaw"})
|
||||
|
||||
(defn- create-node-statement
|
||||
[table attrs]
|
||||
(let [props (str/join ", "
|
||||
(map (fn [[k v]]
|
||||
(str "`" (name k) "`: " (ladybug/format-value v)))
|
||||
attrs))]
|
||||
(str "CREATE (n:`" table "` {" props "});")))
|
||||
|
||||
(defn- merge-edge-statement
|
||||
[from-table from-id to-table to-id position]
|
||||
(str "MATCH (c:`" from-table "` {`id`: " (ladybug/format-uuid from-id) "}), "
|
||||
"(p:`" to-table "` {`id`: " (ladybug/format-uuid to-id) "}) "
|
||||
"MERGE (c)-[:`IsChildOf` {`position`: " (ladybug/format-int position) "}]->(p);"))
|
||||
|
||||
(defn- document-attrs
|
||||
[file data]
|
||||
{:id (or (:id data) (:id file))
|
||||
:name (or (:name data) (:name file) "Untitled")
|
||||
:version (long (or (:version data) 67))
|
||||
:revision (long (or (:revn file) 0))})
|
||||
|
||||
(defn- page-attrs
|
||||
[page index]
|
||||
(cond-> {:id (:id page)
|
||||
:name (:name page)}
|
||||
(some? index) (assoc :index (long index))))
|
||||
|
||||
(defn- shape-table
|
||||
[shape]
|
||||
(get shape-type->table (keyword (:type shape))))
|
||||
|
||||
(defn- project-page
|
||||
[statements stats doc-id page position]
|
||||
(let [page-id (:id page)
|
||||
objects (:objects page)
|
||||
root (get objects root-frame-id)
|
||||
top-level-ids (when root (vec (reverse (:shapes root))))
|
||||
statements' (conj statements
|
||||
(create-node-statement "Page" (page-attrs page position))
|
||||
(merge-edge-statement "Page" page-id "Document" doc-id position))
|
||||
stats' (update stats :pages inc)]
|
||||
(if (seq top-level-ids)
|
||||
(reduce
|
||||
(fn [[stmts st] [shape-pos shape-id]]
|
||||
(if-let [shape (get objects shape-id)]
|
||||
(if-let [table (shape-table shape)]
|
||||
[(-> stmts
|
||||
(conj (create-node-statement table {:id (:id shape)
|
||||
:name (:name shape)})
|
||||
(merge-edge-statement table (:id shape)
|
||||
"Page" page-id shape-pos)))
|
||||
(update st :shapes inc)]
|
||||
(do
|
||||
(l/wrn :hint "unsupported shape type for graph slice"
|
||||
:shape-id (str shape-id)
|
||||
:type (:type shape))
|
||||
[stmts st]))
|
||||
[stmts st]))
|
||||
[statements' stats']
|
||||
(map-indexed vector top-level-ids))
|
||||
[statements' stats'])))
|
||||
|
||||
(defn projection-statements
|
||||
"Build Cypher statements for projecting `data` into Ladybug.
|
||||
|
||||
Returns `{:statements [...] :stats {...}}` without executing them."
|
||||
[data file]
|
||||
(let [doc-id (or (:id data) (:id file))
|
||||
pages (seq (reverse (:pages data)))
|
||||
initial [(create-node-statement "Document" (document-attrs file data))]
|
||||
[statements stats]
|
||||
(if (empty? pages)
|
||||
[initial {:documents 1 :pages 0 :shapes 0}]
|
||||
(reduce (fn [[stmts st] [position page-id]]
|
||||
(if-let [page (get-in data [:pages-index page-id])]
|
||||
(project-page stmts st doc-id page position)
|
||||
(do
|
||||
(l/wrn :hint "missing page in pages-index"
|
||||
:page-id (str page-id))
|
||||
[stmts st])))
|
||||
[initial {:documents 1 :pages 0 :shapes 0}]
|
||||
(map-indexed vector pages)))]
|
||||
{:statements statements
|
||||
:stats stats}))
|
||||
15
backend/src/app/graph/project/transforms.clj
Normal file
15
backend/src/app/graph/project/transforms.clj
Normal file
@ -0,0 +1,15 @@
|
||||
;; This Source Code Form is subject to the terms of the Mozilla Public
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.graph.project.transforms
|
||||
"Derived graph links (instances, tokens, nested containment, etc.).
|
||||
|
||||
Stub for now: beadpot's `apply_transformations` will be ported here.")
|
||||
|
||||
(defn apply-transforms!
|
||||
"Apply derived transformations to an already projected graph."
|
||||
[_system _db-path _data _file]
|
||||
{:transforms 0})
|
||||
60
backend/src/app/graph/report.clj
Normal file
60
backend/src/app/graph/report.clj
Normal file
@ -0,0 +1,60 @@
|
||||
;; This Source Code Form is subject to the terms of the Mozilla Public
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.graph.report
|
||||
(:require
|
||||
[clojure.core :as c]))
|
||||
|
||||
(defn- println!
|
||||
[& lines]
|
||||
(doseq [line lines]
|
||||
(println line)))
|
||||
|
||||
(defn- section-title
|
||||
[title]
|
||||
(println! (str "\n" title)
|
||||
(str (apply str (repeat (count title) "─")))))
|
||||
|
||||
(defn- kv-line
|
||||
[k v]
|
||||
(format " %-14s %s" (str k ":") v))
|
||||
|
||||
(defn- print-node-counts
|
||||
[nodes]
|
||||
(doseq [[table count] (sort-by first nodes)
|
||||
:when (pos? (long count))]
|
||||
(println! (kv-line table count))))
|
||||
|
||||
(defn print-ingest!
|
||||
"Pretty-print the result map returned by `app.graph.ingest/ingest-file!`."
|
||||
[{:keys [file-id revn name db-path schema-version projection transforms stats]}]
|
||||
(section-title "Graph ingest")
|
||||
(println! (kv-line "File" (str name " (" file-id ")"))
|
||||
(kv-line "Revision" revn)
|
||||
(kv-line "Schema" schema-version)
|
||||
(kv-line "Database" db-path))
|
||||
|
||||
(when-let [pstats (:stats projection)]
|
||||
(section-title "Projection")
|
||||
(doseq [[k v] (sort-by key pstats)]
|
||||
(println! (kv-line (c/name k) v))))
|
||||
|
||||
(section-title "Transforms")
|
||||
(println! (kv-line "Applied" (or (:transforms transforms) 0)))
|
||||
|
||||
(when stats
|
||||
(section-title "Graph counts")
|
||||
(when-let [nodes (:nodes stats)]
|
||||
(println! " Nodes")
|
||||
(print-node-counts nodes))
|
||||
(when-let [edges (:edges stats)]
|
||||
(println! " Edges")
|
||||
(doseq [[rel count] (sort-by key edges)
|
||||
:when (pos? (long count))]
|
||||
(println! (kv-line (c/name rel) count)))))
|
||||
|
||||
(println!)
|
||||
nil)
|
||||
81
backend/src/app/graph/schema.clj
Normal file
81
backend/src/app/graph/schema.clj
Normal file
@ -0,0 +1,81 @@
|
||||
;; This Source Code Form is subject to the terms of the Mozilla Public
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.graph.schema
|
||||
"Ladybug DDL for the graph-backed Penpot vertical slice.
|
||||
|
||||
Table names follow beadpot conventions. This is an intentionally small
|
||||
subset of the full schema and will grow incrementally."
|
||||
(:require
|
||||
[clojure.string :as str]))
|
||||
|
||||
(def schema-version
|
||||
"penpot-graph-slice-1")
|
||||
|
||||
(def node-tables
|
||||
[{:name "Document"
|
||||
:columns [["id" "UUID"]
|
||||
["name" "STRING"]
|
||||
["version" "INT64"]
|
||||
["revision" "INT64"]]}
|
||||
{:name "Page"
|
||||
:columns [["id" "UUID"]
|
||||
["name" "STRING"]
|
||||
["index" "INT64"]]}
|
||||
{:name "Frame"
|
||||
:columns [["id" "UUID"]
|
||||
["name" "STRING"]]}
|
||||
{:name "Rectangle"
|
||||
:columns [["id" "UUID"]
|
||||
["name" "STRING"]]}
|
||||
{:name "Group"
|
||||
:columns [["id" "UUID"]
|
||||
["name" "STRING"]]}
|
||||
{:name "Circle"
|
||||
:columns [["id" "UUID"]
|
||||
["name" "STRING"]]}
|
||||
{:name "Path"
|
||||
:columns [["id" "UUID"]
|
||||
["name" "STRING"]]}
|
||||
{:name "Text"
|
||||
:columns [["id" "UUID"]
|
||||
["name" "STRING"]]}
|
||||
{:name "Boolean"
|
||||
:columns [["id" "UUID"]
|
||||
["name" "STRING"]]}
|
||||
{:name "Image"
|
||||
:columns [["id" "UUID"]
|
||||
["name" "STRING"]]}
|
||||
{:name "SVGRaw"
|
||||
:columns [["id" "UUID"]
|
||||
["name" "STRING"]]}])
|
||||
|
||||
(def shape-node-tables
|
||||
(mapv :name (drop 2 node-tables)))
|
||||
|
||||
(defn- create-node-table-ddl
|
||||
[{:keys [name columns]}]
|
||||
(let [cols (str/join ", "
|
||||
(concat
|
||||
(map (fn [[col type]]
|
||||
(str "`" col "` " type))
|
||||
columns)
|
||||
["PRIMARY KEY (`id`)"]))]
|
||||
(str "CREATE NODE TABLE `" name "` (" cols ");")))
|
||||
|
||||
(def is-child-of-ddl
|
||||
(str "CREATE REL TABLE `IsChildOf` ("
|
||||
"FROM `Page` TO `Document`, "
|
||||
(str/join ", "
|
||||
(map (fn [shape]
|
||||
(str "FROM `" shape "` TO `Page`"))
|
||||
shape-node-tables))
|
||||
", `position` INT64);"))
|
||||
|
||||
(defn ddl-statements
|
||||
[]
|
||||
(conj (vec (map create-node-table-ddl node-tables))
|
||||
is-child-of-ddl))
|
||||
34
backend/src/app/graph/stats.clj
Normal file
34
backend/src/app/graph/stats.clj
Normal file
@ -0,0 +1,34 @@
|
||||
;; This Source Code Form is subject to the terms of the Mozilla Public
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS INC Sucursal en España SL
|
||||
|
||||
(ns app.graph.stats
|
||||
(:require
|
||||
[app.graph.ladybug :as ladybug]
|
||||
[app.graph.schema :as schema]))
|
||||
|
||||
(defn- node-label-for-match
|
||||
[table]
|
||||
(if (#{"Group" "Boolean"} table)
|
||||
(str "`" table "`")
|
||||
table))
|
||||
|
||||
(defn- count-query
|
||||
[system db-path statement]
|
||||
(or (ladybug/query-scalar! system db-path statement) 0))
|
||||
|
||||
(defn summarize
|
||||
"Return node/edge counts from the graph database."
|
||||
[system db-path]
|
||||
{:nodes (into {}
|
||||
(map (fn [{:keys [name]}]
|
||||
[name (count-query
|
||||
system db-path
|
||||
(str "MATCH (n:" (node-label-for-match name) ") "
|
||||
"RETURN count(n) AS " name "_c;"))])
|
||||
schema/node-tables))
|
||||
:edges {:IsChildOf (count-query
|
||||
system db-path
|
||||
"MATCH ()-[e:IsChildOf]->() RETURN count(e) AS IsChildOf_c;")}})
|
||||
@ -27,6 +27,7 @@
|
||||
[app.features.file-snapshots :as fsnap]
|
||||
[app.graph.ingest :as graph.ingest]
|
||||
[app.graph.ladybug :as graph.ladybug]
|
||||
[app.graph.report :as graph.report]
|
||||
[app.http.session :as session]
|
||||
[app.loggers.audit :as audit]
|
||||
[app.main :as main]
|
||||
@ -412,14 +413,27 @@
|
||||
[& {:keys [db-path] :or {db-path ":memory:"}}]
|
||||
(graph.ladybug/smoke-test! main/system :db-path db-path))
|
||||
|
||||
(defn ingest-file-to-graph!
|
||||
"Skeleton graph ingest for a Penpot file.
|
||||
(defn graph-query-test!
|
||||
"Query Document count for a file's graph db (REPL diagnostic)."
|
||||
[file-id & {:keys [db-path]}]
|
||||
(let [file-id (h/parse-uuid file-id)
|
||||
db-path (or db-path (graph.ladybug/db-path-for-file file-id))
|
||||
stmt "MATCH (n:Document) RETURN count(n) AS Document_c;"]
|
||||
(graph.ladybug/query-scalar! main/system db-path stmt)))
|
||||
|
||||
Loads and realizes the file from the database, prepares the per-file
|
||||
Ladybug database path, and (for now) runs the Ladybug smoke test.
|
||||
Full document projection is not implemented yet."
|
||||
[file-id & {:as opts}]
|
||||
(graph.ingest/ingest-file! main/system file-id opts))
|
||||
(defn ingest-file-to-graph!
|
||||
"Project a Penpot file into a per-file Ladybug database.
|
||||
|
||||
Loads and realizes the file from the database, ensures the slice schema,
|
||||
projects Document/Page/shape nodes, and returns graph stats.
|
||||
|
||||
Options:
|
||||
- `:db-path` path or `:memory:`
|
||||
- `:reset-db?` delete any existing db first (default true)"
|
||||
[file-id & opts]
|
||||
(let [result (graph.ingest/ingest-file! main/system file-id opts)]
|
||||
(graph.report/print-ingest! result)
|
||||
result))
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; PROCESSING
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user