diff --git a/backend/deps.edn b/backend/deps.edn index aad21e4455..a55b85032c 100644 --- a/backend/deps.edn +++ b/backend/deps.edn @@ -11,11 +11,12 @@ funcool/datoteka {:mvn/version "1.1.0"} expound/expound {:mvn/version "0.7.2"} instaparse/instaparse {:mvn/version "1.4.10"} + com.cognitect/transit-clj {:mvn/version "0.8.319"} ;; vertx deps + ;; vertx-clojure/vertx {:mvn/version "0.0.0-SNAPSHOT"} metosin/reitit-core {:mvn/version "0.3.10"} metosin/sieppari {:mvn/version "0.0.0-alpha8"} - com.cognitect/transit-clj {:mvn/version "0.8.319"} io.vertx/vertx-core {:mvn/version "3.8.1"} io.vertx/vertx-web {:mvn/version "3.8.1"} io.vertx/vertx-pg-client {:mvn/version "3.8.1"} diff --git a/backend/src/uxbox/core.clj b/backend/src/uxbox/core.clj index 441210f0cd..c5586f7b87 100644 --- a/backend/src/uxbox/core.clj +++ b/backend/src/uxbox/core.clj @@ -6,11 +6,10 @@ (ns uxbox.core (:require - [vertx.core :as vx] + [vertx.core :as vc] + [vertx.timers :as vt] [mount.core :as mount :refer [defstate]])) (defstate system - :start (vx/system) + :start (vc/system) :stop (.close system)) - - diff --git a/backend/src/uxbox/http.clj b/backend/src/uxbox/http.clj index ec1abd7e77..4f153007a1 100644 --- a/backend/src/uxbox/http.clj +++ b/backend/src/uxbox/http.clj @@ -38,7 +38,6 @@ :allow-headers #{:x-requested-with :content-type :cookie}} interceptors [(vxi/cookies) - (vxi/headers) (vxi/params) (vxi/cors cors-opts) interceptors/parse-request-body @@ -66,6 +65,6 @@ (vh/server ctx {:handler handler :port (:http-server-port cfg/config)}))) -(defstate http-verticle +(defstate server :start (let [factory (vc/verticle {:on-start on-start})] @(vc/deploy! system factory {:instances 4}))) diff --git a/backend/src/uxbox/http/handlers.clj b/backend/src/uxbox/http/handlers.clj index fec037f925..1df4eab7c8 100644 --- a/backend/src/uxbox/http/handlers.clj +++ b/backend/src/uxbox/http/handlers.clj @@ -83,4 +83,3 @@ :body {:params (:params req) :cookies (:cookies req) :headers (:headers req)}}) - diff --git a/backend/src/uxbox/media_loader.clj b/backend/src/uxbox/media_loader.clj new file mode 100644 index 0000000000..c49eef0e65 --- /dev/null +++ b/backend/src/uxbox/media_loader.clj @@ -0,0 +1,252 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) 2016-2019 Andrey Antukh + +(ns uxbox.media-loader + "Media collections importer (command line helper)." + (:require + [clojure.tools.logging :as log] + [clojure.spec.alpha :as s] + [clojure.pprint :refer [pprint]] + [clojure.java.io :as io] + [clojure.edn :as edn] + [promesa.core :as p] + [mount.core :as mount] + [cuerdas.core :as str] + [datoteka.storages :as st] + [datoteka.core :as fs] + [uxbox.config] + [uxbox.db :as db] + [uxbox.http] + [uxbox.migrations] + [uxbox.media :as media] + [uxbox.util.svg :as svg] + [uxbox.util.transit :as t] + [uxbox.util.spec :as us] + [uxbox.util.blob :as blob] + [uxbox.util.uuid :as uuid] + [uxbox.util.data :as data]) + (:import + java.io.Reader + java.io.PushbackReader + org.im4java.core.Info)) + +;; --- Constants & Helpers + +(def ^:const +images-uuid-ns+ #uuid "3642a582-565f-4070-beba-af797ab27a6e") +(def ^:const +icons-uuid-ns+ #uuid "3642a582-565f-4070-beba-af797ab27a6f") + +(s/def ::name ::us/string) +(s/def ::path ::us/string) +(s/def ::regex us/regex?) +(s/def ::import-item + (s/keys :req-un [::name ::path ::regex])) + +(defn exit! + ([] (exit! 0)) + ([code] + (System/exit code))) + +;; --- Icons Collections Importer + +(defn- create-icons-collection + "Create or replace icons collection by its name." + [conn {:keys [name] :as item}] + (log/info "Creating or updating icons collection:" name) + (let [id (uuid/namespaced +icons-uuid-ns+ name) + sql "insert into icons_collections (id, user_id, name) + values ($1, '00000000-0000-0000-0000-000000000000'::uuid, $2) + on conflict (id) + do update set name = $2 + returning *;" + sqlv [sql id name]] + (-> (db/query-one conn [sql id name]) + (p/then' (constantly id))))) + +(def create-icon-sql + "insert into icons (user_id, id, collection_id, name, metadata, content) + values ('00000000-0000-0000-0000-000000000000'::uuid, $1, $2, $3, $4, $5) + on conflict (id) + do update set name = $3, + metadata = $4, + content = $5, + collection_id = $2, + user_id = '00000000-0000-0000-0000-000000000000'::uuid + returning *;") + +(defn- create-or-update-icon + [conn id icon-id localpath] + (s/assert fs/path? localpath) + (s/assert ::us/uuid id) + (s/assert ::us/uuid icon-id) + (let [filename (fs/name localpath) + extension (second (fs/split-ext filename)) + data (svg/parse localpath) + mdata (select-keys data [:width :height :view-box])] + (db/query-one conn [create-icon-sql icon-id id + (:name data filename) + (blob/encode mdata) + (:content data)]))) + +(defn- import-icon + [conn id fpath] + (s/assert ::us/uuid id) + (s/assert fs/path? fpath) + (let [filename (fs/name fpath) + icon-id (uuid/namespaced +icons-uuid-ns+ (str id filename))] + (log/info "Creating or updating icon" filename icon-id) + (-> (create-or-update-icon conn id icon-id fpath) + (p/then (constantly nil))))) + +(defn- import-icons + [conn coll-id {:keys [path regex] :as item}] + (p/run! (fn [fpath] + (when (re-matches regex (str fpath)) + (import-icon conn coll-id fpath))) + (->> (fs/list-dir path) + (filter fs/regular-file?)))) + +(defn- process-icons-collection + [conn basedir {:keys [path regex] :as item}] + (s/assert ::import-item item) + (-> (create-icons-collection conn item) + (p/then (fn [coll-id] + (->> (assoc item :path (fs/join basedir path)) + (import-icons conn coll-id)))))) + +;; --- Images Collections Importer + +(defn- create-images-collection + "Create or replace image collection by its name." + [conn {:keys [name] :as item}] + (log/info "Creating or updating image collection:" name) + (let [id (uuid/namespaced +icons-uuid-ns+ name) + sql "insert into images_collections (id, user_id, name) + values ($1, '00000000-0000-0000-0000-000000000000'::uuid, $2) + on conflict (id) + do update set name = $2 + returning *;" + sqlv [sql id name]] + (-> (db/query-one conn [sql id name]) + (p/then' (constantly id))))) + +(defn- retrieve-image-size + [path] + (let [info (Info. (str path) true)] + [(.getImageWidth info) (.getImageHeight info)])) + +(defn- image-exists? + [conn id] + (s/assert ::us/uuid id) + (let [sql "select id + from images as i + where i.id = $1 + and i.user_id = '00000000-0000-0000-0000-000000000000'::uuid"] + (-> (db/query-one conn [sql id]) + (p/then (fn [row] (if row true false)))))) + +(def create-image-sql + "insert into images (user_id, id, collection_id, name, path, width, height, mimetype) + values ('00000000-0000-0000-0000-000000000000'::uuid, $1, $2, $3, $4, $5, $6, $7) + returning *;") + +(defn- create-image + [conn id image-id localpath] + (s/assert fs/path? localpath) + (s/assert ::us/uuid id) + (s/assert ::us/uuid image-id) + (let [storage media/images-storage + filename (fs/name localpath) + [width height] (retrieve-image-size localpath) + extension (second (fs/split-ext filename)) + mimetype (case extension + ".jpg" "image/jpeg" + ".png" "image/png")] + (-> (st/save storage filename localpath) + (p/then (fn [path] + (db/query-one conn [create-image-sql image-id id + filename + (str path) + width + height + mimetype]))) + (p/then (constantly nil))))) + +(defn- import-image + [conn id fpath] + (s/assert ::us/uuid id) + (s/assert fs/path? fpath) + (let [filename (fs/name fpath) + image-id (uuid/namespaced +images-uuid-ns+ (str id filename))] + (-> (image-exists? conn image-id) + (p/then (fn [exists?] + (when-not exists? + (log/info "Creating image" filename image-id) + (create-image conn id image-id fpath)))) + (p/then (constantly nil))))) + +(defn- import-images + [conn coll-id {:keys [path regex] :as item}] + (p/run! (fn [fpath] + (when (re-matches regex (str fpath)) + (import-image conn coll-id fpath))) + (->> (fs/list-dir path) + (filter fs/regular-file?)))) + +(defn- process-images-collection + [conn basedir {:keys [path regex] :as item}] + (s/assert ::import-item item) + (-> (create-images-collection conn item) + (p/then (fn [coll-id] + (->> (assoc item :path (fs/join basedir path)) + (import-images conn coll-id)))))) + +;; --- Entry Point + +(defn- validate-path + [path] + (when-not path + (log/error "No path is provided") + (exit! -1)) + (when-not (fs/exists? path) + (log/error "Path does not exists.") + (exit! -1)) + (when (fs/directory? path) + (log/error "The provided path is a directory.") + (exit! -1)) + (fs/path path)) + +(defn- read-import-file + [path] + (let [path (validate-path path) + reader (java.io.PushbackReader. (io/reader path))] + [(fs/parent path) + (read reader)])) + +(defn- start-system + [] + (-> (mount/except #{#'uxbox.http/server}) + (mount/start))) + +(defn- stop-system + [] + (mount/stop)) + +(defn- importer + [conn basedir data] + (let [images (:images data) + icons (:icons data)] + (p/do! + (p/run! #(process-images-collection conn basedir %) images) + (p/run! #(process-icons-collection conn basedir %) icons)))) + +(defn -main + [& [path]] + (let [[basedir data] (read-import-file path)] + (start-system) + (-> (db/with-atomic [conn db/pool] + (importer conn basedir data)) + (p/finally (fn [_ _] + (stop-system)))))) diff --git a/backend/src/uxbox/services/svgparse.clj b/backend/src/uxbox/services/svgparse.clj deleted file mode 100644 index 0e1e122078..0000000000 --- a/backend/src/uxbox/services/svgparse.clj +++ /dev/null @@ -1,99 +0,0 @@ -;; This Source Code Form is subject to the terms of the Mozilla Public -;; License, v. 2.0. If a copy of the MPL was not distributed with this -;; file, You can obtain one at http://mozilla.org/MPL/2.0/. -;; -;; Copyright (c) 2016 Andrey Antukh - -(ns uxbox.services.svgparse - (:require [clojure.spec.alpha :as s] - [cuerdas.core :as str] - [uxbox.util.spec :as us] - [uxbox.services.core :as core] - [uxbox.util.exceptions :as ex]) - (:import org.jsoup.Jsoup - java.io.InputStream)) - -;; (s/def ::content string?) -;; (s/def ::width number?) -;; (s/def ::height number?) -;; (s/def ::name string?) -;; (s/def ::view-box (s/coll-of number? :min-count 4 :max-count 4)) -;; (s/def ::svg-entity (s/keys :req-un [::content ::width ::height ::view-box] -;; :opt-un [::name])) - -;; ;; --- Implementation - -;; (defn- parse-double -;; [data] -;; {:pre [(string? data)]} -;; (Double/parseDouble data)) - -;; (defn- parse-viewbox -;; [data] -;; {:pre [(string? data)]} -;; (mapv parse-double (str/split data #"\s+"))) - -;; (defn- assoc-attr -;; [acc attr] -;; (let [key (.getKey attr) -;; val (.getValue attr)] -;; (case key -;; "width" (assoc acc :width (parse-double val)) -;; "height" (assoc acc :height (parse-double val)) -;; "viewbox" (assoc acc :view-box (parse-viewbox val)) -;; "sodipodi:docname" (assoc acc :name val) -;; acc))) - -;; (defn- parse-attrs -;; [element] -;; (let [attrs (.attributes element)] -;; (reduce assoc-attr {} attrs))) - -;; (defn- parse-svg -;; [data] -;; (try -;; (let [document (Jsoup/parse data) -;; svgelement (some-> (.body document) -;; (.getElementsByTag "svg") -;; (first)) -;; innerxml (.html svgelement) -;; attrs (parse-attrs svgelement)] -;; (merge {:content innerxml} attrs)) -;; (catch java.lang.IllegalArgumentException e -;; (ex/raise :type :validation -;; :code ::invalid-input -;; :message "Input does not seems to be a valid svg.")) -;; (catch java.lang.NullPointerException e -;; (ex/raise :type :validation -;; :code ::invalid-input -;; :message "Input does not seems to be a valid svg.")) - -;; (catch org.jsoup.UncheckedIOException e -;; (ex/raise :type :validation -;; :code ::invalid-input -;; :message "Input does not seems to be a valid svg.")) - -;; (catch Exception e -;; (ex/raise :code ::unexpected)))) - -;; ;; --- Public Api - -;; (defn parse-string -;; "Parse SVG from a string." -;; [data] -;; {:pre [(string? data)]} -;; (let [result (parse-svg data)] -;; (if (s/valid? ::svg-entity result) -;; result -;; (ex/raise :type :validation -;; :code ::invalid-result -;; :message "The result does not conform valid svg entity.")))) - -;; (defn parse -;; [data] -;; (parse-string (slurp data))) - -;; (defmethod core/query :parse-svg -;; [{:keys [data] :as params}] -;; {:pre [(string? data)]} -;; (parse-string data)) diff --git a/backend/src/uxbox/util/spec.clj b/backend/src/uxbox/util/spec.clj index 5a948424e1..dca9783552 100644 --- a/backend/src/uxbox/util/spec.clj +++ b/backend/src/uxbox/util/spec.clj @@ -2,7 +2,7 @@ ;; License, v. 2.0. If a copy of the MPL was not distributed with this ;; file, You can obtain one at http://mozilla.org/MPL/2.0/. ;; -;; Copyright (c) 2016 Andrey Antukh +;; Copyright (c) 2016-2019 Andrey Antukh (ns uxbox.util.spec (:refer-clojure :exclude [keyword uuid vector boolean map set]) @@ -21,6 +21,9 @@ (def uuid-rx #"^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$") +(def number-rx + #"^[+-]?([0-9]*\.?[0-9]+|[0-9]+\.?[0-9]*)([eE][+-]?[0-9]+)?$") + ;; --- Public Api (defn conform @@ -104,12 +107,23 @@ (fs/path? v) v :else ::s/invalid)) +(defn- number-conformer + [v] + (cond + (number? v) v + (re-matches number-rx v) (Double/parseDouble v) + :else ::s/invalid)) + + ;; --- Default Specs (s/def ::string string?) (s/def ::integer (s/conformer integer-conformer str)) (s/def ::uuid (s/conformer uuid-conformer str)) (s/def ::boolean (s/conformer boolean-conformer boolean-unformer)) +(s/def ::number (s/conformer number-conformer str)) +(s/def ::path (s/conformer path-conformer str)) + (s/def ::positive pos?) (s/def ::negative neg?) (s/def ::uploaded-file any?) @@ -118,7 +132,6 @@ (s/def ::file any?) (s/def ::name ::string) -(s/def ::path (s/conformer path-conformer str)) (s/def ::size ::integer) (s/def ::mtype ::string) (s/def ::upload diff --git a/backend/src/uxbox/util/svg.clj b/backend/src/uxbox/util/svg.clj new file mode 100644 index 0000000000..f646cad26a --- /dev/null +++ b/backend/src/uxbox/util/svg.clj @@ -0,0 +1,99 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) 2016-2019 Andrey Antukh + +(ns uxbox.util.svg + "Icons SVG parsing helpers." + (:require + [clojure.spec.alpha :as s] + [cuerdas.core :as str] + [uxbox.util.spec :as us] + [uxbox.util.exceptions :as ex]) + (:import + org.jsoup.Jsoup + org.jsoup.nodes.Attribute + org.jsoup.nodes.Element + org.jsoup.nodes.Document + java.io.InputStream)) + +(s/def ::content ::us/string) +(s/def ::width ::us/number) +(s/def ::height ::us/number) +(s/def ::name ::us/string) +(s/def ::view-box (s/coll-of ::us/number :min-count 4 :max-count 4)) + +(s/def ::svg-entity + (s/keys :req-un [::content ::width ::height ::view-box] + :opt-un [::name])) + +;; --- Implementation + +(defn- parse-double + [data] + (s/assert ::us/string data) + (Double/parseDouble data)) + +(defn- parse-viewbox + [data] + (s/assert ::us/string data) + (mapv parse-double (str/split data #"\s+"))) + +(defn- parse-attrs + [^Element element] + (persistent! + (reduce (fn [acc ^Attribute attr] + (let [key (.getKey attr) + val (.getValue attr)] + (case key + "width" (assoc! acc :width (parse-double val)) + "height" (assoc! acc :height (parse-double val)) + "viewbox" (assoc! acc :view-box (parse-viewbox val)) + "sodipodi:docname" (assoc! acc :name val) + acc))) + (transient {}) + (.attributes element)))) + +(defn- impl-parse + [data] + (try + (let [document (Jsoup/parse ^String data) + element (some-> (.body ^Document document) + (.getElementsByTag "svg") + (first)) + content (.html element) + attrs (parse-attrs element)] + (assoc attrs :content content)) + (catch java.lang.IllegalArgumentException e + (ex/raise :type :validation + :code ::invalid-input + :message "Input does not seems to be a valid svg.")) + (catch java.lang.NullPointerException e + (ex/raise :type :validation + :code ::invalid-input + :message "Input does not seems to be a valid svg.")) + (catch org.jsoup.UncheckedIOException e + (ex/raise :type :validation + :code ::invalid-input + :message "Input does not seems to be a valid svg.")) + (catch Exception e + (ex/raise :type :internal + :code ::unexpected)))) + +;; --- Public Api + +(defn parse-string + "Parse SVG from a string." + [data] + (s/assert ::us/string data) + (let [result (impl-parse data)] + (if (s/valid? ::svg-entity result) + result + (ex/raise :type :validation + :code ::invalid-result + :message "The result does not conform valid svg entity.")))) + +(defn parse + [data] + (parse-string (slurp data))) diff --git a/backend/src/uxbox/util/template.clj b/backend/src/uxbox/util/template.clj index be1fb38c00..a2ac929153 100644 --- a/backend/src/uxbox/util/template.clj +++ b/backend/src/uxbox/util/template.clj @@ -8,6 +8,7 @@ "A lightweight abstraction over mustache.java template engine. The documentation can be found: http://mustache.github.io/mustache.5.html" (:require + [clojure.tools.logging :as log] [clojure.walk :as walk] [clojure.java.io :as io] [uxbox.util.exceptions :as ex]) @@ -33,7 +34,11 @@ (fn? x) (reify Function (apply [this content] - (x content))) + (try + (x content) + (catch Exception e + (log/error e "Error on executing" x) + "")))) (or (vector? x) (list? x)) (java.util.ArrayList. x)