Merge pull request #184 from uxbox/other/redis-and-emails-refactor

Emails Sending Refactor & Redis base infrastructure.

backend/deps.edn

@@ -4,8 +4,7 @@
   "jcenter" {:url "https://jcenter.bintray.com/"}}
  :deps
  {org.clojure/clojure {:mvn/version "1.10.1"}
-  funcool/promesa {:mvn/version "5.1.0"}
-  funcool/cuerdas {:mvn/version "2020.03.26-3"}
+  org.clojure/data.json {:mvn/version "1.0.0"}
 
   ;; Logging
   org.clojure/tools.logging {:mvn/version "0.5.0"}
@@ -13,30 +12,34 @@
   org.apache.logging.log4j/log4j-core {:mvn/version "2.13.0"}
   org.apache.logging.log4j/log4j-jul {:mvn/version "2.13.0"}
   org.apache.logging.log4j/log4j-slf4j-impl {:mvn/version "2.13.0"}
-  funcool/datoteka {:mvn/version "1.2.0"}
+
   expound/expound {:mvn/version "0.8.4"}
   instaparse/instaparse {:mvn/version "1.4.10"}
   com.cognitect/transit-clj {:mvn/version "0.8.319"}
 
   ;; TODO: vendorize pgclient under `vertx-clojure/vertx-pgclient`
   io.vertx/vertx-pg-client {:mvn/version "4.0.0-milestone4"}
+  io.lettuce/lettuce-core {:mvn/version "5.2.2.RELEASE"}
+  java-http-clj/java-http-clj {:mvn/version "0.4.1"}
 
   vertx-clojure/vertx
   {:local/root "vendor/vertx"
    :deps/manifest :pom}
 
+  funcool/datoteka {:mvn/version "1.2.0"}
+  funcool/promesa {:mvn/version "5.1.0"}
+  funcool/cuerdas {:mvn/version "2020.03.26-3"}
   funcool/sodi
   {:local/root "vendor/sodi"
    :deps/manifest :pom}
 
-  lambdaisland/uri {:mvn/version "1.1.0"}
+  lambdaisland/uri {:mvn/version "1.1.0"
+                    :exclusions [org.clojure/data.json]}
 
   danlentz/clj-uuid {:mvn/version "0.1.9"}
   org.jsoup/jsoup {:mvn/version "1.12.1"}
   org.im4java/im4java {:mvn/version "1.4.0"}
-
   org.lz4/lz4-java {:mvn/version "1.7.1"}
-
   com.github.spullara.mustache.java/compiler {:mvn/version "0.9.6"}
   commons-io/commons-io {:mvn/version "2.6"}
   com.draines/postal {:mvn/version "2.0.3"

backend/src/uxbox/config.clj

@@ -25,13 +25,17 @@
    :database-uri "postgresql://127.0.0.1/uxbox"
    :database-username "uxbox"
    :database-password "uxbox"
+
+   :redis-uri "redis://redis/0"
    :media-directory "resources/public/media"
    :assets-directory "resources/public/static"
    :media-uri "http://localhost:6060/media/"
    :assets-uri "http://localhost:6060/static/"
-   :email-reply-to "no-reply@nodomain.com"
-   :email-from "no-reply@nodomain.com"
-   :smtp-enabled false
+
+   :sendmail-backend "console"
+   :sendmail-reply-to "no-reply@example.com"
+   :sendmail-from "no-reply@example.com"
+
    :allow-demo-users true
    :registration-enabled true
    :registration-domain-whitelist ""
@@ -44,19 +48,21 @@
 (s/def ::database-username (s/nilable ::us/string))
 (s/def ::database-password (s/nilable ::us/string))
 (s/def ::database-uri ::us/string)
+(s/def ::redis-uri ::us/string)
 (s/def ::assets-uri ::us/string)
 (s/def ::assets-directory ::us/string)
 (s/def ::media-uri ::us/string)
 (s/def ::media-directory ::us/string)
-(s/def ::email-reply-to ::us/email)
-(s/def ::email-from ::us/email)
+(s/def ::sendmail-backend ::us/string)
+(s/def ::sendmail-backend-apikey ::us/string)
+(s/def ::sendmail-reply-to ::us/email)
+(s/def ::sendmail-from ::us/email)
 (s/def ::smtp-host ::us/string)
 (s/def ::smtp-port ::us/integer)
 (s/def ::smtp-user (s/nilable ::us/string))
 (s/def ::smtp-password (s/nilable ::us/string))
 (s/def ::smtp-tls ::us/boolean)
 (s/def ::smtp-ssl ::us/boolean)
-(s/def ::smtp-enabled ::us/boolean)
 (s/def ::allow-demo-users ::us/boolean)
 (s/def ::registration-enabled ::us/boolean)
 (s/def ::registration-domain-whitelist ::us/string)
@@ -73,15 +79,16 @@
                    ::assets-uri
                    ::media-directory
                    ::media-uri
-                   ::email-reply-to
-                   ::email-from
+                   ::sendmail-reply-to
+                   ::sendmail-from
+                   ::sendmail-backend
+                   ::sendmail-backend-apikey
                    ::smtp-host
                    ::smtp-port
                    ::smtp-user
                    ::smtp-password
                    ::smtp-tls
                    ::smtp-ssl
-                   ::smtp-enabled
                    ::debug-humanize-transit
                    ::allow-demo-users
                    ::registration-enabled]))

backend/src/uxbox/emails.clj

@@ -34,16 +34,16 @@
 (defn send!
   "Schedule the email for sending."
   ([email context] (send! db/pool email context))
-  ([conn email context]
-   (us/verify fn? email)
+  ([conn email-factory context]
+   (us/verify fn? email-factory)
    (us/verify map? context)
-   (let [defaults {:from (:email-from cfg/config)
-                   :reply-to (:email-reply-to cfg/config)}
-         data (->> (merge defaults context)
-                   (email))]
+   (let [defaults {:from (:sendmail-from cfg/config)
+                   :reply-to (:sendmail-reply-to cfg/config)}
+         data (merge defaults context)
+         email (email-factory data)]
      (tasks/schedule! conn {:name "sendmail"
                             :delay 0
-                            :props data}))))
+                            :props email}))))
 
 ;; --- Emails
 
@@ -62,4 +62,3 @@
 (def password-recovery
   "A password recovery notification email."
   (emails/build ::password-recovery default-context))
-

backend/src/uxbox/redis.clj

@@ -0,0 +1,49 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) 2019 Andrey Antukh <niwi@niwi.nz>
+
+(ns uxbox.redis
+  (:refer-clojure :exclude [run!])
+  (:require
+   [clojure.tools.logging :as log]
+   [lambdaisland.uri :refer [uri]]
+   [mount.core :as mount :refer [defstate]]
+   [promesa.core :as p]
+   [uxbox.common.exceptions :as ex]
+   [uxbox.config :as cfg]
+   [uxbox.core :refer [system]]
+   [uxbox.util.redis :as redis]
+   [uxbox.util.data :as data]
+   [vertx.util :as vu])
+  (:import
+   java.lang.AutoCloseable))
+
+;; --- Connection Handling & State
+
+(defn- create-client
+  [config]
+  (let [uri (:redis-uri config "redis://redis/0")]
+    (log/info "creating redis client with" uri)
+    (redis/client uri)))
+
+(defstate client
+  :start (create-client cfg/config)
+  :stop (.close ^AutoCloseable client))
+
+(defstate conn
+  :start (redis/connect client)
+  :stop (.close ^AutoCloseable conn))
+
+;; --- API FORWARD
+
+(defmacro with-conn
+  [& args]
+  `(redis/with-conn ~@args))
+
+(defn run!
+  [conn cmd params]
+  (let [ctx (vu/get-or-create-context system)]
+    (-> (redis/run! conn cmd params)
+        (vu/handle-on-context ctx))))

backend/src/uxbox/tasks/impl.clj

@@ -117,8 +117,7 @@
           (p/then decode-task-row)
           (p/then (fn [item]
                     (when item
-                      (log/info "Execute task" (:name item)
-                                "with props" (pr-str (:props item)))
+                      (log/info "Execute task" (:name item))
                       (-> (p/do! (handle-task tasks item))
                           (p/handle (fn [v e]
                                       (if e

backend/src/uxbox/tasks/sendmail.clj

@@ -5,33 +5,24 @@
 ;; This Source Code Form is "Incompatible With Secondary Licenses", as
 ;; defined by the Mozilla Public License, v. 2.0.
 ;;
-;; Copyright (c) 2020 Andrey Antukh <niwi@niwi.nz>
+;; Copyright (c) 2020 UXBOX Labs SL
 
 (ns uxbox.tasks.sendmail
-  "Email sending jobs."
   (:require
+   [clojure.data.json :as json]
    [clojure.tools.logging :as log]
-   [cuerdas.core :as str]
    [postal.core :as postal]
-   [vertx.util :as vu]
    [promesa.core :as p]
+   [uxbox.common.data :as d]
    [uxbox.common.exceptions :as ex]
    [uxbox.config :as cfg]
-   [uxbox.core :refer [system]]
-   [uxbox.util.blob :as blob]))
+   [uxbox.util.http :as http]
+   [vertx.util :as vu]))
 
-(defn- get-smtp-config
-  [config]
-  {:host (:smtp-host config)
-   :port (:smtp-port config)
-   :user (:smtp-user config)
-   :pass (:smtp-password config)
-   :ssl (:smtp-ssl config)
-   :tls (:smtp-tls config)
-   :enabled (:smtp-enabled config)})
+(defmulti sendmail (fn [config email] (:sendmail-backend config)))
 
-(defn- send-email-to-console
-  [email]
+(defmethod sendmail "console"
+  [config email]
   (let [out (with-out-str
               (println "email console dump:")
               (println "******** start email" (:id email) "**********")
@@ -40,28 +31,73 @@
               (println " reply-to: " (:reply-to email))
               (println " subject: " (:subject email))
               (println " content:")
-              (doseq [item (rest (:body email))]
-                (when (str/starts-with? (:type item) "text/plain")
-                  (println (:content item))))
+              (doseq [item (:content email)]
+                (when (= (:type item) "text/plain")
+                  (println (:value item))))
               (println "******** end email "(:id email) "**********"))]
-    (log/info out)
-    {:error :SUCCESS}))
+    (log/info out)))
 
-(defn send-email
+(defmethod sendmail "sendgrid"
+  [config email]
+  (let [apikey  (:sendmail-backend-apikey config)
+        dest    (mapv #(array-map :email %) (:to email))
+        params  {:personalizations [{:to dest
+                                     :subject (:subject email)}]
+                 :from {:email (:from email)}
+                 :reply_to {:email (:reply-to email)}
+                 :content (:content email)}
+        headers {"Authorization" (str "Bearer " apikey)
+                 "Content-Type" "application/json"}
+        body    (json/write-str params)]
+    (-> (http/send! {:method :post
+                     :headers headers
+                     :uri "https://api.sendgrid.com/v3/mail/send"
+                     :body body})
+        (p/handle
+         (fn [response error]
+           (cond
+             error
+             (log/error "Error on sending email to sendgrid:" (pr-str error))
+
+             (= 202 (:status response))
+             nil
+
+             :else
+             (log/error "Unexpected status from sendgrid:" (pr-str response))))))))
+
+(defn- get-smtp-config
+  [config]
+  {:host (:smtp-host config)
+   :port (:smtp-port config)
+   :user (:smtp-user config)
+   :pass (:smtp-password config)
+   :ssl (:smtp-ssl config)
+   :tls (:smtp-tls config)})
+
+(defn- email->postal
   [email]
+  {:from (:from email)
+   :to (:to email)
+   :subject (:subject email)
+   :body (d/concat [:alternative]
+                   (map (fn [{:keys [type value]}]
+                          {:type (str type "; charset=utf-8")
+                           :content value})
+                        (:content email)))})
+
+(defmethod sendmail "smtp"
+  [config email]
   (vu/blocking
-   (let [config (get-smtp-config cfg/config)
-         result (if (:enabled config)
-                  (postal/send-message config email)
-                  (send-email-to-console email))]
+   (let [config (get-smtp-config config)
+         email  (email->postal email)
+         result (postal/send-message config email)]
      (when (not= (:error result) :SUCCESS)
        (ex/raise :type :sendmail-error
                  :code :email-not-sent
-                 :context result))
-     nil)))
+                 :context result)))))
 
 (defn handler
   {:uxbox.tasks/name "sendmail"}
   [{:keys [props] :as task}]
-  (send-email props))
+  (sendmail cfg/config props))
 

backend/src/uxbox/util/emails.clj

@@ -30,7 +30,7 @@
        "eol = ('\\n' | '\\r\\n'); "))
 
 (def ^:private parse-fn (insta/parser grammar))
-(def ^:private email-path "emails/%(lang)s/%(id)s.mustache")
+(def ^:private email-path "emails/%(id)s/%(lang)s.mustache")
 
 (defn- parse-template
   [content]
@@ -49,7 +49,8 @@
 (s/def ::body-html string?)
 
 (s/def ::parsed-email
-  (s/keys :req-un [::subject ::body-html ::body-html]))
+  (s/keys :req-un [::subject ::body-text]
+          :opt-un [::body-html]))
 
 (defn- build-base-email
   [data context]
@@ -59,11 +60,11 @@
               :hint "Seems like the email template has invalid data."
               :contex data))
   {:subject (:subject data)
-   :body [:alternative
-          {:type "text/plain; charset=utf-8"
-           :content (:body-text data)}
-          {:type "text/html; charset=utf-8"
-           :content (:body-html data)}]})
+   :content (cond-> []
+              (:body-text data) (conj {:type "text/plain"
+                                       :value (:body-text data)})
+              (:body-html data) (conj {:type "text/html"
+                                       :value (:body-html data)}))})
 
 (defn- impl-build-email
   [id context]
@@ -102,6 +103,6 @@
                    :hint "seems like the template is wrong or does not exists."
                    ::id id))
        (cond-> (assoc email :id (name id))
-         (:to context) (assoc :to (:to context))
+         (:to context) (assoc :to [(:to context)])
          (:from context) (assoc :from (:from context))
          (:reply-to context) (assoc :reply-to (:reply-to context)))))))

backend/src/uxbox/util/http.clj

@@ -5,4 +5,15 @@
 ;; Copyright (c) 2019 Andrey Antukh <niwi@niwi.nz>
 
 (ns uxbox.util.http
-  "Http related helpers.")
+  "Http client abstraction layer."
+  (:require
+   [promesa.core :as p]
+   [promesa.exec :as px]
+   [java-http-clj.core :as http]))
+
+(def default-client
+  (delay (http/build-client {:executor @px/default-executor})))
+
+(defn send!
+  [req]
+  (http/send-async req {:client @default-client :as :string}))

backend/src/uxbox/util/redis.clj

@@ -0,0 +1,99 @@
+;; This Source Code Form is subject to the terms of the Mozilla Public
+;; License, v. 2.0. If a copy of the MPL was not distributed with this
+;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;;
+;; Copyright (c) 2019 Andrey Antukh <niwi@niwi.nz>
+
+(ns uxbox.util.redis
+  "Asynchronous posgresql client."
+  (:refer-clojure :exclude [get set run!])
+  (:require
+   [promesa.core :as p])
+  (:import
+   io.lettuce.core.RedisClient
+   io.lettuce.core.RedisURI
+   io.lettuce.core.codec.StringCodec
+   io.lettuce.core.api.async.RedisAsyncCommands
+   io.lettuce.core.api.StatefulRedisConnection
+   ))
+
+(defrecord Client [conn uri]
+  java.lang.AutoCloseable
+  (close [_]
+    (.shutdown ^RedisClient conn)))
+
+(defrecord Connection [cmd conn]
+  java.lang.AutoCloseable
+  (close [_]
+    (.close ^StatefulRedisConnection conn)))
+
+(defn client
+  [uri]
+  (->Client (RedisClient/create) (RedisURI/create uri)))
+
+(defn connect
+  [client]
+  (let [^RedisURI uri (:uri client)
+        ^RedisClient conn (:conn client)
+        ^StatefulRedisConnection conn' (.connect conn StringCodec/UTF8 uri)]
+    (->Connection (.async conn') conn')))
+
+(declare impl-with-conn)
+
+(defmacro with-conn
+  [[csym sym] & body]
+  `(impl-with-conn ~sym (fn [~csym] ~@body)))
+
+(defn impl-with-conn
+  [client f]
+  (let [^RedisURI uri (:uri client)
+        ^RedisClient conn (:conn client)]
+    (-> (.connectAsync conn StringCodec/UTF8 uri)
+        (p/then (fn [^StatefulRedisConnection conn]
+                  (let [cmd  (.async conn)
+                        conn (->Connection cmd conn)]
+                    (-> (p/do! (f conn))
+                        (p/handle (fn [v e]
+                                    (.close conn)
+                                    (if e
+                                      (throw e)
+                                      v))))))))))
+
+(defn- resolve-to-bool
+  [v]
+  (if (= v 1)
+    true
+    false))
+
+(defmulti impl-run (fn [conn cmd parmas] cmd))
+
+(defn run!
+  [conn cmd params]
+  (let [^RedisAsyncCommands conn (:cmd conn)]
+    (impl-run conn cmd params)))
+
+(defmethod impl-run :get
+  [conn _ {:keys [key]}]
+  (.get ^RedisAsyncCommands conn ^String key))
+
+(defmethod impl-run :set
+  [conn _ {:keys [key val]}]
+  (.set ^RedisAsyncCommands conn ^String key ^String val))
+
+(defmethod impl-run :smembers
+  [conn _ {:keys [key]}]
+  (-> (.smembers ^RedisAsyncCommands conn ^String key)
+      (p/then' #(into #{} %))))
+
+(defmethod impl-run :sadd
+  [conn _ {:keys [key val]}]
+  (let [keys (into-array String [val])]
+    (-> (.sadd ^RedisAsyncCommands conn ^String key ^"[S;" keys)
+        (p/then resolve-to-bool))))
+
+(defmethod impl-run :srem
+  [conn _ {:keys [key val]}]
+  (let [keys (into-array String [val])]
+    (-> (.srem ^RedisAsyncCommands conn ^String key ^"[S;" keys)
+        (p/then resolve-to-bool))))
+

backend/tests/user.clj

@@ -23,6 +23,7 @@
    [promesa.exec :as px]
    [uxbox.migrations]
    [uxbox.db :as db]
+   [uxbox.redis :as rd]
    [uxbox.util.storage :as st]
    [uxbox.util.time :as tm]
    [uxbox.util.blob :as blob]

docker/devenv/Dockerfile

@@ -32,6 +32,7 @@ RUN set -ex; \
         imagemagick \
         webp \
         jq \
+        redis-tools \
     ; \
     rm -rf /var/lib/apt/lists/*;
 

docker/devenv/docker-compose.yaml

@@ -1,4 +1,4 @@
-version: '3'
+version: "3"
 
 networks:
   default:
@@ -15,13 +15,16 @@ services:
   main:
     privileged: true
     image: "uxbox-devenv"
-    hostname: 'uxbox-devenv-main'
-    container_name: 'uxbox-devenv-main'
+    hostname: "uxbox-devenv-main"
+    container_name: "uxbox-devenv-main"
     command: "/home/uxbox/init.sh"
     stop_signal: SIGINT
+
     depends_on:
       - postgres
       - smtp
+      - redis
+
     volumes:
       - "user_data:/home/uxbox/local"
       - "${HOME}/.m2:/home/uxbox/.m2"
@@ -39,9 +42,12 @@ services:
       - UXBOX_DATABASE_URI=postgresql://postgres/uxbox
       - UXBOX_DATABASE_USERNAME=uxbox
       - UXBOX_DATABASE_PASSWORD=uxbox
+      - UXBOX_SENDMAIL_BACKEND=smtp
+      - UXBOX_SMTP_HOST=smtp
+      - UXBOX_SMTP_PORT=25
 
   smtp:
-    container_name: 'uxbox-devenv-smtp'
+    container_name: "uxbox-devenv-smtp"
     image: mwader/postfix-relay
     restart: always
     environment:
@@ -51,8 +57,8 @@ services:
   postgres:
     image: postgres:12
     command: postgres -c config_file=/etc/postgresql.conf
-    hostname: 'uxbox-devenv-postgres'
-    container_name: 'uxbox-devenv-postgres'
+    hostname: "uxbox-devenv-postgres"
+    container_name: "uxbox-devenv-postgres"
     restart: always
     stop_signal: SIGINT
     ports:
@@ -66,3 +72,12 @@ services:
       - ./files/postgresql.conf:/etc/postgresql.conf
       - ./files/postgresql_init.sql:/docker-entrypoint-initdb.d/init.sql
       - postgres_data:/var/lib/postgresql/data
+
+  redis:
+    image: redis:6.0-rc3
+    hostname: "uxbox-devenv-redis"
+    container_name: "uxbox-devenv-redis"
+    restart: always
+
+    ports:
+      - 6379:6379