mirror of
https://github.com/penpot/penpot.git
synced 2026-11-13 13:31:24 +00:00
8f91f7cd4e
SSE endpoints always encoded event payloads in transit, ignoring request content negotiation. WebSocket notifications always used transit as well, with no way to request plain JSON. Extract the response format negotiation (Accept header and _fmt query param) into the shared app.http.content-negotiation helper and reuse it from the response formatting middleware, the SSE response builder and the WebSocket notifications endpoint. The _fmt=json parameter takes precedence over Accept, transit stays the default, and unknown formats fall back to transit. SSE event payloads are now encoded as transit (default) or plain JSON depending on the request, while the text/event-stream transport remains unchanged. The WebSocket format is negotiated once on the upgrade request and applies bidirectionally: outbound notification messages use the negotiated encoder and inbound client messages the matching decoder, with a normalization step converting JSON messages (:type to keyword, entity ids to UUID) into the same shape produced by transit. The generic websocket listener now accepts encode/decode functions as options (defaulting to the historical transit codecs) and exposes the resolved options in the handler metadata under ::options for testing and introspection. The notifications http-handler is now public so tests can cover the wiring. On the frontend, stream requests accept a :response-format :json option that switches the Accept header and decodes each event with the plain JSON decoder. Add unit tests for negotiate-format, SSE response encoding and the WebSocket codec wiring (JSON via Accept header or _fmt param, transit default for unsignaled requests). Closes #11409 AI-assisted-by: glm-5.3-flash AI-assisted-by: muse-spark-1.3-contributor
385 lines
14 KiB
Clojure
385 lines
14 KiB
Clojure
;; This Source Code Form is subject to the terms of the Mozilla Public
|
||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||
;;
|
||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||
|
||
(ns app.main.repo
|
||
(:require
|
||
[app.common.data :as d]
|
||
[app.common.exceptions :as ex]
|
||
[app.common.json :as json]
|
||
[app.common.logging :as log]
|
||
[app.common.time :as ct]
|
||
[app.common.transit :as t]
|
||
[app.common.uri :as u]
|
||
[app.config :as cf]
|
||
[app.main.data.event :as-alias ev]
|
||
[app.util.http :as http]
|
||
[app.util.sse :as sse]
|
||
[beicon.v2.core :as rx]
|
||
[cuerdas.core :as str]))
|
||
|
||
(log/set-level! :info)
|
||
|
||
;; -- Retry helpers -----------------------------------------------------------
|
||
|
||
(def retryable-types
|
||
"Set of error types that are considered transient and safe to retry
|
||
for idempotent (GET) requests. Also the single source of truth for the
|
||
transient transport classification consumed by `persistence/transient-error?`
|
||
and `errors/environment-error-types`: extend this set (never a copy)
|
||
when a new retryable transport failure appears."
|
||
#{:network ; js/fetch network-level failure
|
||
:bad-gateway ; 502
|
||
:service-unavailable ; 503
|
||
:gateway-error ; 504 and the Cloudflare 52x family
|
||
:offline}) ; status 0 (browser offline)
|
||
|
||
(defn retryable-error?
|
||
"Return true when `error` represents a transient failure that is safe
|
||
to retry. Only errors whose `ex-data` `:type` belongs to
|
||
`retryable-types` qualify."
|
||
[error]
|
||
(contains? retryable-types (:type (ex-data error))))
|
||
|
||
(def default-retry-config
|
||
"Default configuration for the retry mechanism on idempotent requests."
|
||
{:max-retries 3
|
||
:base-delay-ms 1000})
|
||
|
||
(defn with-retry
|
||
"Wrap `observable-fn` (a zero-arg function returning an Observable) so
|
||
that retryable errors are retried up to `:max-retries` times with
|
||
exponential back-off. Non-retryable errors propagate immediately.
|
||
|
||
Accepts an optional `config` map with:
|
||
:max-retries – maximum number of retries (default 3)
|
||
:base-delay-ms – base delay in ms; doubles each attempt (default 1000)"
|
||
([observable-fn]
|
||
(with-retry observable-fn default-retry-config))
|
||
([observable-fn config]
|
||
(with-retry observable-fn config 0))
|
||
([observable-fn config attempt]
|
||
(let [{:keys [max-retries base-delay-ms]} (merge default-retry-config config)]
|
||
(->> (observable-fn)
|
||
(rx/catch
|
||
(fn [cause]
|
||
(if (and (retryable-error? cause)
|
||
(< attempt max-retries))
|
||
;; bit-shift-left 1 N is equivalent to 2^N: shift the bits of the
|
||
;; number 1 to the left N positions (e.g. 1 -> 2 -> 4 -> 8 -> 16),
|
||
;; producing exponential backoff delays of 1x, 2x, 4x, 8x, 16x.
|
||
(let [delay-ms (* base-delay-ms (bit-shift-left 1 attempt))]
|
||
(log/wrn :hint "retrying request"
|
||
:attempt (inc attempt)
|
||
:delay delay-ms
|
||
:error (ex-message cause))
|
||
(->> (rx/timer delay-ms)
|
||
(rx/mapcat (fn [_] (with-retry observable-fn config (inc attempt))))))
|
||
(rx/throw cause))))))))
|
||
|
||
;; -- Response handling -------------------------------------------------------
|
||
|
||
(def ^:private gateway-statuses
|
||
"Statuses raised by an edge proxy when the request never reached the
|
||
backend or its answer never came back."
|
||
#{504 ; gateway timeout
|
||
520 ; cloudflare: unknown origin error
|
||
521 ; cloudflare: origin is down
|
||
522 ; cloudflare: connection timed out
|
||
523 ; cloudflare: origin unreachable
|
||
524}) ; cloudflare: origin timed out
|
||
|
||
(def ^:private max-body-excerpt
|
||
"Number of characters kept from a body that we could not interpret."
|
||
500)
|
||
|
||
(defn- body-excerpt
|
||
"Keeps the head of an uninterpretable body. Error pages from proxies run
|
||
into tens of kilobytes and the excerpt is enough to recognize them."
|
||
[body]
|
||
(if (and (string? body) (> (count body) max-body-excerpt))
|
||
(str (subs body 0 max-body-excerpt) "...")
|
||
body))
|
||
|
||
(defn handle-response
|
||
[{:keys [status body headers uri] :as response}]
|
||
(cond
|
||
(= 204 status)
|
||
;; We need to send "something" so the streams listening downstream can act
|
||
(rx/of nil)
|
||
|
||
(= 502 status)
|
||
(rx/throw (ex-info "http error" {:type :bad-gateway}))
|
||
|
||
(and (= 503 status)
|
||
(= :nitrate-unavailable (:type body)))
|
||
(rx/throw (ex-info "http error" {:type :nitrate-unavailable}))
|
||
|
||
(and (= 503 status)
|
||
(= :nitrate-not-configured (:type body)))
|
||
(rx/throw (ex-info "http error" {:type :nitrate-not-configured}))
|
||
|
||
(= 503 status)
|
||
(rx/throw (ex-info "http error" {:type :service-unavailable}))
|
||
|
||
(= 0 (:status response))
|
||
(rx/throw (ex-info "http error" {:type :offline}))
|
||
|
||
(= 200 status)
|
||
(rx/of body)
|
||
|
||
(= 413 status)
|
||
(rx/throw (ex-info "http error"
|
||
{:type :validation
|
||
:code :request-body-too-large}))
|
||
|
||
(and (= status 403)
|
||
(or (= "cloudflare" (get headers "server"))
|
||
(= "challenge" (get headers "cf-mitigated"))))
|
||
(rx/throw (ex-info "http error"
|
||
{:type :authorization
|
||
:code :challenge-required}))
|
||
|
||
(and (>= status 400) (map? body))
|
||
(rx/throw (ex-info "http error" (assoc body :uri uri :status status)))
|
||
|
||
;; The branches below handle responses the backend never wrote: the
|
||
;; body carries no error code, so the status is all we can read.
|
||
|
||
(contains? gateway-statuses status)
|
||
(rx/throw
|
||
(ex/error :type :gateway-error
|
||
:code :gateway-error
|
||
:hint "the request did not reach the backend"
|
||
:uri uri
|
||
:status status))
|
||
|
||
(= 429 status)
|
||
(rx/throw
|
||
(ex/error :type :rate-limit
|
||
:code :rate-limited
|
||
:hint "too many requests"
|
||
:uri uri
|
||
:status status))
|
||
|
||
(>= status 400)
|
||
(rx/throw
|
||
(ex/error :type :unexpected-response
|
||
:code :unexpected-error-response
|
||
:hint "unexpected error response"
|
||
:uri uri
|
||
:status status
|
||
:headers headers
|
||
:data (body-excerpt body)))
|
||
|
||
:else
|
||
(rx/throw
|
||
(ex/error :type :internal
|
||
:code :unable-to-process-repository-response
|
||
:hint "unable to process repository response"
|
||
:uri uri
|
||
:status status
|
||
:headers headers
|
||
:data (body-excerpt body)))))
|
||
|
||
(def default-options
|
||
{:update-file {:query-params [:id]}
|
||
:get-raw-file {:rename-to :get-file :raw-transit? true}
|
||
|
||
:create-file-object-thumbnail
|
||
{:query-params [:file-id :object-id :tag]
|
||
:form-data? true}
|
||
|
||
:create-file-thumbnail
|
||
{:query-params [:file-id :revn]
|
||
:form-data? true}
|
||
|
||
::sse/export-binfile
|
||
{:stream? true}
|
||
|
||
::sse/clone-template
|
||
{:stream? true}
|
||
|
||
::sse/import-binfile
|
||
{:stream? true}
|
||
|
||
::sse/permanently-delete-team-files
|
||
{:stream? true}
|
||
|
||
::sse/restore-deleted-team-files
|
||
{:stream? true}
|
||
|
||
:export-binfile {:response-type :blob}
|
||
|
||
:retrieve-list-of-builtin-templates {:query-params :all}})
|
||
|
||
(defn- send!
|
||
[id params options]
|
||
(let [{:keys [response-type
|
||
stream?
|
||
response-format
|
||
form-data?
|
||
raw-transit?
|
||
query-params
|
||
rename-to]}
|
||
(-> (get default-options id)
|
||
(merge options))
|
||
|
||
decode-fn
|
||
(if raw-transit?
|
||
http/conditional-error-decode-transit
|
||
http/conditional-decode-transit)
|
||
|
||
id (or rename-to id)
|
||
nid (name id)
|
||
method (cond
|
||
(= query-params :all) :get
|
||
(str/starts-with? nid "get-") :get
|
||
:else :post)
|
||
|
||
response-type
|
||
(d/nilv response-type :text)
|
||
|
||
accept
|
||
(if (and stream? (= response-format :json))
|
||
"application/json,text/event-stream,*/*"
|
||
"application/transit+json,text/event-stream,*/*")
|
||
|
||
request
|
||
{:method method
|
||
:uri (u/join cf/public-uri "api/main/methods/" nid)
|
||
:credentials "include"
|
||
:headers {"accept" accept
|
||
"x-external-session-id" (cf/external-session-id)
|
||
"x-session-id" (str cf/session-id)
|
||
"x-event-origin" (::ev/origin (meta params))}
|
||
:body (when (= method :post)
|
||
(if form-data?
|
||
(http/form-data params)
|
||
(http/transit-data params)))
|
||
:query (if (= method :get)
|
||
params
|
||
(if query-params
|
||
(select-keys params query-params)
|
||
nil))
|
||
:response-type
|
||
(if stream? nil response-type)}
|
||
|
||
tpoint
|
||
(ct/tpoint-ms)]
|
||
|
||
(log/trc :hint "make request" :id id)
|
||
|
||
(let [make-request
|
||
(fn []
|
||
(->> (http/fetch request)
|
||
(rx/map http/response->map)
|
||
(rx/mapcat (fn [{:keys [headers body] :as response}]
|
||
(log/trc :hint "response received" :id id :elapsed (tpoint))
|
||
|
||
(let [ctype (get headers "content-type")
|
||
response-stream? (str/starts-with? ctype "text/event-stream")
|
||
tpoint (ct/tpoint-ms)]
|
||
|
||
(when (and response-stream? (not stream?))
|
||
(ex/raise :type :assertion
|
||
:code :unexpected-response
|
||
:hint "expected normal response, received sse stream"
|
||
:uri (:uri response)
|
||
:status (:status response)))
|
||
|
||
(if response-stream?
|
||
(-> (sse/create-stream body)
|
||
(sse/read-stream (if (= response-format :json)
|
||
json/decode
|
||
t/decode-str)))
|
||
|
||
(->> response
|
||
(http/process-response-type response-type)
|
||
(rx/map decode-fn)
|
||
(rx/tap (fn [_]
|
||
(log/trc :hint "response decoded" :id id :elapsed (tpoint))))
|
||
(rx/mapcat handle-response))))))))]
|
||
|
||
;; Idempotent (GET) requests are automatically retried on
|
||
;; transient network / server errors. Mutations are never
|
||
;; retried to avoid unintended side-effects.
|
||
(if (= :get method)
|
||
(with-retry make-request)
|
||
(make-request)))))
|
||
|
||
(defmulti cmd! (fn [id _] id))
|
||
|
||
(defmethod cmd! :default
|
||
[id params]
|
||
(send! id params nil))
|
||
|
||
(defmethod cmd! :login-with-oidc
|
||
[_ params]
|
||
(let [uri (u/join cf/public-uri "api/auth/oidc")]
|
||
(->> (http/send! {:method :post
|
||
:uri uri
|
||
:credentials "include"
|
||
:headers {"x-external-session-id" (cf/external-session-id)
|
||
"x-event-origin" (::ev/origin (meta params))}
|
||
:query params})
|
||
(rx/map http/conditional-decode-transit)
|
||
(rx/mapcat handle-response))))
|
||
|
||
(defn- send-export
|
||
[{:keys [blob?] :as params}]
|
||
(->> (http/send! {:method :post
|
||
:uri (u/join cf/public-uri "api/export")
|
||
:body (http/transit-data (dissoc params :blob?))
|
||
:headers {"x-external-session-id" (cf/external-session-id)
|
||
"x-event-origin" (::ev/origin (meta params))}
|
||
:credentials "include"
|
||
:response-type (if blob? :blob :text)})
|
||
(rx/map http/conditional-decode-transit)
|
||
(rx/mapcat handle-response)))
|
||
|
||
(defmethod cmd! :export
|
||
[_ params]
|
||
(let [default {:wait false :blob? false}]
|
||
(send-export (merge default params))))
|
||
|
||
(defmethod cmd! :create-export-job
|
||
[_ params]
|
||
(->> (http/send! {:method :post
|
||
:uri (u/join cf/public-uri "api/export/jobs")
|
||
:body (http/transit-data params)
|
||
:headers {"x-external-session-id" (cf/external-session-id)
|
||
"x-event-origin" (::ev/origin (meta params))}
|
||
:credentials "include"
|
||
:response-type :text})
|
||
(rx/map http/conditional-decode-transit)
|
||
(rx/mapcat handle-response)))
|
||
|
||
(defmethod cmd! :cancel-export-job
|
||
[_ {:keys [job-id]}]
|
||
(->> (http/send! {:method :delete
|
||
:uri (u/join cf/public-uri "api/export/jobs/" (str job-id))
|
||
:headers {"x-external-session-id" (cf/external-session-id)}
|
||
:credentials "include"
|
||
:response-type :text})
|
||
(rx/map http/conditional-decode-transit)
|
||
(rx/mapcat handle-response)))
|
||
|
||
(defn- multipart-upload
|
||
[id params]
|
||
(->> (http/send! {:method :post
|
||
:uri (u/join cf/public-uri "api/main/methods/" (name id))
|
||
:credentials "include"
|
||
:headers {"x-external-session-id" (cf/external-session-id)
|
||
"x-event-origin" (::ev/origin (meta params))}
|
||
:body (http/form-data params)})
|
||
(rx/map http/conditional-decode-transit)
|
||
(rx/mapcat handle-response)))
|
||
|
||
(defmethod cmd! :upload-file-media-object [id params] (multipart-upload id params))
|
||
(defmethod cmd! :upload-chunk [id params] (multipart-upload id params))
|
||
(defmethod cmd! :update-profile-photo [id params] (multipart-upload id params))
|
||
(defmethod cmd! :update-team-photo [id params] (multipart-upload id params))
|