mirror of
https://github.com/penpot/penpot.git
synced 2026-05-11 19:13:49 +00:00
🐛 Fix maximum call stack size exceeded in SSE read-stream (#9484)
The recursive `read-items` function in `app.util.sse/read-stream` caused a synchronous stack overflow when reading buffered stream data. Each `rx/mapcat` call chained another recursive invocation on the same call stack without yielding to the event loop. Replace the recursive pattern with an `rx/create`-based async pump that uses Promise `.then()` chaining, keeping the call stack depth constant regardless of stream size. Also add progress reporting with names and IDs during binfile export and import, and bump `eventsource-parser` dependency. Closes #9470 Signed-off-by: Andrey Antukh <niwi@niwi.nz>
This commit is contained in:
Andrey Antukh
GitHub
parent
f2c631b8b7
commit
ec0d692856
@ -281,7 +281,7 @@
|
||||
|
||||
thumbnails (bfc/get-file-object-thumbnails cfg file-id)]
|
||||
|
||||
(events/tap :progress {:section :file :id file-id})
|
||||
(events/tap :progress {:section :file :id file-id :name (:name file)})
|
||||
|
||||
(vswap! bfc/*state* update :files assoc file-id
|
||||
{:id file-id
|
||||
@ -301,6 +301,7 @@
|
||||
(write-entry! output path file))
|
||||
|
||||
(doseq [[index page-id] (d/enumerate pages)]
|
||||
|
||||
(let [path (str "files/" file-id "/pages/" page-id ".json")
|
||||
page (get pages-index page-id)
|
||||
objects (:objects page)
|
||||
@ -311,6 +312,8 @@
|
||||
|
||||
(write-entry! output path page)
|
||||
|
||||
(events/tap :progress {:section :page :id page-id :name (:name page) :file-id file-id})
|
||||
|
||||
(doseq [[shape-id shape] objects]
|
||||
(let [path (str "files/" file-id "/pages/" page-id "/" shape-id ".json")
|
||||
shape (assoc shape :page-id page-id)
|
||||
@ -323,6 +326,8 @@
|
||||
(doseq [{:keys [id] :as media} media]
|
||||
(let [path (str "files/" file-id "/media/" id ".json")
|
||||
media (encode-media media)]
|
||||
|
||||
(events/tap :progress {:section :media :id id :file-id file-id})
|
||||
(write-entry! output path media)))
|
||||
|
||||
(doseq [thumbnail thumbnails]
|
||||
@ -332,11 +337,13 @@
|
||||
data (-> data
|
||||
(assoc :media-id (:media-id thumbnail))
|
||||
(encode-file-thumbnail))]
|
||||
(events/tap :progress {:section :thumbnails :id (:object-id thumbnail) :file-id file-id})
|
||||
(write-entry! output path data)))
|
||||
|
||||
(doseq [[id component] components]
|
||||
(let [path (str "files/" file-id "/components/" id ".json")
|
||||
component (encode-component component)]
|
||||
(events/tap :progress {:section :component :id id :file-id file-id})
|
||||
(write-entry! output path component)))
|
||||
|
||||
(doseq [[id color] colors]
|
||||
@ -347,17 +354,20 @@
|
||||
(and (contains? color :path)
|
||||
(str/empty? (:path color)))
|
||||
(dissoc :path))]
|
||||
(events/tap :progress {:section :color :id id :file-id file-id})
|
||||
(write-entry! output path color)))
|
||||
|
||||
(doseq [[id object] typographies]
|
||||
(let [path (str "files/" file-id "/typographies/" id ".json")
|
||||
typography (encode-typography object)]
|
||||
(events/tap :progress {:section :typography :id id :file-id file-id})
|
||||
(write-entry! output path typography)))
|
||||
|
||||
(when (and tokens-lib
|
||||
(not (ctob/empty-lib? tokens-lib)))
|
||||
(let [path (str "files/" file-id "/tokens.json")
|
||||
encoded-tokens (encode-tokens-lib tokens-lib)]
|
||||
(events/tap :progress {:section :tokens-lib :file-id file-id})
|
||||
(write-entry! output path encoded-tokens)))))
|
||||
|
||||
(defn- export-files
|
||||
@ -600,6 +610,7 @@
|
||||
(let [object (->> (read-entry input entry)
|
||||
(decode-color)
|
||||
(validate-color))]
|
||||
(events/tap :progress {:section :color :id id :file-id file-id})
|
||||
(if (= id (:id object))
|
||||
(assoc result id object)
|
||||
result)))
|
||||
@ -631,6 +642,7 @@
|
||||
(clean-component-pre-decode)
|
||||
(decode-component)
|
||||
(clean-component-post-decode))]
|
||||
(events/tap :progress {:section :component :id id :file-id file-id})
|
||||
(if (= id (:id object))
|
||||
(assoc result id object)
|
||||
result)))
|
||||
@ -644,6 +656,7 @@
|
||||
(let [object (->> (read-entry input entry)
|
||||
(decode-typography)
|
||||
(validate-typography))]
|
||||
(events/tap :progress {:section :typography :id id :file-id file-id})
|
||||
(if (= id (:id object))
|
||||
(assoc result id object)
|
||||
result)))
|
||||
@ -653,6 +666,7 @@
|
||||
(defn- read-file-tokens-lib
|
||||
[{:keys [::bfc/input ::entries]} file-id]
|
||||
(when-let [entry (d/seek (match-tokens-lib-entry-fn file-id) entries)]
|
||||
(events/tap :progress {:section :tokens-lib :file-id file-id})
|
||||
(->> (read-plain-entry input entry)
|
||||
(decode-tokens-lib)
|
||||
(validate-tokens-lib))))
|
||||
@ -678,6 +692,7 @@
|
||||
(let [page (->> (read-entry input entry)
|
||||
(decode-page))
|
||||
page (dissoc page :options)]
|
||||
(events/tap :progress {:section :page :id id :file-id file-id})
|
||||
(when (= id (:id page))
|
||||
(let [objects (read-file-shapes cfg file-id id)]
|
||||
(assoc page :objects objects))))))
|
||||
@ -693,6 +708,7 @@
|
||||
(let [object (->> (read-entry input entry)
|
||||
(decode-file-thumbnail)
|
||||
(validate-file-thumbnail))]
|
||||
|
||||
(if (and (= frame-id (:frame-id object))
|
||||
(= page-id (:page-id object))
|
||||
(= tag (:tag object)))
|
||||
@ -733,8 +749,6 @@
|
||||
|
||||
(vswap! bfc/*state* update :index bfc/update-index media :id)
|
||||
|
||||
(events/tap :progress {:section :media :file-id file-id})
|
||||
|
||||
(doseq [item media]
|
||||
(let [params (-> item
|
||||
(update :id bfc/lookup-index)
|
||||
@ -742,6 +756,8 @@
|
||||
(d/update-when :media-id bfc/lookup-index)
|
||||
(d/update-when :thumbnail-id bfc/lookup-index))]
|
||||
|
||||
(events/tap :progress {:section :media :id (:id params) :file-id file-id})
|
||||
|
||||
(l/dbg :hint "inserting media object"
|
||||
:file-id (str file-id')
|
||||
:id (str (:id params))
|
||||
@ -753,8 +769,6 @@
|
||||
(db/insert! conn :file-media-object params
|
||||
::db/on-conflict-do-nothing? (::bfc/overwrite cfg))))
|
||||
|
||||
(events/tap :progress {:section :thumbnails :file-id file-id})
|
||||
|
||||
(doseq [item thumbnails]
|
||||
(let [media-id (bfc/lookup-index (:media-id item))
|
||||
object-id (-> (assoc item :file-id file-id')
|
||||
@ -769,6 +783,8 @@
|
||||
:media-id (str media-id)
|
||||
::l/sync? true)
|
||||
|
||||
(events/tap :progress {:section :thumbnail :file-id file-id :object-id object-id})
|
||||
|
||||
(db/insert! conn :file-tagged-object-thumbnail params
|
||||
::db/on-conflict-do-nothing? true)))
|
||||
|
||||
|
||||
@ -72,7 +72,7 @@
|
||||
"concurrently": "^9.2.1",
|
||||
"date-fns": "^4.1.0",
|
||||
"esbuild": "^0.28.0",
|
||||
"eventsource-parser": "^3.0.6",
|
||||
"eventsource-parser": "^3.0.8",
|
||||
"express": "^5.1.0",
|
||||
"fancy-log": "^2.0.0",
|
||||
"getopts": "^2.3.0",
|
||||
|
||||
54
frontend/pnpm-lock.yaml
generated
54
frontend/pnpm-lock.yaml
generated
@ -83,8 +83,8 @@ importers:
|
||||
specifier: ^0.28.0
|
||||
version: 0.28.0
|
||||
eventsource-parser:
|
||||
specifier: ^3.0.6
|
||||
version: 3.0.6
|
||||
specifier: ^3.0.8
|
||||
version: 3.0.8
|
||||
express:
|
||||
specifier: ^5.1.0
|
||||
version: 5.2.1
|
||||
@ -1353,12 +1353,16 @@ packages:
|
||||
'@hapi/topo@6.0.2':
|
||||
resolution: {integrity: sha512-KR3rD5inZbGMrHmgPxsJ9dbi6zEK+C3ZwUwTa+eMwWLz7oijWUTWD2pMSNNYJAU6Qq+65NkxXjqHr/7LM2Xkqg==}
|
||||
|
||||
'@humanfs/core@0.19.1':
|
||||
resolution: {integrity: sha512-5DyQ4+1JEUzejeK1JGICcideyfUbGixgS9jNgex5nqkW+cY7WZhxBigmieN5Qnw9ZosSNVC9KQKyb+GUaGyKUA==}
|
||||
'@humanfs/core@0.19.2':
|
||||
resolution: {integrity: sha512-UhXNm+CFMWcbChXywFwkmhqjs3PRCmcSa/hfBgLIb7oQ5HNb1wS0icWsGtSAUNgefHeI+eBrA8I1fxmbHsGdvA==}
|
||||
engines: {node: '>=18.18.0'}
|
||||
|
||||
'@humanfs/node@0.16.7':
|
||||
resolution: {integrity: sha512-/zUx+yOsIrG4Y43Eh2peDeKCxlRt/gET6aHfaKpuq267qXdYDFViVHfMaLyygZOnl0kGWxFIgsBy8QFuTLUXEQ==}
|
||||
'@humanfs/node@0.16.8':
|
||||
resolution: {integrity: sha512-gE1eQNZ3R++kTzFUpdGlpmy8kDZD/MLyHqDwqjkVQI0JMdI1D51sy1H958PNXYkM2rAac7e5/CnIKZrHtPh3BQ==}
|
||||
engines: {node: '>=18.18.0'}
|
||||
|
||||
'@humanfs/types@0.15.0':
|
||||
resolution: {integrity: sha512-ZZ1w0aoQkwuUuC7Yf+7sdeaNfqQiiLcSRbfI08oAxqLtpXQr9AIVX7Ay7HLDuiLYAaFPu8oBYNq/QIi9URHJ3Q==}
|
||||
engines: {node: '>=18.18.0'}
|
||||
|
||||
'@humanwhocodes/module-importer@1.0.1':
|
||||
@ -2232,6 +2236,9 @@ packages:
|
||||
'@types/estree@1.0.8':
|
||||
resolution: {integrity: sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==}
|
||||
|
||||
'@types/estree@1.0.9':
|
||||
resolution: {integrity: sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==}
|
||||
|
||||
'@types/json-schema@7.0.15':
|
||||
resolution: {integrity: sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==}
|
||||
|
||||
@ -2468,8 +2475,8 @@ packages:
|
||||
ajv:
|
||||
optional: true
|
||||
|
||||
ajv@6.14.0:
|
||||
resolution: {integrity: sha512-IWrosm/yrn43eiKqkfkHis7QioDleaXQHdDVPKg0FSwwd/DuvyX79TZnFOnYpB7dcsFAMmtFztZuXPDvSePkFw==}
|
||||
ajv@6.15.0:
|
||||
resolution: {integrity: sha512-fgFx7Hfoq60ytK2c7DhnF8jIvzYgOMxfugjLOSMHjLIPgenqa7S7oaagATUq99mV6IYvN2tRmC0wnTYX6iPbMw==}
|
||||
|
||||
ajv@8.12.0:
|
||||
resolution: {integrity: sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA==}
|
||||
@ -3451,8 +3458,8 @@ packages:
|
||||
resolution: {integrity: sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==}
|
||||
engines: {node: '>=0.8.x'}
|
||||
|
||||
eventsource-parser@3.0.6:
|
||||
resolution: {integrity: sha512-Vo1ab+QXPzZ4tCa8SwIHJFaSzy4R6SHf7BY79rFBDf0idraZWAkYrDjDj8uWaSm3S2TK+hJ7/t1CEmZ7jXw+pg==}
|
||||
eventsource-parser@3.0.8:
|
||||
resolution: {integrity: sha512-70QWGkr4snxr0OXLRWsFLeRBIRPuQOvt4s8QYjmUlmlkyTZkRqS7EDVRZtzU3TiyDbXSzaOeF0XUKy8PchzukQ==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
|
||||
execa@8.0.1:
|
||||
@ -7077,7 +7084,7 @@ snapshots:
|
||||
|
||||
'@eslint/eslintrc@3.3.5':
|
||||
dependencies:
|
||||
ajv: 6.14.0
|
||||
ajv: 6.15.0
|
||||
debug: 4.4.3(supports-color@5.5.0)
|
||||
espree: 10.4.0
|
||||
globals: 14.0.0
|
||||
@ -7118,13 +7125,18 @@ snapshots:
|
||||
dependencies:
|
||||
'@hapi/hoek': 11.0.7
|
||||
|
||||
'@humanfs/core@0.19.1': {}
|
||||
|
||||
'@humanfs/node@0.16.7':
|
||||
'@humanfs/core@0.19.2':
|
||||
dependencies:
|
||||
'@humanfs/core': 0.19.1
|
||||
'@humanfs/types': 0.15.0
|
||||
|
||||
'@humanfs/node@0.16.8':
|
||||
dependencies:
|
||||
'@humanfs/core': 0.19.2
|
||||
'@humanfs/types': 0.15.0
|
||||
'@humanwhocodes/retry': 0.4.3
|
||||
|
||||
'@humanfs/types@0.15.0': {}
|
||||
|
||||
'@humanwhocodes/module-importer@1.0.1': {}
|
||||
|
||||
'@humanwhocodes/retry@0.4.3': {}
|
||||
@ -7891,6 +7903,8 @@ snapshots:
|
||||
|
||||
'@types/estree@1.0.8': {}
|
||||
|
||||
'@types/estree@1.0.9': {}
|
||||
|
||||
'@types/json-schema@7.0.15': {}
|
||||
|
||||
'@types/json5@0.0.29': {}
|
||||
@ -8187,7 +8201,7 @@ snapshots:
|
||||
optionalDependencies:
|
||||
ajv: 8.13.0
|
||||
|
||||
ajv@6.14.0:
|
||||
ajv@6.15.0:
|
||||
dependencies:
|
||||
fast-deep-equal: 3.1.3
|
||||
fast-json-stable-stringify: 2.1.0
|
||||
@ -9374,11 +9388,11 @@ snapshots:
|
||||
'@eslint/eslintrc': 3.3.5
|
||||
'@eslint/js': 9.39.2
|
||||
'@eslint/plugin-kit': 0.4.1
|
||||
'@humanfs/node': 0.16.7
|
||||
'@humanfs/node': 0.16.8
|
||||
'@humanwhocodes/module-importer': 1.0.1
|
||||
'@humanwhocodes/retry': 0.4.3
|
||||
'@types/estree': 1.0.8
|
||||
ajv: 6.14.0
|
||||
'@types/estree': 1.0.9
|
||||
ajv: 6.15.0
|
||||
chalk: 4.1.2
|
||||
cross-spawn: 7.0.6
|
||||
debug: 4.4.3(supports-color@5.5.0)
|
||||
@ -9433,7 +9447,7 @@ snapshots:
|
||||
|
||||
events@3.3.0: {}
|
||||
|
||||
eventsource-parser@3.0.6: {}
|
||||
eventsource-parser@3.0.8: {}
|
||||
|
||||
execa@8.0.1:
|
||||
dependencies:
|
||||
|
||||
@ -17,22 +17,30 @@
|
||||
|
||||
(defn read-stream
|
||||
[^js/ReadableStream stream decode-fn]
|
||||
(letfn [(read-items [^js reader]
|
||||
(->> (rx/from (.read reader))
|
||||
(rx/mapcat (fn [result]
|
||||
(if (.-done result)
|
||||
(rx/empty)
|
||||
(rx/concat
|
||||
(rx/of (.-value result))
|
||||
(read-items reader)))))))]
|
||||
(->> (read-items (.getReader stream))
|
||||
(rx/mapcat (fn [^js event]
|
||||
(let [type (.-event event)
|
||||
data (.-data event)
|
||||
data (decode-fn data)]
|
||||
(if (= "error" type)
|
||||
(rx/throw (ex-info "stream exception" data))
|
||||
(rx/of #js {:type type :data data}))))))))
|
||||
(->> (rx/create
|
||||
(fn [subs]
|
||||
(let [reader (.getReader stream)]
|
||||
(letfn [(pump []
|
||||
(-> (.read reader)
|
||||
(.then (fn [result]
|
||||
(if (.-done result)
|
||||
(rx/end! subs)
|
||||
(do
|
||||
(rx/push! subs (.-value result))
|
||||
(pump)))))
|
||||
(.catch (fn [cause]
|
||||
(rx/error! subs cause)))))]
|
||||
(pump)
|
||||
;; teardown: cancel the reader when unsubscribed
|
||||
(fn [] (.cancel reader))))))
|
||||
(rx/mapcat (fn [^js event]
|
||||
(let [type (.-event event)
|
||||
data (.-data event)
|
||||
data (decode-fn data)]
|
||||
(if (= "error" type)
|
||||
(rx/throw (ex-info "stream exception" data))
|
||||
(rx/of #js {:type type :data data})))))))
|
||||
|
||||
|
||||
(defn get-type
|
||||
[event]
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user