From ec0d692856ae8ae7695b2e12509397cdcb3cd197 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Mon, 11 May 2026 10:59:35 +0200 Subject: [PATCH] :bug: Fix maximum call stack size exceeded in SSE read-stream (#9484) The recursive `read-items` function in `app.util.sse/read-stream` caused a synchronous stack overflow when reading buffered stream data. Each `rx/mapcat` call chained another recursive invocation on the same call stack without yielding to the event loop. Replace the recursive pattern with an `rx/create`-based async pump that uses Promise `.then()` chaining, keeping the call stack depth constant regardless of stream size. Also add progress reporting with names and IDs during binfile export and import, and bump `eventsource-parser` dependency. Closes #9470 Signed-off-by: Andrey Antukh --- backend/src/app/binfile/v3.clj | 26 ++++++++++++---- frontend/package.json | 2 +- frontend/pnpm-lock.yaml | 54 +++++++++++++++++++++------------- frontend/src/app/util/sse.cljs | 40 +++++++++++++++---------- 4 files changed, 80 insertions(+), 42 deletions(-) diff --git a/backend/src/app/binfile/v3.clj b/backend/src/app/binfile/v3.clj index 0db826e407..b4ed065317 100644 --- a/backend/src/app/binfile/v3.clj +++ b/backend/src/app/binfile/v3.clj @@ -281,7 +281,7 @@ thumbnails (bfc/get-file-object-thumbnails cfg file-id)] - (events/tap :progress {:section :file :id file-id}) + (events/tap :progress {:section :file :id file-id :name (:name file)}) (vswap! bfc/*state* update :files assoc file-id {:id file-id @@ -301,6 +301,7 @@ (write-entry! output path file)) (doseq [[index page-id] (d/enumerate pages)] + (let [path (str "files/" file-id "/pages/" page-id ".json") page (get pages-index page-id) objects (:objects page) @@ -311,6 +312,8 @@ (write-entry! output path page) + (events/tap :progress {:section :page :id page-id :name (:name page) :file-id file-id}) + (doseq [[shape-id shape] objects] (let [path (str "files/" file-id "/pages/" page-id "/" shape-id ".json") shape (assoc shape :page-id page-id) @@ -323,6 +326,8 @@ (doseq [{:keys [id] :as media} media] (let [path (str "files/" file-id "/media/" id ".json") media (encode-media media)] + + (events/tap :progress {:section :media :id id :file-id file-id}) (write-entry! output path media))) (doseq [thumbnail thumbnails] @@ -332,11 +337,13 @@ data (-> data (assoc :media-id (:media-id thumbnail)) (encode-file-thumbnail))] + (events/tap :progress {:section :thumbnails :id (:object-id thumbnail) :file-id file-id}) (write-entry! output path data))) (doseq [[id component] components] (let [path (str "files/" file-id "/components/" id ".json") component (encode-component component)] + (events/tap :progress {:section :component :id id :file-id file-id}) (write-entry! output path component))) (doseq [[id color] colors] @@ -347,17 +354,20 @@ (and (contains? color :path) (str/empty? (:path color))) (dissoc :path))] + (events/tap :progress {:section :color :id id :file-id file-id}) (write-entry! output path color))) (doseq [[id object] typographies] (let [path (str "files/" file-id "/typographies/" id ".json") typography (encode-typography object)] + (events/tap :progress {:section :typography :id id :file-id file-id}) (write-entry! output path typography))) (when (and tokens-lib (not (ctob/empty-lib? tokens-lib))) (let [path (str "files/" file-id "/tokens.json") encoded-tokens (encode-tokens-lib tokens-lib)] + (events/tap :progress {:section :tokens-lib :file-id file-id}) (write-entry! output path encoded-tokens))))) (defn- export-files @@ -600,6 +610,7 @@ (let [object (->> (read-entry input entry) (decode-color) (validate-color))] + (events/tap :progress {:section :color :id id :file-id file-id}) (if (= id (:id object)) (assoc result id object) result))) @@ -631,6 +642,7 @@ (clean-component-pre-decode) (decode-component) (clean-component-post-decode))] + (events/tap :progress {:section :component :id id :file-id file-id}) (if (= id (:id object)) (assoc result id object) result))) @@ -644,6 +656,7 @@ (let [object (->> (read-entry input entry) (decode-typography) (validate-typography))] + (events/tap :progress {:section :typography :id id :file-id file-id}) (if (= id (:id object)) (assoc result id object) result))) @@ -653,6 +666,7 @@ (defn- read-file-tokens-lib [{:keys [::bfc/input ::entries]} file-id] (when-let [entry (d/seek (match-tokens-lib-entry-fn file-id) entries)] + (events/tap :progress {:section :tokens-lib :file-id file-id}) (->> (read-plain-entry input entry) (decode-tokens-lib) (validate-tokens-lib)))) @@ -678,6 +692,7 @@ (let [page (->> (read-entry input entry) (decode-page)) page (dissoc page :options)] + (events/tap :progress {:section :page :id id :file-id file-id}) (when (= id (:id page)) (let [objects (read-file-shapes cfg file-id id)] (assoc page :objects objects)))))) @@ -693,6 +708,7 @@ (let [object (->> (read-entry input entry) (decode-file-thumbnail) (validate-file-thumbnail))] + (if (and (= frame-id (:frame-id object)) (= page-id (:page-id object)) (= tag (:tag object))) @@ -733,8 +749,6 @@ (vswap! bfc/*state* update :index bfc/update-index media :id) - (events/tap :progress {:section :media :file-id file-id}) - (doseq [item media] (let [params (-> item (update :id bfc/lookup-index) @@ -742,6 +756,8 @@ (d/update-when :media-id bfc/lookup-index) (d/update-when :thumbnail-id bfc/lookup-index))] + (events/tap :progress {:section :media :id (:id params) :file-id file-id}) + (l/dbg :hint "inserting media object" :file-id (str file-id') :id (str (:id params)) @@ -753,8 +769,6 @@ (db/insert! conn :file-media-object params ::db/on-conflict-do-nothing? (::bfc/overwrite cfg)))) - (events/tap :progress {:section :thumbnails :file-id file-id}) - (doseq [item thumbnails] (let [media-id (bfc/lookup-index (:media-id item)) object-id (-> (assoc item :file-id file-id') @@ -769,6 +783,8 @@ :media-id (str media-id) ::l/sync? true) + (events/tap :progress {:section :thumbnail :file-id file-id :object-id object-id}) + (db/insert! conn :file-tagged-object-thumbnail params ::db/on-conflict-do-nothing? true))) diff --git a/frontend/package.json b/frontend/package.json index 82047594c1..bcba3291f9 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -72,7 +72,7 @@ "concurrently": "^9.2.1", "date-fns": "^4.1.0", "esbuild": "^0.28.0", - "eventsource-parser": "^3.0.6", + "eventsource-parser": "^3.0.8", "express": "^5.1.0", "fancy-log": "^2.0.0", "getopts": "^2.3.0", diff --git a/frontend/pnpm-lock.yaml b/frontend/pnpm-lock.yaml index 5fa2751411..577effcd4e 100644 --- a/frontend/pnpm-lock.yaml +++ b/frontend/pnpm-lock.yaml @@ -83,8 +83,8 @@ importers: specifier: ^0.28.0 version: 0.28.0 eventsource-parser: - specifier: ^3.0.6 - version: 3.0.6 + specifier: ^3.0.8 + version: 3.0.8 express: specifier: ^5.1.0 version: 5.2.1 @@ -1353,12 +1353,16 @@ packages: '@hapi/topo@6.0.2': resolution: {integrity: sha512-KR3rD5inZbGMrHmgPxsJ9dbi6zEK+C3ZwUwTa+eMwWLz7oijWUTWD2pMSNNYJAU6Qq+65NkxXjqHr/7LM2Xkqg==} - '@humanfs/core@0.19.1': - resolution: {integrity: sha512-5DyQ4+1JEUzejeK1JGICcideyfUbGixgS9jNgex5nqkW+cY7WZhxBigmieN5Qnw9ZosSNVC9KQKyb+GUaGyKUA==} + '@humanfs/core@0.19.2': + resolution: {integrity: sha512-UhXNm+CFMWcbChXywFwkmhqjs3PRCmcSa/hfBgLIb7oQ5HNb1wS0icWsGtSAUNgefHeI+eBrA8I1fxmbHsGdvA==} engines: {node: '>=18.18.0'} - '@humanfs/node@0.16.7': - resolution: {integrity: sha512-/zUx+yOsIrG4Y43Eh2peDeKCxlRt/gET6aHfaKpuq267qXdYDFViVHfMaLyygZOnl0kGWxFIgsBy8QFuTLUXEQ==} + '@humanfs/node@0.16.8': + resolution: {integrity: sha512-gE1eQNZ3R++kTzFUpdGlpmy8kDZD/MLyHqDwqjkVQI0JMdI1D51sy1H958PNXYkM2rAac7e5/CnIKZrHtPh3BQ==} + engines: {node: '>=18.18.0'} + + '@humanfs/types@0.15.0': + resolution: {integrity: sha512-ZZ1w0aoQkwuUuC7Yf+7sdeaNfqQiiLcSRbfI08oAxqLtpXQr9AIVX7Ay7HLDuiLYAaFPu8oBYNq/QIi9URHJ3Q==} engines: {node: '>=18.18.0'} '@humanwhocodes/module-importer@1.0.1': @@ -2232,6 +2236,9 @@ packages: '@types/estree@1.0.8': resolution: {integrity: sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==} + '@types/estree@1.0.9': + resolution: {integrity: sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==} + '@types/json-schema@7.0.15': resolution: {integrity: sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==} @@ -2468,8 +2475,8 @@ packages: ajv: optional: true - ajv@6.14.0: - resolution: {integrity: sha512-IWrosm/yrn43eiKqkfkHis7QioDleaXQHdDVPKg0FSwwd/DuvyX79TZnFOnYpB7dcsFAMmtFztZuXPDvSePkFw==} + ajv@6.15.0: + resolution: {integrity: sha512-fgFx7Hfoq60ytK2c7DhnF8jIvzYgOMxfugjLOSMHjLIPgenqa7S7oaagATUq99mV6IYvN2tRmC0wnTYX6iPbMw==} ajv@8.12.0: resolution: {integrity: sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA==} @@ -3451,8 +3458,8 @@ packages: resolution: {integrity: sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==} engines: {node: '>=0.8.x'} - eventsource-parser@3.0.6: - resolution: {integrity: sha512-Vo1ab+QXPzZ4tCa8SwIHJFaSzy4R6SHf7BY79rFBDf0idraZWAkYrDjDj8uWaSm3S2TK+hJ7/t1CEmZ7jXw+pg==} + eventsource-parser@3.0.8: + resolution: {integrity: sha512-70QWGkr4snxr0OXLRWsFLeRBIRPuQOvt4s8QYjmUlmlkyTZkRqS7EDVRZtzU3TiyDbXSzaOeF0XUKy8PchzukQ==} engines: {node: '>=18.0.0'} execa@8.0.1: @@ -7077,7 +7084,7 @@ snapshots: '@eslint/eslintrc@3.3.5': dependencies: - ajv: 6.14.0 + ajv: 6.15.0 debug: 4.4.3(supports-color@5.5.0) espree: 10.4.0 globals: 14.0.0 @@ -7118,13 +7125,18 @@ snapshots: dependencies: '@hapi/hoek': 11.0.7 - '@humanfs/core@0.19.1': {} - - '@humanfs/node@0.16.7': + '@humanfs/core@0.19.2': dependencies: - '@humanfs/core': 0.19.1 + '@humanfs/types': 0.15.0 + + '@humanfs/node@0.16.8': + dependencies: + '@humanfs/core': 0.19.2 + '@humanfs/types': 0.15.0 '@humanwhocodes/retry': 0.4.3 + '@humanfs/types@0.15.0': {} + '@humanwhocodes/module-importer@1.0.1': {} '@humanwhocodes/retry@0.4.3': {} @@ -7891,6 +7903,8 @@ snapshots: '@types/estree@1.0.8': {} + '@types/estree@1.0.9': {} + '@types/json-schema@7.0.15': {} '@types/json5@0.0.29': {} @@ -8187,7 +8201,7 @@ snapshots: optionalDependencies: ajv: 8.13.0 - ajv@6.14.0: + ajv@6.15.0: dependencies: fast-deep-equal: 3.1.3 fast-json-stable-stringify: 2.1.0 @@ -9374,11 +9388,11 @@ snapshots: '@eslint/eslintrc': 3.3.5 '@eslint/js': 9.39.2 '@eslint/plugin-kit': 0.4.1 - '@humanfs/node': 0.16.7 + '@humanfs/node': 0.16.8 '@humanwhocodes/module-importer': 1.0.1 '@humanwhocodes/retry': 0.4.3 - '@types/estree': 1.0.8 - ajv: 6.14.0 + '@types/estree': 1.0.9 + ajv: 6.15.0 chalk: 4.1.2 cross-spawn: 7.0.6 debug: 4.4.3(supports-color@5.5.0) @@ -9433,7 +9447,7 @@ snapshots: events@3.3.0: {} - eventsource-parser@3.0.6: {} + eventsource-parser@3.0.8: {} execa@8.0.1: dependencies: diff --git a/frontend/src/app/util/sse.cljs b/frontend/src/app/util/sse.cljs index 8e1044ec37..40a02f4535 100644 --- a/frontend/src/app/util/sse.cljs +++ b/frontend/src/app/util/sse.cljs @@ -17,22 +17,30 @@ (defn read-stream [^js/ReadableStream stream decode-fn] - (letfn [(read-items [^js reader] - (->> (rx/from (.read reader)) - (rx/mapcat (fn [result] - (if (.-done result) - (rx/empty) - (rx/concat - (rx/of (.-value result)) - (read-items reader)))))))] - (->> (read-items (.getReader stream)) - (rx/mapcat (fn [^js event] - (let [type (.-event event) - data (.-data event) - data (decode-fn data)] - (if (= "error" type) - (rx/throw (ex-info "stream exception" data)) - (rx/of #js {:type type :data data})))))))) + (->> (rx/create + (fn [subs] + (let [reader (.getReader stream)] + (letfn [(pump [] + (-> (.read reader) + (.then (fn [result] + (if (.-done result) + (rx/end! subs) + (do + (rx/push! subs (.-value result)) + (pump))))) + (.catch (fn [cause] + (rx/error! subs cause)))))] + (pump) + ;; teardown: cancel the reader when unsubscribed + (fn [] (.cancel reader)))))) + (rx/mapcat (fn [^js event] + (let [type (.-event event) + data (.-data event) + data (decode-fn data)] + (if (= "error" type) + (rx/throw (ex-info "stream exception" data)) + (rx/of #js {:type type :data data}))))))) + (defn get-type [event]