From d1aa07087dffdc2ee5b3ac6e5c40e23d88a182e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Barrag=C3=A1n=20Merino?= Date: Wed, 30 Sep 2026 13:47:24 +0200 Subject: [PATCH 1/8] :construction_worker: Pin GitHub Actions to commit SHAs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: David Barragán Merino --- .github/dependabot.yml | 16 ++++++++++ .github/workflows/auto-label.yml | 4 +-- .github/workflows/build-bundle.yml | 4 +-- .github/workflows/build-docker-devenv.yml | 10 +++---- .github/workflows/build-docker.yml | 26 ++++++++-------- .github/workflows/commit-checker.yml | 2 +- .github/workflows/plugins-deploy-api-doc.yml | 6 ++-- .github/workflows/plugins-deploy-package.yml | 2 +- .github/workflows/plugins-deploy-packages.yml | 4 +-- .../workflows/plugins-deploy-styles-doc.yml | 6 ++-- .github/workflows/release.yml | 21 ++++++++----- .github/workflows/tests-backend.yml | 2 +- .github/workflows/tests-common.yml | 2 +- .github/workflows/tests-e2e.yml | 30 +++++++++---------- .github/workflows/tests-exporter.yml | 2 +- .github/workflows/tests-frontend.yml | 2 +- .github/workflows/tests-library.yml | 2 +- .github/workflows/tests-mcp.yml | 2 +- .github/workflows/tests-plugins.yml | 2 +- .github/workflows/tests-wasm.yml | 2 +- .github/zizmor.yml | 26 ---------------- 21 files changed, 84 insertions(+), 89 deletions(-) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000000..f0c1a8ddff --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,16 @@ +# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file +version: 2 +updates: + # Actions are pinned to a commit SHA with a `# vX.Y.Z` comment; Dependabot + # bumps both. All updates are grouped into a single monthly PR. + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "monthly" + open-pull-requests-limit: 5 + commit-message: + prefix: ":arrow_up:" + groups: + github-actions: + patterns: + - "*" diff --git a/.github/workflows/auto-label.yml b/.github/workflows/auto-label.yml index 491f422d1f..6b32b61c52 100644 --- a/.github/workflows/auto-label.yml +++ b/.github/workflows/auto-label.yml @@ -12,14 +12,14 @@ jobs: steps: - name: Generate GitHub App token id: triage-app-token - uses: actions/create-github-app-token@v1 + uses: actions/create-github-app-token@d72941d797fd3113feb6b93fd0dec494b13a2547 # v1.12.0 with: app-id: ${{ secrets.TRIAGE_APP_ID }} private-key: ${{ secrets.TRIAGE_APP_PRIVATE_KEY }} owner: penpot - name: Process Issue or PR - uses: actions/github-script@v7 + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 with: github-token: ${{ steps.triage-app-token.outputs.token }} script: | diff --git a/.github/workflows/build-bundle.yml b/.github/workflows/build-bundle.yml index 43ce74c1b6..6295834e30 100644 --- a/.github/workflows/build-bundle.yml +++ b/.github/workflows/build-bundle.yml @@ -54,7 +54,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: fetch-depth: 0 ref: ${{ inputs.gh_ref }} @@ -121,7 +121,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: fetch-depth: 0 ref: ${{ inputs.gh_ref }} diff --git a/.github/workflows/build-docker-devenv.yml b/.github/workflows/build-docker-devenv.yml index 2eab6be51e..c7093e002f 100644 --- a/.github/workflows/build-docker-devenv.yml +++ b/.github/workflows/build-docker-devenv.yml @@ -15,26 +15,26 @@ jobs: echo "DOCKER_CONFIG=${{ runner.temp }}/.docker-${{ github.run_id }}-${{ github.job }}" >> $GITHUB_ENV - name: Checkout code - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v4 + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 - name: Login to Docker Registry (push destination) - uses: docker/login-action@v4 + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 with: username: ${{ secrets.PUB_DOCKER_USERNAME }} password: ${{ secrets.PUB_DOCKER_PASSWORD }} - name: Login to Docker Hardened Images registry (base image pull) - uses: docker/login-action@v4 + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 with: registry: dhi.io username: ${{ secrets.PUB_DOCKER_USERNAME }} password: ${{ secrets.PUB_DOCKER_PASSWORD }} - name: Build and push DevEnv Docker image - uses: docker/build-push-action@v7 + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 env: DOCKER_IMAGE: 'penpotapp/devenv' with: diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 271042700a..ee1ea61c07 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -65,7 +65,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: fetch-depth: 0 ref: ${{ inputs.sha != '' && inputs.sha || inputs.gh_ref }} @@ -159,12 +159,12 @@ jobs: echo "DOCKER_CONFIG=${{ runner.temp }}/.docker-${{ github.run_id }}-${{ github.job }}-${{ matrix.image }}" >> $GITHUB_ENV - name: Checkout code - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: ref: ${{ inputs.sha != '' && inputs.sha || inputs.gh_ref }} - name: Login to Docker Registry - uses: docker/login-action@v4 + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 with: registry: ${{ secrets.DOCKER_REGISTRY }} username: ${{ secrets.DOCKER_USERNAME }} @@ -174,7 +174,7 @@ jobs: # images from DockerHub for unregistered users. # https://docs.docker.com/docker-hub/usage/ - name: Login to DockerHub Registry - uses: docker/login-action@v4 + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 with: username: ${{ secrets.PUB_DOCKER_USERNAME }} password: ${{ secrets.PUB_DOCKER_PASSWORD }} @@ -186,7 +186,7 @@ jobs: # reuses the same PUB_DOCKER_* credentials as the DockerHub # login above. - name: Login to Docker Hardened Images registry (base image pull) - uses: docker/login-action@v4 + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 with: registry: dhi.io username: ${{ secrets.PUB_DOCKER_USERNAME }} @@ -218,23 +218,23 @@ jobs: popd - name: Set up QEMU (stable) - uses: docker/setup-qemu-action@v4 + uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4.4.0 with: platforms: linux/amd64,linux/arm64 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v4 + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 - name: Extract metadata (tags, labels) id: meta - uses: docker/metadata-action@v6 + uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0 with: images: ${{ matrix.image }} labels: | bundle_version=${{ needs.prepare.outputs.bundle_version }} - name: Build and push Docker image - uses: docker/build-push-action@v7 + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 with: context: ./docker/images/ file: ./docker/images/Dockerfile.${{ matrix.image }} @@ -262,10 +262,10 @@ jobs: echo "DOCKER_CONFIG=${{ runner.temp }}/.docker-${{ github.run_id }}-${{ github.job }}" >> $GITHUB_ENV - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v4 + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 - name: Login to Docker Registry - uses: docker/login-action@v4 + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 with: registry: ${{ secrets.DOCKER_REGISTRY }} username: ${{ secrets.DOCKER_USERNAME }} @@ -331,10 +331,10 @@ jobs: echo "DOCKER_CONFIG=${{ runner.temp }}/.docker-${{ github.run_id }}-${{ github.job }}" >> $GITHUB_ENV - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v4 + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 - name: Login to Docker Registry - uses: docker/login-action@v4 + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 with: registry: ${{ secrets.DOCKER_REGISTRY }} username: ${{ secrets.DOCKER_USERNAME }} diff --git a/.github/workflows/commit-checker.yml b/.github/workflows/commit-checker.yml index a80e6e4cc0..c876d10160 100644 --- a/.github/workflows/commit-checker.yml +++ b/.github/workflows/commit-checker.yml @@ -27,7 +27,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check Commit Type - uses: gsactions/commit-message-checker@v2 + uses: gsactions/commit-message-checker@16fa2d5de096ae0d35626443bcd24f1e756cafee # v2.0.0 with: pattern: '^(((:(lipstick|globe_with_meridians|wrench|books|arrow_up|arrow_down|zap|ambulance|construction|boom|fire|whale|bug|sparkles|paperclip|tada|recycle|rewind|construction_worker):)\s[A-Z].*[^.])|(Merge|Revert|Reapply).+[^.])$' flags: 'gm' diff --git a/.github/workflows/plugins-deploy-api-doc.yml b/.github/workflows/plugins-deploy-api-doc.yml index dc74f61fdd..b3efd118c5 100644 --- a/.github/workflows/plugins-deploy-api-doc.yml +++ b/.github/workflows/plugins-deploy-api-doc.yml @@ -39,14 +39,14 @@ jobs: echo "gh_ref=$GH_REF" >> $GITHUB_OUTPUT - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: fetch-depth: 0 ref: ${{ steps.vars.outputs.gh_ref }} # START: Setup Node and PNPM enabling cache - name: Setup pnpm + Node.js - uses: pnpm/setup@v2 + uses: pnpm/setup@84cb39b217b10273981911c288cd62326dc7c6d2 # v2.0.2 with: working-directory: plugins runtime: node@24.21.0 @@ -59,7 +59,7 @@ jobs: run: echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_OUTPUT - name: Cache pnpm store - uses: actions/cache@v5 + uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: ${{ steps.pnpm-store.outputs.STORE_PATH }} key: ${{ runner.os }}-pnpm-${{ hashFiles('plugins/pnpm-lock.yaml') }} diff --git a/.github/workflows/plugins-deploy-package.yml b/.github/workflows/plugins-deploy-package.yml index eec741a009..71700801c7 100644 --- a/.github/workflows/plugins-deploy-package.yml +++ b/.github/workflows/plugins-deploy-package.yml @@ -58,7 +58,7 @@ jobs: fi - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: fetch-depth: 0 ref: ${{ inputs.gh_ref }} diff --git a/.github/workflows/plugins-deploy-packages.yml b/.github/workflows/plugins-deploy-packages.yml index 943e4b790d..a05b7b5944 100644 --- a/.github/workflows/plugins-deploy-packages.yml +++ b/.github/workflows/plugins-deploy-packages.yml @@ -36,9 +36,9 @@ jobs: # [For new plugins] # Add more outputs here steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 - id: filter - uses: dorny/paths-filter@v4 + uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4.0.3 with: filters: | colors_to_tokens: diff --git a/.github/workflows/plugins-deploy-styles-doc.yml b/.github/workflows/plugins-deploy-styles-doc.yml index 2d356962bf..9610045f4e 100644 --- a/.github/workflows/plugins-deploy-styles-doc.yml +++ b/.github/workflows/plugins-deploy-styles-doc.yml @@ -37,14 +37,14 @@ jobs: echo "gh_ref=$GH_REF" >> $GITHUB_OUTPUT - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: fetch-depth: 0 ref: ${{ steps.vars.outputs.gh_ref }} # START: Setup Node and PNPM enabling cache - name: Setup pnpm + Node.js - uses: pnpm/setup@v2 + uses: pnpm/setup@84cb39b217b10273981911c288cd62326dc7c6d2 # v2.0.2 with: working-directory: plugins runtime: node@24.21.0 @@ -57,7 +57,7 @@ jobs: run: echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_OUTPUT - name: Cache pnpm store - uses: actions/cache@v5 + uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: ${{ steps.pnpm-store.outputs.STORE_PATH }} key: ${{ runner.os }}-pnpm-${{ hashFiles('plugins/pnpm-lock.yaml') }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7468f8d4d9..7b7e6a1a1f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -32,14 +32,14 @@ jobs: echo "gh_ref=$GH_REF" >> $GITHUB_OUTPUT - name: Checkout code - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: fetch-depth: 0 ref: ${{ steps.vars.outputs.gh_ref }} # --- Publicly release the docker images --- - name: Configure ECR credentials - uses: aws-actions/configure-aws-credentials@v4 + uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1 with: aws-access-key-id: ${{ secrets.DOCKER_USERNAME }} aws-secret-access-key: ${{ secrets.DOCKER_PASSWORD }} @@ -94,14 +94,19 @@ jobs: echo "EOF" >> $GITHUB_OUTPUT # --- Create GitHub release --- + # Updates the release if it already exists, so re-running is safe. - name: Create GitHub release - uses: softprops/action-gh-release@v2 env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - tag_name: ${{ steps.vars.outputs.gh_ref }} - name: ${{ steps.vars.outputs.gh_ref }} - body: ${{ steps.extract_release_notes.outputs.release_notes }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + TAG: ${{ steps.vars.outputs.gh_ref }} + NOTES: ${{ steps.extract_release_notes.outputs.release_notes }} + run: | + if gh release view "$TAG" > /dev/null 2>&1; then + gh release edit "$TAG" --title "$TAG" --notes "$NOTES" + else + gh release create "$TAG" --verify-tag --title "$TAG" --notes "$NOTES" + fi - name: Notify Mattermost if: failure() diff --git a/.github/workflows/tests-backend.yml b/.github/workflows/tests-backend.yml index 02f839c062..580f24002d 100644 --- a/.github/workflows/tests-backend.yml +++ b/.github/workflows/tests-backend.yml @@ -60,7 +60,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 - name: Fmt working-directory: ./backend diff --git a/.github/workflows/tests-common.yml b/.github/workflows/tests-common.yml index e513efc54f..560a5b6427 100644 --- a/.github/workflows/tests-common.yml +++ b/.github/workflows/tests-common.yml @@ -39,7 +39,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 - name: Lint working-directory: ./common diff --git a/.github/workflows/tests-e2e.yml b/.github/workflows/tests-e2e.yml index ccbde28970..b32b0e3868 100644 --- a/.github/workflows/tests-e2e.yml +++ b/.github/workflows/tests-e2e.yml @@ -107,7 +107,7 @@ jobs: # An empty `ref` makes checkout fall back to its default (the PR merge # ref on pull_request, the pushed ref on push). - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: ref: ${{ inputs.gh_ref }} @@ -120,7 +120,7 @@ jobs: - name: Restore Bundle Cache id: restore - uses: actions/cache/restore@v5 + uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: key: ${{ steps.vars.outputs.bundle_key }} path: frontend/resources/public @@ -133,7 +133,7 @@ jobs: - name: Store Bundle Cache if: steps.restore.outputs.cache-hit != 'true' - uses: actions/cache/save@v5 + uses: actions/cache/save@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: key: ${{ steps.vars.outputs.bundle_key }} path: frontend/resources/public @@ -163,12 +163,12 @@ jobs: steps: - name: Checkout Repository - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: ref: ${{ inputs.gh_ref }} - name: Restore Cache - uses: actions/cache/restore@v5 + uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: key: ${{ needs.build-bundle.outputs.bundle_key }} path: frontend/resources/public @@ -201,7 +201,7 @@ jobs: --reporter=blob - name: Upload blob report - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 if: always() with: name: integration-blob-report-${{ matrix.shard }} @@ -210,7 +210,7 @@ jobs: retention-days: 3 - name: Upload test result - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 if: always() with: name: integration-tests-result-${{ matrix.shard }} @@ -234,11 +234,11 @@ jobs: - /var/cache/github-runner/gitlib:/root/.gitlibs steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 # The driver serves the prebuilt bundle from frontend/resources/public. - name: Restore Cache - uses: actions/cache/restore@v5 + uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: key: ${{ needs.build-bundle.outputs.bundle_key }} path: frontend/resources/public @@ -272,11 +272,11 @@ jobs: - /var/cache/github-runner/gitlib:/root/.gitlibs steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 # Mocked mode serves the prebuilt bundle from frontend/resources/public. - name: Restore Cache - uses: actions/cache/restore@v5 + uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: key: ${{ needs.build-bundle.outputs.bundle_key }} path: frontend/resources/public @@ -318,7 +318,7 @@ jobs: steps: - name: Checkout Repository - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: ref: ${{ inputs.gh_ref }} @@ -328,7 +328,7 @@ jobs: pnpm install --frozen-lockfile; - name: Download blob reports - uses: actions/download-artifact@v7 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: path: frontend/all-blob-reports pattern: integration-blob-report-* @@ -357,7 +357,7 @@ jobs: >> "$GITHUB_STEP_SUMMARY" - name: Upload JSON report - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 if: always() with: name: integration-json-report @@ -367,7 +367,7 @@ jobs: retention-days: 7 - name: Upload HTML report - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 if: always() with: name: integration-html-report diff --git a/.github/workflows/tests-exporter.yml b/.github/workflows/tests-exporter.yml index aefbfc1dbc..7f74d97390 100644 --- a/.github/workflows/tests-exporter.yml +++ b/.github/workflows/tests-exporter.yml @@ -43,7 +43,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 - name: Lint working-directory: ./exporter diff --git a/.github/workflows/tests-frontend.yml b/.github/workflows/tests-frontend.yml index f08f34054d..a2f3453763 100644 --- a/.github/workflows/tests-frontend.yml +++ b/.github/workflows/tests-frontend.yml @@ -43,7 +43,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 - name: Lint working-directory: ./frontend diff --git a/.github/workflows/tests-library.yml b/.github/workflows/tests-library.yml index 84ab5c9dc9..a434a71e34 100644 --- a/.github/workflows/tests-library.yml +++ b/.github/workflows/tests-library.yml @@ -41,7 +41,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 - name: Lint working-directory: ./library diff --git a/.github/workflows/tests-mcp.yml b/.github/workflows/tests-mcp.yml index f51c56b8a3..1e7e46d57d 100644 --- a/.github/workflows/tests-mcp.yml +++ b/.github/workflows/tests-mcp.yml @@ -33,7 +33,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 - name: Setup working-directory: ./mcp diff --git a/.github/workflows/tests-plugins.yml b/.github/workflows/tests-plugins.yml index 9f6c461da2..c18fc4c55b 100644 --- a/.github/workflows/tests-plugins.yml +++ b/.github/workflows/tests-plugins.yml @@ -38,7 +38,7 @@ jobs: - /var/cache/github-runner/gitlib:/root/.gitlibs steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 - name: Install deps working-directory: ./plugins diff --git a/.github/workflows/tests-wasm.yml b/.github/workflows/tests-wasm.yml index 9f26175554..cc58fa0ccf 100644 --- a/.github/workflows/tests-wasm.yml +++ b/.github/workflows/tests-wasm.yml @@ -39,7 +39,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 - name: Format working-directory: ./render-wasm diff --git a/.github/zizmor.yml b/.github/zizmor.yml index bdaa8777e5..b6acd14809 100644 --- a/.github/zizmor.yml +++ b/.github/zizmor.yml @@ -82,10 +82,6 @@ rules: # Style nudge towards the `$/...` syntax; not worth enforcing. self-repository: disable: true - # Replace actions that duplicate built-in runner tools. - superfluous-actions: - ignore: - - release.yml # Pin container images to a digest. unpinned-images: ignore: @@ -99,25 +95,3 @@ rules: - tests-mcp.yml - tests-plugins.yml - tests-wasm.yml - # Pin actions to a full commit SHA. - unpinned-uses: - ignore: - - auto-label.yml - - build-bundle.yml - - build-docker-devenv.yml - - build-docker.yml - - commit-checker.yml - - plugins-deploy-api-doc.yml - - plugins-deploy-package.yml - - plugins-deploy-packages.yml - - plugins-deploy-styles-doc.yml - - release.yml - - tests-backend.yml - - tests-common.yml - - tests-e2e.yml - - tests-exporter.yml - - tests-frontend.yml - - tests-library.yml - - tests-mcp.yml - - tests-plugins.yml - - tests-wasm.yml From fa3e01f7c719a1133e15a93b663e26e3028b1b6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Barrag=C3=A1n=20Merino?= Date: Wed, 30 Sep 2026 13:18:20 +0000 Subject: [PATCH 2/8] :construction_worker: Disable checkout credential persistence MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Set persist-credentials: false on every actions/checkout step, so the job token is not left in .git/config for the rest of the job. No step after checkout pushes or fetches with it. The only authenticated operation, gh release in release.yml, uses GH_TOKEN. Signed-off-by: David Barragán Merino --- .github/workflows/build-bundle.yml | 2 ++ .github/workflows/build-docker-devenv.yml | 2 ++ .github/workflows/build-docker.yml | 2 ++ .github/workflows/plugins-deploy-api-doc.yml | 1 + .github/workflows/plugins-deploy-package.yml | 1 + .github/workflows/plugins-deploy-packages.yml | 2 ++ .../workflows/plugins-deploy-styles-doc.yml | 1 + .github/workflows/release.yml | 1 + .github/workflows/tests-backend.yml | 2 ++ .github/workflows/tests-common.yml | 2 ++ .github/workflows/tests-e2e.yml | 7 +++++++ .github/workflows/tests-exporter.yml | 2 ++ .github/workflows/tests-frontend.yml | 2 ++ .github/workflows/tests-library.yml | 2 ++ .github/workflows/tests-mcp.yml | 2 ++ .github/workflows/tests-plugins.yml | 2 ++ .github/workflows/tests-wasm.yml | 2 ++ .github/zizmor.yml | 20 ------------------- 18 files changed, 35 insertions(+), 20 deletions(-) diff --git a/.github/workflows/build-bundle.yml b/.github/workflows/build-bundle.yml index 6295834e30..5cecd3e233 100644 --- a/.github/workflows/build-bundle.yml +++ b/.github/workflows/build-bundle.yml @@ -56,6 +56,7 @@ jobs: - name: Checkout repository uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: + persist-credentials: false fetch-depth: 0 ref: ${{ inputs.gh_ref }} @@ -123,6 +124,7 @@ jobs: - name: Checkout repository uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: + persist-credentials: false fetch-depth: 0 ref: ${{ inputs.gh_ref }} diff --git a/.github/workflows/build-docker-devenv.yml b/.github/workflows/build-docker-devenv.yml index c7093e002f..39d3c2594f 100644 --- a/.github/workflows/build-docker-devenv.yml +++ b/.github/workflows/build-docker-devenv.yml @@ -16,6 +16,8 @@ jobs: - name: Checkout code uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: + persist-credentials: false - name: Set up Docker Buildx uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index ee1ea61c07..6970b9b026 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -67,6 +67,7 @@ jobs: - name: Checkout code uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: + persist-credentials: false fetch-depth: 0 ref: ${{ inputs.sha != '' && inputs.sha || inputs.gh_ref }} @@ -161,6 +162,7 @@ jobs: - name: Checkout code uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: + persist-credentials: false ref: ${{ inputs.sha != '' && inputs.sha || inputs.gh_ref }} - name: Login to Docker Registry diff --git a/.github/workflows/plugins-deploy-api-doc.yml b/.github/workflows/plugins-deploy-api-doc.yml index b3efd118c5..a3598af4bd 100644 --- a/.github/workflows/plugins-deploy-api-doc.yml +++ b/.github/workflows/plugins-deploy-api-doc.yml @@ -41,6 +41,7 @@ jobs: - name: Checkout uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: + persist-credentials: false fetch-depth: 0 ref: ${{ steps.vars.outputs.gh_ref }} diff --git a/.github/workflows/plugins-deploy-package.yml b/.github/workflows/plugins-deploy-package.yml index 71700801c7..83fa91df3c 100644 --- a/.github/workflows/plugins-deploy-package.yml +++ b/.github/workflows/plugins-deploy-package.yml @@ -60,6 +60,7 @@ jobs: - name: Checkout uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: + persist-credentials: false fetch-depth: 0 ref: ${{ inputs.gh_ref }} diff --git a/.github/workflows/plugins-deploy-packages.yml b/.github/workflows/plugins-deploy-packages.yml index a05b7b5944..ee269ba616 100644 --- a/.github/workflows/plugins-deploy-packages.yml +++ b/.github/workflows/plugins-deploy-packages.yml @@ -37,6 +37,8 @@ jobs: # Add more outputs here steps: - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: + persist-credentials: false - id: filter uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4.0.3 with: diff --git a/.github/workflows/plugins-deploy-styles-doc.yml b/.github/workflows/plugins-deploy-styles-doc.yml index 9610045f4e..3c04eb6b58 100644 --- a/.github/workflows/plugins-deploy-styles-doc.yml +++ b/.github/workflows/plugins-deploy-styles-doc.yml @@ -39,6 +39,7 @@ jobs: - name: Checkout uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: + persist-credentials: false fetch-depth: 0 ref: ${{ steps.vars.outputs.gh_ref }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7b7e6a1a1f..8aa43c26c3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -34,6 +34,7 @@ jobs: - name: Checkout code uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: + persist-credentials: false fetch-depth: 0 ref: ${{ steps.vars.outputs.gh_ref }} diff --git a/.github/workflows/tests-backend.yml b/.github/workflows/tests-backend.yml index 580f24002d..1557a8685a 100644 --- a/.github/workflows/tests-backend.yml +++ b/.github/workflows/tests-backend.yml @@ -61,6 +61,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: + persist-credentials: false - name: Fmt working-directory: ./backend diff --git a/.github/workflows/tests-common.yml b/.github/workflows/tests-common.yml index 560a5b6427..b622958277 100644 --- a/.github/workflows/tests-common.yml +++ b/.github/workflows/tests-common.yml @@ -40,6 +40,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: + persist-credentials: false - name: Lint working-directory: ./common diff --git a/.github/workflows/tests-e2e.yml b/.github/workflows/tests-e2e.yml index b32b0e3868..07e1cc47a9 100644 --- a/.github/workflows/tests-e2e.yml +++ b/.github/workflows/tests-e2e.yml @@ -109,6 +109,7 @@ jobs: - name: Checkout repository uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: + persist-credentials: false ref: ${{ inputs.gh_ref }} # The cache key must come from the SHA actually checked out: on a manual @@ -165,6 +166,7 @@ jobs: - name: Checkout Repository uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: + persist-credentials: false ref: ${{ inputs.gh_ref }} - name: Restore Cache @@ -235,6 +237,8 @@ jobs: steps: - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: + persist-credentials: false # The driver serves the prebuilt bundle from frontend/resources/public. - name: Restore Cache @@ -273,6 +277,8 @@ jobs: steps: - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: + persist-credentials: false # Mocked mode serves the prebuilt bundle from frontend/resources/public. - name: Restore Cache @@ -320,6 +326,7 @@ jobs: - name: Checkout Repository uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: + persist-credentials: false ref: ${{ inputs.gh_ref }} - name: Install deps diff --git a/.github/workflows/tests-exporter.yml b/.github/workflows/tests-exporter.yml index 7f74d97390..8441af3a75 100644 --- a/.github/workflows/tests-exporter.yml +++ b/.github/workflows/tests-exporter.yml @@ -44,6 +44,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: + persist-credentials: false - name: Lint working-directory: ./exporter diff --git a/.github/workflows/tests-frontend.yml b/.github/workflows/tests-frontend.yml index a2f3453763..8fcfdc8066 100644 --- a/.github/workflows/tests-frontend.yml +++ b/.github/workflows/tests-frontend.yml @@ -44,6 +44,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: + persist-credentials: false - name: Lint working-directory: ./frontend diff --git a/.github/workflows/tests-library.yml b/.github/workflows/tests-library.yml index a434a71e34..df2c478ca1 100644 --- a/.github/workflows/tests-library.yml +++ b/.github/workflows/tests-library.yml @@ -42,6 +42,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: + persist-credentials: false - name: Lint working-directory: ./library diff --git a/.github/workflows/tests-mcp.yml b/.github/workflows/tests-mcp.yml index 1e7e46d57d..68ff0584bd 100644 --- a/.github/workflows/tests-mcp.yml +++ b/.github/workflows/tests-mcp.yml @@ -34,6 +34,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: + persist-credentials: false - name: Setup working-directory: ./mcp diff --git a/.github/workflows/tests-plugins.yml b/.github/workflows/tests-plugins.yml index c18fc4c55b..180f125489 100644 --- a/.github/workflows/tests-plugins.yml +++ b/.github/workflows/tests-plugins.yml @@ -39,6 +39,8 @@ jobs: steps: - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: + persist-credentials: false - name: Install deps working-directory: ./plugins diff --git a/.github/workflows/tests-wasm.yml b/.github/workflows/tests-wasm.yml index cc58fa0ccf..fdbc8820ff 100644 --- a/.github/workflows/tests-wasm.yml +++ b/.github/workflows/tests-wasm.yml @@ -40,6 +40,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: + persist-credentials: false - name: Format working-directory: ./render-wasm diff --git a/.github/zizmor.yml b/.github/zizmor.yml index b6acd14809..03445803e4 100644 --- a/.github/zizmor.yml +++ b/.github/zizmor.yml @@ -12,26 +12,6 @@ rules: - plugins-deploy-api-doc.yml - plugins-deploy-package.yml - plugins-deploy-styles-doc.yml - # Use `persist-credentials: false` in actions/checkout. - artipacked: - ignore: - - build-bundle.yml - - build-docker-devenv.yml - - build-docker.yml - - plugins-deploy-api-doc.yml - - plugins-deploy-package.yml - - plugins-deploy-packages.yml - - plugins-deploy-styles-doc.yml - - release.yml - - tests-backend.yml - - tests-common.yml - - tests-e2e.yml - - tests-exporter.yml - - tests-frontend.yml - - tests-library.yml - - tests-mcp.yml - - tests-plugins.yml - - tests-wasm.yml # Avoid restoring caches in release/publish workflows. cache-poisoning: ignore: From 86fc3dd765665350e9c22007ddeeed7cb95f2335 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Barrag=C3=A1n=20Merino?= Date: Wed, 30 Sep 2026 13:18:21 +0000 Subject: [PATCH 3/8] :construction_worker: Pass explicit secrets to reusable workflows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace secrets: inherit with the secrets each reusable workflow actually uses, and declare them under on.workflow_call.secrets in the called workflow. Declared as required: false so behaviour is unchanged if a secret is missing. Signed-off-by: David Barragán Merino --- .github/workflows/build-adhoc.yml | 22 ++++++++++-- .github/workflows/build-bundle.yml | 11 ++++++ .github/workflows/build-develop.yml | 22 ++++++++++-- .github/workflows/build-docker.yml | 21 +++++++++++ .github/workflows/build-staging.yml | 22 ++++++++++-- .github/workflows/build-tag.yml | 31 +++++++++++++--- .github/workflows/build-tmp-tokens.yml | 19 ++++++++-- .github/workflows/plugins-deploy-package.yml | 7 ++++ .github/workflows/plugins-deploy-packages.yml | 35 +++++++++++++++---- .github/workflows/release.yml | 15 ++++++++ .github/zizmor.yml | 9 ----- 11 files changed, 183 insertions(+), 31 deletions(-) diff --git a/.github/workflows/build-adhoc.yml b/.github/workflows/build-adhoc.yml index abfe01d5a1..f5f9b88854 100644 --- a/.github/workflows/build-adhoc.yml +++ b/.github/workflows/build-adhoc.yml @@ -23,7 +23,12 @@ on: jobs: build-bundle: uses: ./.github/workflows/build-bundle.yml - secrets: inherit + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_REGION: ${{ secrets.AWS_REGION }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }} + S3_BUCKET: ${{ secrets.S3_BUCKET }} with: gh_ref: ${{ inputs.gh_ref }} force: ${{ inputs.force }} @@ -31,7 +36,17 @@ jobs: build-docker: needs: build-bundle uses: ./.github/workflows/build-docker.yml - secrets: inherit + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_REGION: ${{ secrets.AWS_REGION }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + DOCKER_REGISTRY: ${{ secrets.DOCKER_REGISTRY }} + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }} + PUB_DOCKER_PASSWORD: ${{ secrets.PUB_DOCKER_PASSWORD }} + PUB_DOCKER_USERNAME: ${{ secrets.PUB_DOCKER_USERNAME }} + S3_BUCKET: ${{ secrets.S3_BUCKET }} with: gh_ref: ${{ inputs.gh_ref }} # Pin build-docker to the commit build-bundle actually bundled. @@ -40,7 +55,8 @@ jobs: build-docker-admin-console: uses: ./.github/workflows/build-docker-admin-console.yml - secrets: inherit + secrets: + ORG_WORKFLOW_TOKEN: ${{ secrets.ORG_WORKFLOW_TOKEN }} with: gh_ref: ${{ inputs.nitrate_ref || inputs.gh_ref }} force: ${{ inputs.force }} diff --git a/.github/workflows/build-bundle.yml b/.github/workflows/build-bundle.yml index 5cecd3e233..752ff096f6 100644 --- a/.github/workflows/build-bundle.yml +++ b/.github/workflows/build-bundle.yml @@ -29,6 +29,17 @@ on: sha: description: 'Bundled commit, full 40-char SHA.' value: ${{ jobs.check.outputs.sha }} + secrets: + AWS_ACCESS_KEY_ID: + required: false + AWS_REGION: + required: false + AWS_SECRET_ACCESS_KEY: + required: false + MATTERMOST_WEBHOOK: + required: false + S3_BUCKET: + required: false # Literal group name: under `workflow_call`, `github.workflow` resolves to the # caller's workflow, which put this workflow and the other reusable one called diff --git a/.github/workflows/build-develop.yml b/.github/workflows/build-develop.yml index 8f12163965..942f840576 100644 --- a/.github/workflows/build-develop.yml +++ b/.github/workflows/build-develop.yml @@ -21,7 +21,12 @@ concurrency: jobs: build-bundle: uses: ./.github/workflows/build-bundle.yml - secrets: inherit + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_REGION: ${{ secrets.AWS_REGION }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }} + S3_BUCKET: ${{ secrets.S3_BUCKET }} with: gh_ref: "develop" force: ${{ inputs.force || false }} @@ -29,7 +34,17 @@ jobs: build-docker: needs: build-bundle uses: ./.github/workflows/build-docker.yml - secrets: inherit + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_REGION: ${{ secrets.AWS_REGION }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + DOCKER_REGISTRY: ${{ secrets.DOCKER_REGISTRY }} + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }} + PUB_DOCKER_PASSWORD: ${{ secrets.PUB_DOCKER_PASSWORD }} + PUB_DOCKER_USERNAME: ${{ secrets.PUB_DOCKER_USERNAME }} + S3_BUCKET: ${{ secrets.S3_BUCKET }} with: gh_ref: "develop" # Pin build-docker to the commit build-bundle actually bundled. @@ -38,7 +53,8 @@ jobs: build-docker-admin-console: uses: ./.github/workflows/build-docker-admin-console.yml - secrets: inherit + secrets: + ORG_WORKFLOW_TOKEN: ${{ secrets.ORG_WORKFLOW_TOKEN }} with: gh_ref: "develop" force: ${{ inputs.force || false }} diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 6970b9b026..f7348a0998 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -32,6 +32,27 @@ on: type: boolean required: false default: false + secrets: + AWS_ACCESS_KEY_ID: + required: false + AWS_REGION: + required: false + AWS_SECRET_ACCESS_KEY: + required: false + DOCKER_PASSWORD: + required: false + DOCKER_REGISTRY: + required: false + DOCKER_USERNAME: + required: false + MATTERMOST_WEBHOOK: + required: false + PUB_DOCKER_PASSWORD: + required: false + PUB_DOCKER_USERNAME: + required: false + S3_BUCKET: + required: false # Literal group name: under `workflow_call`, `github.workflow` resolves to the # caller's workflow, which put this workflow and the other reusable one called diff --git a/.github/workflows/build-staging.yml b/.github/workflows/build-staging.yml index 249ba39e40..3fb417c801 100644 --- a/.github/workflows/build-staging.yml +++ b/.github/workflows/build-staging.yml @@ -21,7 +21,12 @@ concurrency: jobs: build-bundle: uses: ./.github/workflows/build-bundle.yml - secrets: inherit + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_REGION: ${{ secrets.AWS_REGION }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }} + S3_BUCKET: ${{ secrets.S3_BUCKET }} with: gh_ref: "staging" force: ${{ inputs.force || false }} @@ -29,7 +34,17 @@ jobs: build-docker: needs: build-bundle uses: ./.github/workflows/build-docker.yml - secrets: inherit + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_REGION: ${{ secrets.AWS_REGION }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + DOCKER_REGISTRY: ${{ secrets.DOCKER_REGISTRY }} + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }} + PUB_DOCKER_PASSWORD: ${{ secrets.PUB_DOCKER_PASSWORD }} + PUB_DOCKER_USERNAME: ${{ secrets.PUB_DOCKER_USERNAME }} + S3_BUCKET: ${{ secrets.S3_BUCKET }} with: gh_ref: "staging" # Pin build-docker to the commit build-bundle actually bundled. @@ -38,7 +53,8 @@ jobs: build-docker-admin-console: uses: ./.github/workflows/build-docker-admin-console.yml - secrets: inherit + secrets: + ORG_WORKFLOW_TOKEN: ${{ secrets.ORG_WORKFLOW_TOKEN }} with: gh_ref: "staging" force: ${{ inputs.force || false }} diff --git a/.github/workflows/build-tag.yml b/.github/workflows/build-tag.yml index ed79d398f9..9b0b0fcbb3 100644 --- a/.github/workflows/build-tag.yml +++ b/.github/workflows/build-tag.yml @@ -24,7 +24,12 @@ concurrency: jobs: build-bundle: uses: ./.github/workflows/build-bundle.yml - secrets: inherit + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_REGION: ${{ secrets.AWS_REGION }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }} + S3_BUCKET: ${{ secrets.S3_BUCKET }} with: gh_ref: ${{ github.ref_name }} force: ${{ inputs.force || false }} @@ -32,14 +37,25 @@ jobs: build-docker: needs: build-bundle uses: ./.github/workflows/build-docker.yml - secrets: inherit + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_REGION: ${{ secrets.AWS_REGION }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + DOCKER_REGISTRY: ${{ secrets.DOCKER_REGISTRY }} + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }} + PUB_DOCKER_PASSWORD: ${{ secrets.PUB_DOCKER_PASSWORD }} + PUB_DOCKER_USERNAME: ${{ secrets.PUB_DOCKER_USERNAME }} + S3_BUCKET: ${{ secrets.S3_BUCKET }} with: gh_ref: ${{ github.ref_name }} force: ${{ inputs.force || false }} build-docker-admin-console: uses: ./.github/workflows/build-docker-admin-console.yml - secrets: inherit + secrets: + ORG_WORKFLOW_TOKEN: ${{ secrets.ORG_WORKFLOW_TOKEN }} with: gh_ref: ${{ github.ref_name }} force: ${{ inputs.force || false }} @@ -67,6 +83,13 @@ jobs: - build-docker - build-docker-admin-console uses: ./.github/workflows/release.yml - secrets: inherit + secrets: + AWS_REGION: ${{ secrets.AWS_REGION }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + DOCKER_REGISTRY: ${{ secrets.DOCKER_REGISTRY }} + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }} + PUB_DOCKER_PASSWORD: ${{ secrets.PUB_DOCKER_PASSWORD }} + PUB_DOCKER_USERNAME: ${{ secrets.PUB_DOCKER_USERNAME }} with: gh_ref: ${{ github.ref_name }} diff --git a/.github/workflows/build-tmp-tokens.yml b/.github/workflows/build-tmp-tokens.yml index e9e4183bad..68eeae1706 100644 --- a/.github/workflows/build-tmp-tokens.yml +++ b/.github/workflows/build-tmp-tokens.yml @@ -12,13 +12,28 @@ concurrency: jobs: build-bundle: uses: ./.github/workflows/build-bundle.yml - secrets: inherit + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_REGION: ${{ secrets.AWS_REGION }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }} + S3_BUCKET: ${{ secrets.S3_BUCKET }} with: gh_ref: "hiru-tokens-in-libs" build-docker: needs: build-bundle uses: ./.github/workflows/build-docker.yml - secrets: inherit + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_REGION: ${{ secrets.AWS_REGION }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + DOCKER_REGISTRY: ${{ secrets.DOCKER_REGISTRY }} + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }} + PUB_DOCKER_PASSWORD: ${{ secrets.PUB_DOCKER_PASSWORD }} + PUB_DOCKER_USERNAME: ${{ secrets.PUB_DOCKER_USERNAME }} + S3_BUCKET: ${{ secrets.S3_BUCKET }} with: gh_ref: "hiru-tokens-in-libs" diff --git a/.github/workflows/plugins-deploy-package.yml b/.github/workflows/plugins-deploy-package.yml index 83fa91df3c..7c83df9f5d 100644 --- a/.github/workflows/plugins-deploy-package.yml +++ b/.github/workflows/plugins-deploy-package.yml @@ -27,6 +27,13 @@ on: description: 'Publig name (from plugins/apps/-plugin)' type: string required: true + secrets: + CLOUDFLARE_ACCOUNT_ID: + required: false + CLOUDFLARE_API_TOKEN: + required: false + MATTERMOST_WEBHOOK: + required: false permissions: contents: read diff --git a/.github/workflows/plugins-deploy-packages.yml b/.github/workflows/plugins-deploy-packages.yml index ee269ba616..218620983f 100644 --- a/.github/workflows/plugins-deploy-packages.yml +++ b/.github/workflows/plugins-deploy-packages.yml @@ -74,7 +74,10 @@ jobs: needs: detect-changes if: github.event_name == 'workflow_dispatch' || needs.detect-changes.outputs.colors_to_tokens == 'true' uses: ./.github/workflows/plugins-deploy-package.yml - secrets: inherit + secrets: + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }} with: gh_ref: "${{ inputs.gh_ref || github.ref_name }}" plugin_name: colors-to-tokens @@ -83,7 +86,10 @@ jobs: needs: detect-changes if: github.event_name == 'workflow_dispatch' || needs.detect-changes.outputs.contrast == 'true' uses: ./.github/workflows/plugins-deploy-package.yml - secrets: inherit + secrets: + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }} with: gh_ref: "${{ inputs.gh_ref || github.ref_name }}" plugin_name: contrast @@ -92,7 +98,10 @@ jobs: needs: detect-changes if: github.event_name == 'workflow_dispatch' || needs.detect-changes.outputs.create_palette == 'true' uses: ./.github/workflows/plugins-deploy-package.yml - secrets: inherit + secrets: + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }} with: gh_ref: "${{ inputs.gh_ref || github.ref_name }}" plugin_name: create-palette @@ -101,7 +110,10 @@ jobs: needs: detect-changes if: github.event_name == 'workflow_dispatch' || needs.detect-changes.outputs.icons == 'true' uses: ./.github/workflows/plugins-deploy-package.yml - secrets: inherit + secrets: + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }} with: gh_ref: "${{ inputs.gh_ref || github.ref_name }}" plugin_name: icons @@ -110,7 +122,10 @@ jobs: needs: detect-changes if: github.event_name == 'workflow_dispatch' || needs.detect-changes.outputs.lorem_ipsum == 'true' uses: ./.github/workflows/plugins-deploy-package.yml - secrets: inherit + secrets: + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }} with: gh_ref: "${{ inputs.gh_ref || github.ref_name }}" plugin_name: lorem-ipsum @@ -119,7 +134,10 @@ jobs: needs: detect-changes if: github.event_name == 'workflow_dispatch' || needs.detect-changes.outputs.rename_layers == 'true' uses: ./.github/workflows/plugins-deploy-package.yml - secrets: inherit + secrets: + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }} with: gh_ref: "${{ inputs.gh_ref || github.ref_name }}" plugin_name: rename-layers @@ -128,7 +146,10 @@ jobs: needs: detect-changes if: github.event_name == 'workflow_dispatch' || needs.detect-changes.outputs.table == 'true' uses: ./.github/workflows/plugins-deploy-package.yml - secrets: inherit + secrets: + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }} with: gh_ref: "${{ inputs.gh_ref || github.ref_name }}" plugin_name: table diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8aa43c26c3..e956f2310d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,6 +13,21 @@ on: description: 'Tag to release' type: string required: true + secrets: + AWS_REGION: + required: false + DOCKER_PASSWORD: + required: false + DOCKER_REGISTRY: + required: false + DOCKER_USERNAME: + required: false + MATTERMOST_WEBHOOK: + required: false + PUB_DOCKER_PASSWORD: + required: false + PUB_DOCKER_USERNAME: + required: false permissions: contents: write diff --git a/.github/zizmor.yml b/.github/zizmor.yml index 03445803e4..d4681e9f17 100644 --- a/.github/zizmor.yml +++ b/.github/zizmor.yml @@ -50,15 +50,6 @@ rules: github-app: ignore: - auto-label.yml - # Pass only the secrets each reusable workflow needs. - secrets-inherit: - ignore: - - build-adhoc.yml - - build-develop.yml - - build-staging.yml - - build-tag.yml - - build-tmp-tokens.yml - - plugins-deploy-packages.yml # Style nudge towards the `$/...` syntax; not worth enforcing. self-repository: disable: true From 0509e2b9d22109804686781f9cc2c1b6be60753c Mon Sep 17 00:00:00 2001 From: Alonso Torres Date: Wed, 30 Sep 2026 17:56:01 +0200 Subject: [PATCH 4/8] :bug: Fix problem with connect library (#12002) --- frontend/src/app/plugins/library.cljs | 27 +++-- .../frontend_tests/plugins/library_test.cljs | 109 +++++++++++++++++- 2 files changed, 127 insertions(+), 9 deletions(-) diff --git a/frontend/src/app/plugins/library.cljs b/frontend/src/app/plugins/library.cljs index 998f8b0b5c..56782eb1e2 100644 --- a/frontend/src/app/plugins/library.cljs +++ b/frontend/src/app/plugins/library.cljs @@ -1248,15 +1248,26 @@ (not (r/check-permission plugin-id "library:write")) (u/reject-not-valid reject :connectLibrary "Plugin doesn't have 'library:write' permission") - (not (string? library-id)) + (not (uuid/parse* library-id)) (u/reject-not-valid reject :connectLibrary library-id) :else - (let [file-id (:current-file-id @st/state) + (let [file-id (:current-file-id @st/state) + team-id (:current-team-id @st/state) library-id (uuid/parse library-id)] - (->> st/stream - (rx/filter (ptk/type? ::dwl/link-file-to-library-finished)) - (rx/take 1) - (rx/subs! #(resolve (library-proxy plugin-id library-id)) reject)) - (st/emit! (-> (dwl/link-file-to-library file-id library-id) - (se/add-event plugin-id)))))))))) + (->> (rp/cmd! :get-team-shared-files {:team-id team-id}) + (rx/subs! + (fn [shared-files] + ;; Only published libraries of the team can be linked + (if (or (= library-id file-id) + (not (some #(= library-id (:id %)) shared-files))) + (u/reject-not-valid reject :connectLibrary + (dm/str library-id " is not a published library")) + (do + (->> st/stream + (rx/filter (ptk/type? ::dwl/link-file-to-library-finished)) + (rx/take 1) + (rx/subs! #(resolve (library-proxy plugin-id library-id)) reject)) + (st/emit! (-> (dwl/link-file-to-library file-id library-id) + (se/add-event plugin-id)))))) + reject))))))))) diff --git a/frontend/test/frontend_tests/plugins/library_test.cljs b/frontend/test/frontend_tests/plugins/library_test.cljs index d951cddb05..70d3e3fb7d 100644 --- a/frontend/test/frontend_tests/plugins/library_test.cljs +++ b/frontend/test/frontend_tests/plugins/library_test.cljs @@ -8,16 +8,23 @@ (:require [app.common.types.component :as ctk] [app.common.uuid :as uuid] + [app.main.data.event :as ev] [app.main.data.workspace.libraries :as dwl] [app.main.data.workspace.texts :as dwt] [app.main.data.workspace.variants :as dwv] + [app.main.repo :as rp] [app.main.store :as st] [app.plugins.library :as library] [app.plugins.register :as r] [app.plugins.text :as text] [app.plugins.utils :as u] + [app.util.object :as obj] + [beicon.v2.core :as rx] [cljs.test :as t :include-macros true] - [frontend-tests.helpers.mock :as mock])) + [cuerdas.core :as str] + [frontend-tests.helpers.async :as async] + [frontend-tests.helpers.mock :as mock] + [potok.v2.core :as ptk])) (def ^:private plugin-id "00000000-0000-0000-0000-000000000000") @@ -205,3 +212,103 @@ (t/is (= 1 (count @errors))) (t/is (= [plugin-id :setVariantProperty "Plugin doesn't have 'library:write' permission"] (first @errors))))))) + +;; --------------------------------------------------------------------------- +;; connectLibrary +;; --------------------------------------------------------------------------- + +(defn- shared-files-cmd + "Answers every RPC command asynchronously with `shared-files`, recording + the calls in `mock/rpc-calls`." + [shared-files] + (mock/stub + (fn [cmd params] + (swap! mock/rpc-calls conj {:cmd cmd :params params}) + (rx/observe-on :async (rx/of shared-files))))) + +(defn- settled + "Resolves with `{:resolved value}` or `{:rejected cause}` once `p` settles, + or with `{:timeout true}` when it stays pending for 2 seconds." + [p] + (js/Promise.race + #js [(.then p (fn [v] {:resolved v}) (fn [e] {:rejected e})) + (js/Promise. (fn [resolve] (js/setTimeout #(resolve {:timeout true}) 2000)))])) + +(t/deftest ^:async connect-library-links-a-published-library + (let [file-id (uuid/next) + team-id (uuid/next) + library-id (uuid/next) + stream (rx/subject) + events (atom [])] + (await + (mock/with-mocks* + {st/state (atom {:current-file-id file-id :current-team-id team-id}) + st/stream stream + st/emit! (mock/stub (fn [& emitted] (swap! events into emitted))) + rp/cmd! (shared-files-cmd [{:id library-id :name "Published"}]) + r/check-permission (mock/stub (constantly true))} + (let [context (library/library-subcontext plugin-id) + result (settled (.connectLibrary context (str library-id))) + link? #(= ::dwl/link-file-to-library (ptk/type %))] + (await (async/wait-for #(some link? @events) "link event emitted")) + (rx/push! stream (ptk/data-event ::dwl/link-file-to-library-finished)) + (let [{:keys [resolved]} (await result) + link-data (ev/-data (first (filter link? @events)))] + (t/is (= library-id (obj/get resolved "$id"))) + (t/is (= file-id (:file-id link-data))) + (t/is (= library-id (:library-id link-data))) + (t/is (= [{:cmd :get-team-shared-files :params {:team-id team-id}}] + @mock/rpc-calls)))))))) + +(t/deftest ^:async connect-library-rejects-an-unpublished-file + (let [file-id (uuid/next) + library-id (uuid/next) + events (atom [])] + (await + (mock/with-mocks* + {st/state (atom {:current-file-id file-id :current-team-id (uuid/next)}) + st/emit! (mock/stub (fn [& emitted] (swap! events into emitted))) + rp/cmd! (shared-files-cmd [{:id (uuid/next) :name "Other published"}]) + r/check-permission (mock/stub (constantly true))} + (let [context (library/library-subcontext plugin-id) + {:keys [rejected]} (await (settled (.connectLibrary context (str library-id))))] + (t/is (str/includes? (str rejected) "is not a published library")) + (t/is (empty? @events))))))) + +(t/deftest ^:async connect-library-rejects-the-current-file + (let [file-id (uuid/next) + events (atom [])] + (await + (mock/with-mocks* + {st/state (atom {:current-file-id file-id :current-team-id (uuid/next)}) + st/emit! (mock/stub (fn [& emitted] (swap! events into emitted))) + rp/cmd! (shared-files-cmd [{:id file-id :name "Current file"}]) + r/check-permission (mock/stub (constantly true))} + (let [context (library/library-subcontext plugin-id) + {:keys [rejected]} (await (settled (.connectLibrary context (str file-id))))] + (t/is (str/includes? (str rejected) "is not a published library")) + (t/is (empty? @events))))))) + +(t/deftest ^:async connect-library-rejects-an-invalid-id + (await + (mock/with-mocks* + {st/state (atom {:current-file-id (uuid/next) :current-team-id (uuid/next)}) + st/emit! mock/noop + rp/cmd! (shared-files-cmd []) + r/check-permission (mock/stub (constantly true))} + (let [context (library/library-subcontext plugin-id) + {:keys [rejected]} (await (settled (.connectLibrary context "not-a-uuid")))] + (t/is (str/includes? (str rejected) "not-a-uuid")) + (t/is (empty? @mock/rpc-calls)))))) + +(t/deftest ^:async connect-library-checks-permission + (await + (mock/with-mocks* + {st/state (atom {:current-file-id (uuid/next) :current-team-id (uuid/next)}) + st/emit! mock/noop + rp/cmd! (shared-files-cmd []) + r/check-permission (mock/stub (constantly false))} + (let [context (library/library-subcontext plugin-id) + {:keys [rejected]} (await (settled (.connectLibrary context (str (uuid/next)))))] + (t/is (str/includes? (str rejected) "Plugin doesn't have 'library:write' permission")) + (t/is (empty? @mock/rpc-calls)))))) From d67a00c1d590d401450d7722e0f0988259f21c71 Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Thu, 1 Oct 2026 07:19:21 +0200 Subject: [PATCH 5/8] :sparkles: Normalize storage metadata with a closed schema (#11987) * :sparkles: Add Malli schema for storage metadata with dual decode Phase 1 of the storage_object.metadata migration: reads accept both Transit and plain JSON (sniffed by the marker) and always return the normalized shape; writes validate against a closed per-bucket Malli schema and still serialize as Transit unless the new :storage-metadata-as-json config flag is set. The 0155 migration normalizes existing rows inside Transit (reference to bucket, default bucket, drop of chunk leftovers) and is idempotent; large instances should fake it and run the batched script instead. AI-assisted-by: muse-spark-1.3-contributor * :recycle: Address review findings on storage metadata Phase 1 Collapse the dead :reference leg of the gc-touched bucket fallback (the decode always sets :bucket on non-nil metadata, so it is only reachable with a NULL column) and fix its comment. Pin the write flag off in the transit-assuming metadata tests so the suite proves the same with the flag set, and add coverage for the flag rollback contract, JSON hash survival, NULL metadata in gc-touched, and the 0155 normalization statements. AI-assisted-by: muse-spark-1.3-contributor * :recycle: Backfill NULLs, canonical buckets, comment fix Backfill NULL metadata columns in 0155 via coalesce (the key-missing rule already matches them), derive valid-buckets from the Malli schema dispatch entries so the list lives in one place, and correct the lookup-bucket fallback comment to NULL columns. AI-assisted-by: muse-spark-1.3-contributor * :recycle: Defer corrupt metadata rows in storage gc-touched Decode touched rows individually so one non-map metadata value no longer aborts the whole chunk: corrupt rows are logged and deferred exactly one day in the same transaction, keeping their metadata intact for a later repair, while healthy rows process normally. AI-assisted-by: muse-spark-1.3-contributor * :recycle: Address storage metadata phase 1 review findings Address the review findings on the storage metadata phase 1 branch: - Fix put-and-delete-object: it stored the object with ::sto/expired-at, so the row was already deleted and del-object! returned false. Add delete-expired-object-returns-false to keep the expired-delete case covered. - Cache the Malli decoder and encoder per process. Building them compiles the closed multi-dispatch schema, and decode-metadata runs on every read path (get-object, dedup probes, GC batches). - Catch Exception instead of Throwable in try-decode-row so JVM Errors are not deferred as corrupt metadata. - Add penpot_storage_gc_poison_total, emitted from storage-gc-touched; wire ::mtx/metrics into its handler. - Anchor the encoding sniff to the start of the document so a plain JSON value that begins with a Transit-looking prefix is not read as Transit. - Cover every bucket on both encodings, a JSON roundtrip through the jsonb column, nil metadata, the canonical bucket set and a poison-only GC chunk. - Rename private check-metadata! to check-metadata. AI-assisted-by: deepseek-v4.1-flash * :recycle: Simplify the storage metadata schema to a single map Replace the per-bucket :multi dispatch with a single closed map: the bucket is validated with ::sm/one-of over metadata-buckets (now a plain set) and the remaining keys are typed optional fields. Per-bucket enforcement shrinks to a one-line :fn guard requiring :file-id and :id for file-data, whose ids the GC reads to resolve references. - Drop the dead (sm/register! ::metadata ...): nothing references the schema by keyword. - Define tempfile-bucket and upload-session-bucket in the schema and alias them from app.storage, removing duplicated literals. - Keep content-type required and the map closed, so an unknown bucket or key still fails fast on write. AI-assisted-by: deepseek-v4.1-flash * :recycle: Drop input coercion from encode-metadata encode-metadata no longer runs the json-transformer decoder before validation. On the write path its only effect was coercing string UUIDs to UUID, and every producer already passes native UUIDs (the RPC profile-id, uuid/random, or binfile ids decoded as ::sm/uuid). Reads keep decoding, so stored Transit or JSON values still come back as native types. - Replace encode-accepts-string-uuids with encode-rejects-string-uuids, pinning the stricter contract. - Pass native UUIDs in encode-writes-plain-json-with-flag. AI-assisted-by: deepseek-v4.1-flash * :books: Document each statement in the storage metadata migration Move the per-statement rules out of the header and add a comment to each UPDATE explaining what it does: drop chunk leftovers, promote the legacy "~:reference" to "~:bucket", drop residual "~:reference", and backfill the default bucket. The header keeps the scope, the encoding note, the `->` vs `?` note and the large-instance warning. AI-assisted-by: deepseek-v4.1-flash * :books: Unwrap wrapped lines in the backend storage memory One line per bullet or paragraph, as mem:memory-maintenance requires. Only formatting; no content change. AI-assisted-by: deepseek-v4.1-flash * :recycle: Defer storage GC poison rows in their own transaction process-chunk! no longer takes poison-ids; it only processes the healthy chunk. The deferral moves to defer-poison! and process-touched! runs it in its own transaction, separate from the freeze/delete work. The loop still drains while there is chunk or poison, so a batch made only of poison rows does not leave healthy rows behind the LIMIT 10 waiting for the next run. Add a regression test: ten poison rows plus one healthy row with a later touched_at are all handled in the same run. AI-assisted-by: deepseek-v4.1-flash * :recycle: Declare per-bucket metadata requirements in one map Replace the file-data-specific predicate with bucket-requirements, a map from bucket to the extra keys it must carry. metadata-buckets is derived from its keys and a single generic :fn enforces presence, so a new bucket and its contract are one entry. organization now requires :organization-id; file-data keeps requiring :file-id and :id. Update the http-assets test helper to set organization-id for its organization objects. AI-assisted-by: deepseek-v4.1-flash * :recycle: Drop the ! suffix from storage GC helpers Rename the internal helpers in app.storage.gc-touched (process-chunk, defer-poison, mark-freeze-in-bulk, ...) to drop the trailing !. AI-assisted-by: deepseek-v4.1-flash * :recycle: Drop the ! suffix from storage GC deleted helpers Rename the internal helpers in app.storage.gc-deleted (clean-deleted, delete-sobjects, delete-give-up, ...) to drop the trailing !. AI-assisted-by: deepseek-v4.1-flash * :bug: Fix dedup lookup for JSON-encoded storage metadata get-database-object-by-hash only matched the Transit keys, so once the :storage-metadata-as-json flag wrote plain JSON rows the dedup stopped finding them and duplicated blobs. Match both encodings with a UNION ALL of two indexable branches. - Add migration 0156 with the plain-key dedup index; the legacy 0068 index stays until Transit support is removed. - Cover it with a JSON dedup test and a Transit -> JSON cross test. AI-assisted-by: deepseek-v4.1-flash --- .serena/memories/backend/storage.md | 10 +- backend/src/app/config.clj | 4 + backend/src/app/main.clj | 9 +- backend/src/app/migrations.clj | 8 +- ...0155-normalize-storage-object-metadata.sql | 52 ++++ ...56-add-storage-object-json-dedup-index.sql | 11 + backend/src/app/storage.clj | 62 ++-- backend/src/app/storage/gc_deleted.clj | 24 +- backend/src/app/storage/gc_touched.clj | 111 +++++-- backend/src/app/storage/impl.clj | 4 +- backend/src/app/storage/schema.clj | 149 ++++++++++ .../test/backend_tests/http_assets_test.clj | 12 +- .../backend_tests/rpc_management_test.clj | 12 +- .../backend_tests/storage_metadata_test.clj | 276 ++++++++++++++++++ backend/test/backend_tests/storage_test.clj | 234 ++++++++++++++- 15 files changed, 884 insertions(+), 94 deletions(-) create mode 100644 backend/src/app/migrations/sql/0155-normalize-storage-object-metadata.sql create mode 100644 backend/src/app/migrations/sql/0156-add-storage-object-json-dedup-index.sql create mode 100644 backend/src/app/storage/schema.clj create mode 100644 backend/test/backend_tests/storage_metadata_test.clj diff --git a/.serena/memories/backend/storage.md b/.serena/memories/backend/storage.md index 43c925eeec..4a48cd8c71 100644 --- a/.serena/memories/backend/storage.md +++ b/.serena/memories/backend/storage.md @@ -54,8 +54,7 @@ ### Key Warning (from function notes): -The improved note in `import-storage-objects` and `handle-persistence` warns: -**Do not reuse the main database connection for storage operations within a transaction.** The storage upload process can fail mid-operation, leaving orphaned objects on the backend. If the outer transaction aborts, pending storage objects become unreconciliable because the storage subsystem registers its pending state in separate transactions. +The improved note in `import-storage-objects` and `handle-persistence` warns: **Do not reuse the main database connection for storage operations within a transaction.** The storage upload process can fail mid-operation, leaving orphaned objects on the backend. If the outer transaction aborts, pending storage objects become unreconciliable because the storage subsystem registers its pending state in separate transactions. ### Rule of Thumb for `sto/put-object!`: @@ -66,10 +65,7 @@ Since `put-object!` uses backend-specific operations (`impl/resolve-backend` + ` - Deduplication requires `::sto/deduplicate?`, a content hash, and bucket metadata. - The lookup matches hash, bucket, backend, and `deleted_at IS NULL`. - The lookup only considers rows with `status='valid'`; pending rows are invisible. -- A hit whose blob is missing is repaired in place: the same row/id is kept, - and `put-object!` rewrites the blob under that id. This heals all existing - references to the object. If the rewrite fails, the row is left live and - valid for a later retry. +- A hit whose blob is missing is repaired in place: the same row/id is kept, and `put-object!` rewrites the blob under that id. This heals all existing references to the object. If the rewrite fails, the row is left live and valid for a later retry. - The lookup does not include file ID, profile ID, team ID, or organization ID. - Objects can therefore share content across users and files within one bucket. - Deleted objects are not reused. @@ -130,6 +126,8 @@ Since `put-object!` uses backend-specific operations (`impl/resolve-backend` + ` - Logical storage operations (`app.storage`, `::mtx/metrics` required by the schema): - `penpot_storage_operations_total{op,bucket,backend}` — `put`, `repair`, `get-data`, `get-bytes`, `del`, `touch`, `exists`. All ops are success-only: `put`/`repair` emit after the backend write, `get-*` after the backend fetch opens, `touch`/`del` only when a row actually changed. `touch-object!`/`del-object!` take the object id (UUID) only — no object overload. Labels come from the updated row itself via `UPDATE ... RETURNING id, backend, metadata` (no extra `SELECT`); with no row matched they emit nothing. `del-object!` only matches live rows (`deleted_at IS NULL`): a repeated del returns `false` and emits nothing. Post-open stream read errors stay counted as attempts. `del` only marks `deleted_at`; physical deletion is a GC concern. `exists` is emitted per deduplication-hit probe, always paired with a `hit`/`repair` outcome (never on probe failure), not per user-facing existence check. - `penpot_storage_dedup_total{result,bucket}` — `hit`, `miss`, `repair`, `skip`. +- Storage GC poison (`app.storage.gc-touched`, `::mtx/metrics` required in the handler cfg): + - `penpot_storage_gc_poison_total` — touched rows whose metadata fails to decode. Each one is logged at error level and deferred exactly one day instead of aborting the chunk. No labels; incremented once per row per scan. - Asset serving (`app.http.assets`, `::mtx/metrics` required in the handler cfg): - `penpot_storage_asset_requests_total{route,backend,bucket,result}` — `route` is `by-id`, `by-file-media-id`, or `thumbnail`; `result` is `served` (<400), `not-found` (404), `unauthorized` (401/403), or `error` (everything else, including a nil/non-number status: every serve path must set `::yres/status`). Serve-path exceptions are counted by `serve-object-measured` and then rethrown. Permission-denied file-media requests and tempfile ownership mismatches both answer HTTP 404 (to avoid leaking existence) but are counted as `unauthorized`. Malformed UUIDs raise before any emission point and are never counted. Counts backend requests that trigger a browser GET to the object store (one per cache miss), so it is a proxy for object GETs, not an exact count. - The physical and logical counters intentionally overlap in coverage but differ in meaning; do not sum them. diff --git a/backend/src/app/config.clj b/backend/src/app/config.clj index 48081d6430..08c48476b5 100644 --- a/backend/src/app/config.clj +++ b/backend/src/app/config.clj @@ -304,6 +304,10 @@ [:objects-storage-s3-region {:optional true} :keyword] [:objects-storage-s3-endpoint {:optional true} ::sm/uri] + ;; Write storage_object.metadata as plain JSON instead of + ;; Transit-JSON. Unset by default (Phase 1: keep writing Transit). + [:storage-metadata-as-json {:optional true} ::sm/boolean] + ;; SSRF protection [:ssrf-allowed-hosts {:optional true} [::sm/set :string]] [:ssrf-extra-blocked-cidrs {:optional true} [::sm/set :string]]])) diff --git a/backend/src/app/main.clj b/backend/src/app/main.clj index 4133cc3a8b..de72b9a0ef 100644 --- a/backend/src/app/main.clj +++ b/backend/src/app/main.clj @@ -180,6 +180,12 @@ ::mdef/labels ["route" "backend" "bucket" "result"] ::mdef/type :counter} + :storage-gc-poison + {::mdef/name "penpot_storage_gc_poison_total" + ::mdef/help "Storage objects deferred by storage-gc-touched because their metadata is corrupt." + ::mdef/labels [] + ::mdef/type :counter} + :http-server-dispatch-timing {::mdef/name "penpot_http_server_dispatch_timing" ::mdef/help "Histogram of dispatch handler" @@ -271,7 +277,8 @@ ::sto/storage (ig/ref ::sto/storage)} ::sto.gc-touched/handler - {::db/pool (ig/ref ::db/pool)} + {::db/pool (ig/ref ::db/pool) + ::mtx/metrics (ig/ref ::mtx/metrics)} ::sto.pending-gc/handler {::db/pool (ig/ref ::db/pool) diff --git a/backend/src/app/migrations.clj b/backend/src/app/migrations.clj index fad1f91168..c458f45150 100644 --- a/backend/src/app/migrations.clj +++ b/backend/src/app/migrations.clj @@ -505,7 +505,13 @@ :fn (mg/resource "app/migrations/sql/0153-add-storage-object-status-and-deletion-attempts.sql")} {:name "0154-add-upload-session-chunk-table" - :fn (mg/resource "app/migrations/sql/0154-add-upload-session-chunk-table.sql")}]) + :fn (mg/resource "app/migrations/sql/0154-add-upload-session-chunk-table.sql")} + + {:name "0155-normalize-storage-object-metadata" + :fn (mg/resource "app/migrations/sql/0155-normalize-storage-object-metadata.sql")} + + {:name "0156-add-storage-object-json-dedup-index" + :fn (mg/resource "app/migrations/sql/0156-add-storage-object-json-dedup-index.sql")}]) (defn apply-migrations! [pool name migrations] diff --git a/backend/src/app/migrations/sql/0155-normalize-storage-object-metadata.sql b/backend/src/app/migrations/sql/0155-normalize-storage-object-metadata.sql new file mode 100644 index 0000000000..36ee9b2c97 --- /dev/null +++ b/backend/src/app/migrations/sql/0155-normalize-storage-object-metadata.sql @@ -0,0 +1,52 @@ +--- Normalize storage_object.metadata inside Transit (Phase 1). +--- +--- Brings every row to a canonical shape WITHOUT changing the encoding +--- family (still Transit JSON, "~:" keys), so old and new code keep +--- reading while it runs. Each statement below documents its own rule. +--- +--- NOTE: key-existence is tested with `-> ... IS [NOT] NULL` instead of +--- the `?` operator because the migration runner prepares statements +--- and a bare `?` is parsed as a bind placeholder. +--- +--- Idempotent: re-running changes zero rows. +--- NOTE for large instances (e.g. our production, ~33M rows): DO NOT run +--- this inline. Mark the migration as executed (fake) and run the batched +--- equivalent by hand: .agents/plans/2026-09-11-storage-normalize-phase1.sql + +-- Drop leftovers from the old chunked-upload metadata: no code reads +-- "~:upload-id" / "~:chunk-index" anymore (#11651 moved chunks to +-- upload_session_chunk and objects-gc purges the orphan rows), and the +-- Phase 1 readers already ignore them on decode. +UPDATE storage_object + SET metadata = metadata - '~:upload-id' - '~:chunk-index' + WHERE (metadata -> '~:upload-id') IS NOT NULL + OR (metadata -> '~:chunk-index') IS NOT NULL; + +-- Rows that carry only the legacy "~:reference": promote it to "~:bucket" +-- with the same value. The value comes as a Transit keyword ("~:xxx"), so +-- the "~:" prefix is stripped; plain strings are kept as they are. +UPDATE storage_object + SET metadata = (metadata - '~:reference') + || jsonb_build_object('~:bucket', + CASE WHEN metadata->>'~:reference' LIKE '~:%' + THEN substr(metadata->>'~:reference', 3) + ELSE metadata->>'~:reference' + END) + WHERE (metadata -> '~:reference') IS NOT NULL + AND (metadata -> '~:bucket') IS NULL; + +-- Rows that carry both keys: "~:bucket" wins and the residual +-- "~:reference" is dropped. +UPDATE storage_object + SET metadata = metadata - '~:reference' + WHERE (metadata -> '~:reference') IS NOT NULL + AND (metadata -> '~:bucket') IS NOT NULL; + +-- Rows with neither key get the historical default bucket +-- "file-media-object" (the value `lookup-bucket` used to fall back to). +-- NULL columns match this rule too (`->` on NULL is NULL) and are +-- backfilled by coalesce. +UPDATE storage_object + SET metadata = coalesce(metadata, '{}') || '{"~:bucket": "file-media-object"}' + WHERE (metadata -> '~:bucket') IS NULL + AND (metadata -> '~:reference') IS NULL; diff --git a/backend/src/app/migrations/sql/0156-add-storage-object-json-dedup-index.sql b/backend/src/app/migrations/sql/0156-add-storage-object-json-dedup-index.sql new file mode 100644 index 0000000000..0576318f50 --- /dev/null +++ b/backend/src/app/migrations/sql/0156-add-storage-object-json-dedup-index.sql @@ -0,0 +1,11 @@ +-- Dedup lookup index for the plain-JSON encoding of +-- storage_object.metadata ("hash" / "bucket"). +-- +-- The dedup lookup matches both metadata encodings while Transit rows +-- still exist (a UNION ALL of two indexable branches). The legacy "~:" +-- index (storage_object__hash_backend_bucket__idx, migration 0068) keeps +-- covering the Transit branch until Transit support is removed. + +CREATE INDEX storage_object__hash_backend_bucket_json__idx + ON storage_object ((metadata->>'hash'), (metadata->>'bucket'), backend) + WHERE deleted_at IS NULL AND status = 'valid'; diff --git a/backend/src/app/storage.clj b/backend/src/app/storage.clj index 52163bfc95..25c2d3482a 100644 --- a/backend/src/app/storage.clj +++ b/backend/src/app/storage.clj @@ -20,6 +20,7 @@ [app.storage.fs :as sfs] [app.storage.impl :as impl] [app.storage.s3 :as ss3] + [app.storage.schema :as stsch] [cuerdas.core :as str] [datoteka.fs :as fs] [integrant.core :as ig]) @@ -37,28 +38,18 @@ nil))) (def default-bucket - "file-media-object") + stsch/default-bucket) (def tempfile-bucket "Bucket name for temporary file uploads (10-minute expiry)." - "tempfile") + stsch/tempfile-bucket) (def upload-session-bucket "Bucket name for chunked-upload chunks." - "upload-session") + stsch/upload-session-bucket) (def valid-buckets - #{"file-media-object" - "team-font-variant" - "file-object-thumbnail" - "file-thumbnail" - "profile" - "organization" - tempfile-bucket - upload-session-bucket - "file-data" - "file-data-fragment" - "file-change"}) + stsch/metadata-buckets) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Storage Module State @@ -118,18 +109,37 @@ params params)) +;; Matches both metadata encodings while Transit rows still exist. The +;; UNION ALL keeps each branch indexable: migration 0068 covers the legacy +;; "~:" keys and 0156 covers the plain JSON keys. Phase 3 drops the legacy +;; branch once no "~:" rows remain. +(def ^:private sql:get-database-object-by-hash + "( + select * from storage_object + where (metadata->>'~:hash') = ? + and (metadata->>'~:bucket') = ? + and backend = ? + and deleted_at is null + and status = 'valid' + limit 1 + ) union all ( + select * from storage_object + where (metadata->>'hash') = ? + and (metadata->>'bucket') = ? + and backend = ? + and deleted_at is null + and status = 'valid' + limit 1 + ) limit 1") + +;; NOTE: metadata is left encoded; row->storage-object is responsible for +;; decoding it. (defn- get-database-object-by-hash [connectable backend bucket hash] - (let [sql (str "select * from storage_object " - " where (metadata->>'~:hash') = ? " - " and (metadata->>'~:bucket') = ? " - " and backend = ?" - " and deleted_at is null" - " and status = 'valid'" - " limit 1")] - ;; NOTE: metadata is left encoded; row->storage-object is - ;; responsible for decoding it. - (db/exec-one! connectable [sql hash bucket (name backend)]))) + (let [backend (name backend)] + (db/exec-one! connectable [sql:get-database-object-by-hash + hash bucket backend + hash bucket backend]))) (defn- promote-object! [storage object] @@ -147,7 +157,7 @@ res)) (defn row->storage-object [res] - (let [mdata (or (some-> (:metadata res) (db/decode-transit-pgobject)) {})] + (let [mdata (or (some-> (:metadata res) (stsch/decode-metadata)) {})] (impl/storage-object (:id res) (:size res) @@ -289,7 +299,7 @@ {:id id :size (impl/get-size content) :backend (name backend) - :metadata (db/tjson mdata) + :metadata (stsch/encode-metadata mdata) :deleted-at expired-at :touched-at touched-at :status "pending"}) diff --git a/backend/src/app/storage/gc_deleted.clj b/backend/src/app/storage/gc_deleted.clj index d9dc908e0a..7048bd4d2e 100644 --- a/backend/src/app/storage/gc_deleted.clj +++ b/backend/src/app/storage/gc_deleted.clj @@ -51,7 +51,7 @@ "DELETE FROM storage_object WHERE id = ANY(?::uuid[])") -(defn- delete-sobjects! +(defn- delete-sobjects [conn ids] (let [ids (db/create-array conn "uuid" ids)] (-> (db/exec-one! conn [sql:delete-sobjects ids]) @@ -61,7 +61,7 @@ "DELETE FROM upload_session_chunk WHERE object_id = ANY(?::uuid[])") -(defn- delete-upload-session-chunks! +(defn- delete-upload-session-chunks "Remove the chunk mappings for the given storage object ids. This must run before the storage_object rows are deleted: the upload_session_chunk foreign keys are ON DELETE NO ACTION." @@ -75,7 +75,7 @@ deleted_at = NOW() + INTERVAL '1 day' WHERE id = ANY(?::uuid[])") -(defn- increment-attempts-and-defer! +(defn- increment-attempts-and-defer [conn ids] (let [ids (db/create-array conn "uuid" ids)] (db/exec-one! conn [sql:increment-attempts-and-defer ids]))) @@ -85,7 +85,7 @@ WHERE id = ANY(?::uuid[]) AND deletion_attempts >= ?") -(defn- delete-give-up! +(defn- delete-give-up [conn ids] (let [ids (db/create-array conn "uuid" ids)] (db/exec-one! conn [sql:delete-give-up ids max-attempts]))) @@ -93,7 +93,7 @@ (defn- process-chunk "Attempt to delete a chunk of storage objects from a specific backend. - This function runs inside the caller's transaction (clean-deleted!) — + This function runs inside the caller's transaction (clean-deleted) — it does NOT open its own transaction. The caller is responsible for ensuring the rows are locked via FOR UPDATE SKIP LOCKED before calling. @@ -121,18 +121,18 @@ ;; storage_object rows (NO ACTION foreign keys). It only affects ;; objects of the upload-session bucket; for any other bucket the ;; delete matches no rows. - (delete-upload-session-chunks! conn ok-ids) - (delete-sobjects! conn ok-ids)) + (delete-upload-session-chunks conn ok-ids) + (delete-sobjects conn ok-ids)) (when (seq fail-ids) - (increment-attempts-and-defer! conn fail-ids) + (increment-attempts-and-defer conn fail-ids) ;; NOTE: same NO ACTION ordering as above: the give-up DELETE below ;; removes storage_object rows, so chunk mappings must go first. ;; Deferred objects keep their rows; only the mapping of a ;; permanently given-up object disappears early, and that object is ;; already deleted-marked. - (delete-upload-session-chunks! conn fail-ids) - (let [given-up (delete-give-up! conn fail-ids)] + (delete-upload-session-chunks conn fail-ids) + (let [given-up (delete-give-up conn fail-ids)] (when (pos? (db/get-update-count given-up)) (l/wrn :hint "giving up on orphan blob after max attempts" :ids fail-ids @@ -160,7 +160,7 @@ [conn size] (db/exec! conn [sql:get-deleted-chunk (ct/now) size])) -(defn- clean-deleted! +(defn- clean-deleted [cfg] (loop [total 0] (let [deleted (db/tx-run! cfg @@ -184,6 +184,6 @@ (defmethod ig/init-key ::handler [_ cfg] (fn [_] - (let [total (clean-deleted! cfg)] + (let [total (clean-deleted cfg)] (l/inf :hint "task finished" :total total) {:deleted total}))) diff --git a/backend/src/app/storage/gc_touched.clj b/backend/src/app/storage/gc_touched.clj index 260739b02f..f73b6024f1 100644 --- a/backend/src/app/storage/gc_touched.clj +++ b/backend/src/app/storage/gc_touched.clj @@ -24,6 +24,7 @@ [app.common.logging :as l] [app.common.time :as ct] [app.db :as db] + [app.metrics :as mtx] [app.storage :as sto] [app.storage.impl :as impl] [integrant.core :as ig])) @@ -95,7 +96,7 @@ SET touched_at = NULL WHERE id = ANY(?::uuid[])") -(defn- mark-freeze-in-bulk! +(defn- mark-freeze-in-bulk [conn ids] (let [ids (db/create-array conn "uuid" ids)] (db/exec-one! conn [sql:mark-freeze-in-bulk ids]))) @@ -106,11 +107,21 @@ touched_at = NULL WHERE id = ANY(?::uuid[])") -(defn- mark-delete-in-bulk! +(defn- mark-delete-in-bulk [conn ids] (let [ids (db/create-array conn "uuid" ids)] (db/exec-one! conn [sql:mark-delete-in-bulk (ct/now) ids]))) +(def ^:private sql:defer-in-bulk + "UPDATE storage_object + SET touched_at = ? + WHERE id = ANY(?::uuid[])") + +(defn- defer-in-bulk + [conn ids timestamp] + (let [ids (db/create-array conn "uuid" ids)] + (db/exec-one! conn [sql:defer-in-bulk timestamp ids]))) + ;; NOTE: A getter that retrieves the key which will be used for group ;; ids; previously we have no value, then we introduced the ;; `:reference` prop, and then it is renamed to `:bucket` and now is @@ -125,12 +136,17 @@ ;; have value, it means :file-media-object. (defn- lookup-bucket - [{:keys [metadata]}] + [{:keys [id metadata]}] (or (some-> metadata :bucket) - (some-> metadata :reference d/name) - sto/default-bucket)) + (do + ;; Only reachable when the metadata column is NULL: the decode + ;; always sets :bucket on non-nil metadata (0155 also backfills + ;; NULL columns). Keep working, but make it visible. + (l/wrn :hint "storage object without bucket metadata, using fallback" + :id (str id)) + sto/default-bucket))) -(defn- process-objects! +(defn- process-objects [conn has-refs? bucket objects] (loop [to-freeze #{} to-delete #{} @@ -148,31 +164,37 @@ :bucket bucket) (recur to-freeze (conj to-delete id) (rest objects)))) (do - (some->> (seq to-freeze) (mark-freeze-in-bulk! conn)) - (some->> (seq to-delete) (mark-delete-in-bulk! conn)) + (some->> (seq to-freeze) (mark-freeze-in-bulk conn)) + (some->> (seq to-delete) (mark-delete-in-bulk conn)) [(count to-freeze) (count to-delete)])))) -(defn- process-bucket! +(defn- process-bucket [conn bucket objects] (cond - (= bucket "file-media-object") (process-objects! conn has-file-media-object-refs? bucket objects) - (= bucket "team-font-variant") (process-objects! conn has-team-font-variant-refs? bucket objects) - (= bucket "file-object-thumbnail") (process-objects! conn has-file-object-thumbnails-refs? bucket objects) - (= bucket "file-thumbnail") (process-objects! conn has-file-thumbnails-refs? bucket objects) - (= bucket "profile") (process-objects! conn has-profile-refs? bucket objects) - (= bucket "file-data") (process-objects! conn has-file-data-refs? bucket objects) - (= bucket sto/tempfile-bucket) (process-objects! conn (constantly false) sto/tempfile-bucket objects) - (= bucket sto/upload-session-bucket) (process-objects! conn (constantly false) sto/upload-session-bucket objects) - (= bucket "organization") (process-objects! conn (constantly false) bucket objects) + (= bucket "file-media-object") (process-objects conn has-file-media-object-refs? bucket objects) + (= bucket "team-font-variant") (process-objects conn has-team-font-variant-refs? bucket objects) + (= bucket "file-object-thumbnail") (process-objects conn has-file-object-thumbnails-refs? bucket objects) + (= bucket "file-thumbnail") (process-objects conn has-file-thumbnails-refs? bucket objects) + (= bucket "profile") (process-objects conn has-profile-refs? bucket objects) + (= bucket "file-data") (process-objects conn has-file-data-refs? bucket objects) + (= bucket sto/tempfile-bucket) (process-objects conn (constantly false) sto/tempfile-bucket objects) + (= bucket sto/upload-session-bucket) (process-objects conn (constantly false) sto/upload-session-bucket objects) + (= bucket "organization") (process-objects conn (constantly false) bucket objects) :else (ex/raise :type :internal :code :unexpected-unknown-reference :hint (dm/fmt "unknown reference '%'" bucket)))) -(defn process-chunk! +(defn- defer-poison + "Defer corrupt rows by one day in their own transaction, separate from + the healthy-chunk work." + [{:keys [::db/conn]} poison-ids] + (defer-in-bulk conn poison-ids (ct/plus (ct/now) {:days 1}))) + +(defn process-chunk [{:keys [::db/conn]} chunk] (reduce-kv (fn [[nfo ndo] bucket objects] - (let [[nfo' ndo'] (process-bucket! conn bucket objects)] + (let [[nfo' ndo'] (process-bucket conn bucket objects)] [(+ nfo nfo') (+ ndo ndo')])) [0 0] @@ -189,21 +211,45 @@ SKIP LOCKED LIMIT 10") +(defn- try-decode-row + "Decode a touched row, capturing corrupt metadata as poison instead + of aborting the whole chunk. Poison rows are deferred by the caller." + [row] + (try + [:ok (impl/decode-row row)] + ;; Exception, not Throwable: JVM Errors (OOM, StackOverflow) must + ;; not be swallowed as a deferrable data problem. + (catch Exception cause + (l/err :hint "storage object with corrupt metadata, deferring evaluation" + :id (str (:id row)) + :cause cause) + [:poison (:id row)]))) + (defn get-chunk [conn timestamp] - (->> (db/exec! conn [sql:get-touched-storage-objects timestamp]) - (map impl/decode-row) - (not-empty))) + (let [grouped (->> (db/exec! conn [sql:get-touched-storage-objects timestamp]) + (map try-decode-row) + (group-by first))] + {:chunk (not-empty (mapv second (:ok grouped))) + :poison (not-empty (mapv second (:poison grouped)))})) -(defn- process-touched! - [{:keys [::db/pool ::timestamp] :as cfg}] +(defn- process-touched + [{:keys [::db/pool ::mtx/metrics ::timestamp] :as cfg}] (loop [freezed 0 deleted 0] - (if-let [chunk (get-chunk pool timestamp)] - (let [[nfo ndo] (db/tx-run! cfg process-chunk! chunk)] - (recur (long (+ freezed nfo)) - (long (+ deleted ndo)))) - {:freeze freezed :delete deleted}))) + (let [{:keys [chunk poison]} (get-chunk pool timestamp)] + (when (seq poison) + (mtx/run! metrics :id :storage-gc-poison :inc (count poison)) + (db/tx-run! cfg defer-poison poison)) + ;; Keep draining after a poison-only batch: the deferred rows leave + ;; the selection and the next batch may hold healthy objects. + (if (or (seq chunk) (seq poison)) + (let [[nfo ndo] (if (seq chunk) + (db/tx-run! cfg process-chunk chunk) + [0 0])] + (recur (long (+ freezed nfo)) + (long (+ deleted ndo)))) + {:freeze freezed :delete deleted})))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; HANDLER @@ -211,7 +257,8 @@ (defmethod ig/assert-key ::handler [_ params] - (assert (db/pool? (::db/pool params)) "expect valid storage")) + (assert (db/pool? (::db/pool params)) "expect valid storage") + (assert (mtx/metrics? (::mtx/metrics params)) "expect valid metrics")) (defmethod ig/expand-key ::handler [k v] @@ -223,5 +270,5 @@ (let [threshold (if (:skip-delay props) (ct/now) (ct/minus (ct/now) min-age))] - (process-touched! (assoc cfg ::timestamp threshold))))) + (process-touched (assoc cfg ::timestamp threshold))))) diff --git a/backend/src/app/storage/impl.clj b/backend/src/app/storage/impl.clj index 76fafe2cac..a6b3d2ca09 100644 --- a/backend/src/app/storage/impl.clj +++ b/backend/src/app/storage/impl.clj @@ -9,8 +9,8 @@ (:require [app.common.data.macros :as dm] [app.common.exceptions :as ex] - [app.db :as db] [app.storage :as-alias sto] + [app.storage.schema :as stsch] [buddy.core.codecs :as bc] [buddy.core.hash :as bh] [clojure.java.io :as jio] @@ -26,7 +26,7 @@ [{:keys [metadata] :as row}] (cond-> row (some? metadata) - (assoc :metadata (db/decode-transit-pgobject metadata)))) + (assoc :metadata (stsch/decode-metadata metadata)))) ;; --- API Definition diff --git a/backend/src/app/storage/schema.clj b/backend/src/app/storage/schema.clj new file mode 100644 index 0000000000..2ded471780 --- /dev/null +++ b/backend/src/app/storage/schema.clj @@ -0,0 +1,149 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns app.storage.schema + "Schema, validation and encode/decode helpers for + `storage_object.metadata`. + + Phase 1: reads accept both the legacy Transit encoding and plain + JSON (sniffed by the `\"~:` marker); writes validate + normalize + but still serialize as Transit unless the + `:storage-metadata-as-json` config flag is set." + (:require + [app.common.data :as d] + [app.common.exceptions :as ex] + [app.common.schema :as sm] + [app.config :as cf] + [app.db :as db] + [cuerdas.core :as str]) + (:import + org.postgresql.util.PGobject)) + +(def default-bucket + "file-media-object") + +(def tempfile-bucket + "Bucket name for temporary file uploads (10-minute expiry)." + "tempfile") + +(def upload-session-bucket + "Bucket name for chunked-upload chunks." + "upload-session") + +(def bucket-requirements + "Canonical buckets and the extra keys each one must carry, so a new + bucket and its contract live in one place. Listed keys are declared as + optional in `schema:metadata` and only checked for presence; their type + is enforced by the map. Only buckets whose keys are read somewhere need + an entry: `file-data` ids are resolved by the GC, `organization-id` is + the logo's owner." + {"file-media-object" #{} + "team-font-variant" #{} + "file-object-thumbnail" #{} + "file-thumbnail" #{} + "profile" #{} + "organization" #{:organization-id} + tempfile-bucket #{} + upload-session-bucket #{} + "file-data" #{:file-id :id} + "file-data-fragment" #{} + "file-change" #{}}) + +(def metadata-buckets + "Canonical bucket set. `app.storage/valid-buckets` aliases it so the + list lives in exactly one place." + (set (keys bucket-requirements))) + +(defn- bucket-requirements-present? + [{:keys [bucket] :as mdata}] + (every? #(some? (get mdata %)) (get bucket-requirements bucket #{}))) + +(def schema:metadata + "Closed schema for `storage_object.metadata`. A single shape shared by + every bucket: `:bucket` is checked against `metadata-buckets` and the + rest are typed optional keys. `bucket-requirements` adds the per-bucket + presence checks." + [:and + [:map {:closed true} + [:bucket [::sm/one-of {:format :string} metadata-buckets]] + [:content-type :string] + [:hash {:optional true} :string] + [:profile-id {:optional true} ::sm/uuid] + [:organization-id {:optional true} ::sm/uuid] + [:file-id {:optional true} ::sm/uuid] + [:id {:optional true} ::sm/uuid]] + [:fn {:error/message "storage metadata is missing a required key for its bucket"} + bucket-requirements-present?]]) + +(defn- ->bucket + [v] + (cond + (string? v) v + (keyword? v) (d/name v) + :else (str v))) + +(defn- normalize-metadata + "Bring decoded (or incoming) metadata to its canonical shape: + `:bucket` is always present (`:reference` is mapped to it, keyword + or string, and defaults to `default-bucket`), and the legacy + `:reference`, `:upload-id` and `:chunk-index` keys are dropped." + [mdata] + (let [bucket (or (not-empty (->bucket (:bucket mdata))) + (not-empty (->bucket (:reference mdata))) + default-bucket)] + (-> mdata + (dissoc :reference :upload-id :chunk-index) + (assoc :bucket bucket)))) + +;; Decoder and encoder are built once per process: creating them +;; compiles the closed schema, and `decode-metadata` runs on every read +;; path (get-object, dedup probes, GC batches). +(def ^:private decode-metadata-fn + (sm/decode-fn schema:metadata (sm/json-transformer))) + +(def ^:private encode-metadata-fn + (sm/encoder schema:metadata (sm/json-transformer))) + +(defn- transit-encoded? + "Legacy metadata is Transit json-verbose, where every top-level key is + a keyword, so the document starts with `{\"~:`. Anchoring the check to + the start avoids reading a plain-JSON document as Transit just because + one of its values contains `\"~:`." + [value] + (and (string? value) + (str/starts-with? (str/trim value) "{\"~:"))) + +(defn decode-metadata + "Decode storage metadata from a PGobject, accepting both the legacy + Transit encoding (sniffed by the `\"~:` marker) and plain JSON. + Always returns the normalized shape with id fields as UUIDs." + [o] + (when (some? o) + (let [raw (if (transit-encoded? (.getValue ^PGobject o)) + (db/decode-transit-pgobject o) + (db/decode-json-pgobject o))] + (when-not (map? raw) + (ex/raise :type :internal + :code :invalid-storage-metadata + :hint "expected a map on storage object metadata")) + (decode-metadata-fn (normalize-metadata raw))))) + +(def ^:private check-metadata + (sm/check-fn schema:metadata + :hint "invalid storage object metadata" + :type :validation + :code :invalid-storage-metadata)) + +(defn encode-metadata + "Validate, normalize and encode storage metadata into a PGobject + ready for the `metadata` column. Serializes as Transit while the + `:storage-metadata-as-json` config flag is unset, as plain JSON + when it is set." + [mdata] + (let [mdata (check-metadata (normalize-metadata mdata))] + (if (cf/get :storage-metadata-as-json) + (db/json (encode-metadata-fn mdata)) + (db/tjson mdata)))) diff --git a/backend/test/backend_tests/http_assets_test.clj b/backend/test/backend_tests/http_assets_test.clj index 095d04f3e7..a5e60e9965 100644 --- a/backend/test/backend_tests/http_assets_test.clj +++ b/backend/test/backend_tests/http_assets_test.clj @@ -56,7 +56,17 @@ :bucket bucket :content-type "text/plain"} (some? profile-id) - (assoc :profile-id profile-id))))) + (assoc :profile-id profile-id) + + ;; file-data objects require file/id + ;; references for GC (has-file-data-refs?). + (= bucket "file-data") + (assoc :file-id (uuid/random) + :id (uuid/random)) + + ;; organization objects require their owner id. + (= bucket "organization") + (assoc :organization-id (uuid/random)))))) (defn- make-metrics [] diff --git a/backend/test/backend_tests/rpc_management_test.clj b/backend/test/backend_tests/rpc_management_test.clj index 570b858aaf..4b449af993 100644 --- a/backend/test/backend_tests/rpc_management_test.clj +++ b/backend/test/backend_tests/rpc_management_test.clj @@ -85,8 +85,7 @@ (configure-storage-backend)) sobject (sto/put-object! storage {::sto/content (sto/content "content") - :content-type "text/plain" - :other "data"}) + :content-type "text/plain"}) profile (th/create-profile* 1 {:is-active true}) project (th/create-project* 1 {:team-id (:default-team-id profile) :profile-id (:id profile)}) @@ -157,8 +156,7 @@ (let [storage (-> (:app.storage/storage th/*system*) (configure-storage-backend)) sobject (sto/put-object! storage {::sto/content (sto/content "content") - :content-type "text/plain" - :other "data"}) + :content-type "text/plain"}) profile (th/create-profile* 1 {:is-active true}) project (th/create-project* 1 {:team-id (:default-team-id profile) @@ -214,8 +212,7 @@ (configure-storage-backend)) sobject (sto/put-object! storage {::sto/content (sto/content "content") - :content-type "text/plain" - :other "data"}) + :content-type "text/plain"}) profile (th/create-profile* 1 {:is-active true}) project (th/create-project* 1 {:team-id (:default-team-id profile) @@ -278,8 +275,7 @@ (let [storage (-> (:app.storage/storage th/*system*) (configure-storage-backend)) sobject (sto/put-object! storage {::sto/content (sto/content "content") - :content-type "text/plain" - :other "data"}) + :content-type "text/plain"}) profile (th/create-profile* 1 {:is-active true}) project (th/create-project* 1 {:team-id (:default-team-id profile) :profile-id (:id profile)}) diff --git a/backend/test/backend_tests/storage_metadata_test.clj b/backend/test/backend_tests/storage_metadata_test.clj new file mode 100644 index 0000000000..4efb4c80df --- /dev/null +++ b/backend/test/backend_tests/storage_metadata_test.clj @@ -0,0 +1,276 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns backend-tests.storage-metadata-test + (:require + [app.config :as cf] + [app.db :as db] + [app.storage.schema :as stsch] + [clojure.string :as str] + [clojure.test :as t]) + (:import + org.postgresql.util.PGobject)) + +(defn- pgobject + [^String value] + (doto (PGobject.) + (.setType "jsonb") + (.setValue value))) + +;; Raw Transit payloads shaped like the production rows sampled on +;; 2026-09-11 (verbose transit: "~:key" keys, "~u" uuid values, +;; "~:xxx" keyword values). +(def ^:private transit-media + "{\"~:hash\":\"blake2b:9f1c2e\",\"~:bucket\":\"file-media-object\",\"~:content-type\":\"image/png\"}") + +(def ^:private transit-tempfile + "{\"~:hash\":\"blake2b:9f1c2e\",\"~:bucket\":\"tempfile\",\"~:content-type\":\"application/zip\",\"~:profile-id\":\"~u86907e95-1cb8-8122-8008-4eb7ba07d89d\"}") + +(def ^:private transit-legacy-reference + "{\"~:reference\":\"~:file-media-object\",\"~:content-type\":\"image/png\",\"~:hash\":\"blake2b:9f1c2e\"}") + +(def ^:private transit-no-bucket + "{\"~:content-type\":\"image/svg+xml\"}") + +(def ^:private transit-chunk-leftovers + "{\"~:bucket\":\"tempfile\",\"~:content-type\":\"application/zip\",\"~:upload-id\":\"~u86907e95-1cb8-8122-8008-4eb7ba07d89d\",\"~:chunk-index\":3}") + +(def ^:private json-file-data + "{\"bucket\":\"file-data\",\"content-type\":\"application/octet-stream\",\"file-id\":\"86907e95-1cb8-8122-8008-4eb7ba07d89d\",\"id\":\"83df2f92-6bd4-4e6d-9c9a-3f6d2b1a4c55\"}") + +(t/deftest decode-transit-returns-native-types + (let [mdata (stsch/decode-metadata (pgobject transit-tempfile))] + (t/is (= "tempfile" (:bucket mdata))) + (t/is (= "application/zip" (:content-type mdata))) + (t/is (= "blake2b:9f1c2e" (:hash mdata))) + (t/is (uuid? (:profile-id mdata))) + (t/is (= (parse-uuid "86907e95-1cb8-8122-8008-4eb7ba07d89d") + (:profile-id mdata))))) + +(t/deftest decode-transit-maps-reference-to-bucket + (let [mdata (stsch/decode-metadata (pgobject transit-legacy-reference))] + (t/is (= "file-media-object" (:bucket mdata))) + (t/is (= "image/png" (:content-type mdata))) + (t/is (not (contains? mdata :reference))))) + +(t/deftest decode-transit-defaults-missing-bucket + (let [mdata (stsch/decode-metadata (pgobject transit-no-bucket))] + (t/is (= "file-media-object" (:bucket mdata))) + (t/is (= "image/svg+xml" (:content-type mdata))))) + +(t/deftest decode-transit-drops-chunk-leftovers + (let [mdata (stsch/decode-metadata (pgobject transit-chunk-leftovers))] + (t/is (= "tempfile" (:bucket mdata))) + (t/is (not (contains? mdata :upload-id))) + (t/is (not (contains? mdata :chunk-index))))) + +(t/deftest decode-json-coerces-uuids + (let [mdata (stsch/decode-metadata (pgobject json-file-data))] + (t/is (= "file-data" (:bucket mdata))) + (t/is (uuid? (:file-id mdata))) + (t/is (uuid? (:id mdata))) + (t/is (= (parse-uuid "86907e95-1cb8-8122-8008-4eb7ba07d89d") + (:file-id mdata))))) + +(t/deftest sniff-has-no-false-positives-on-values + ;; "~:" inside a value (not anchored to a quote) must not trigger + ;; the transit branch. + (let [mdata (stsch/decode-metadata + (pgobject "{\"bucket\":\"tempfile\",\"content-type\":\"text/plain;~:x\"}"))] + (t/is (= "tempfile" (:bucket mdata))) + (t/is (= "text/plain;~:x" (:content-type mdata))))) + +(t/deftest encode-writes-transit-by-default + ;; Pinned off: without the binding this test inherits the ambient + ;; config and proves nothing where the flag is set. + (binding [cf/config (assoc cf/config :storage-metadata-as-json nil)] + (let [encoded (stsch/encode-metadata {:bucket "file-media-object" + :content-type "image/png" + :hash "blake2b:9f1c2e"}) + value (.getValue ^PGobject encoded)] + (t/is (string? value)) + (t/is (str/includes? value "\"~:bucket\"")) + (t/is (not (str/includes? value "\"~:reference\"")))))) + +(t/deftest encode-normalizes-legacy-input + (binding [cf/config (assoc cf/config :storage-metadata-as-json nil)] + (let [encoded (stsch/encode-metadata {:reference :file-media-object + :content-type "image/png"}) + value (.getValue ^PGobject encoded) + decoded (stsch/decode-metadata encoded)] + (t/is (= "file-media-object" (:bucket decoded))) + (t/is (not (str/includes? value "\"~:reference\"")))))) + +(t/deftest encode-writes-plain-json-with-flag + (binding [cf/config (assoc cf/config :storage-metadata-as-json true)] + (let [file-id (parse-uuid "86907e95-1cb8-8122-8008-4eb7ba07d89d") + encoded (stsch/encode-metadata {:bucket "file-data" + :content-type "application/octet-stream" + :file-id file-id + :id (parse-uuid "83df2f92-6bd4-4e6d-9c9a-3f6d2b1a4c55")}) + value (.getValue ^PGobject encoded)] + (t/is (str/includes? value "\"bucket\"")) + (t/is (not (str/includes? value "\"~:"))) + ;; uuids travel as plain strings on the JSON encoding + (t/is (str/includes? value (str file-id))) + (let [decoded (stsch/decode-metadata encoded)] + (t/is (uuid? (:file-id decoded))) + (t/is (= file-id (:file-id decoded))))))) + +(t/deftest encode-rejects-string-uuids + ;; Encoding does not coerce input types: callers must pass native UUIDs + ;; (every producer does; reads still coerce on the way back). + (t/is (thrown-with-msg? clojure.lang.ExceptionInfo + #"invalid storage object metadata" + (stsch/encode-metadata {:bucket "tempfile" + :content-type "application/zip" + :profile-id "86907e95-1cb8-8122-8008-4eb7ba07d89d"})))) + +(t/deftest encode-rejects-unknown-bucket + (t/is (thrown-with-msg? clojure.lang.ExceptionInfo + #"invalid storage object metadata" + (stsch/encode-metadata {:bucket "no-such-bucket" + :content-type "image/png"})))) + +(t/deftest encode-rejects-extra-key + (t/is (thrown-with-msg? clojure.lang.ExceptionInfo + #"invalid storage object metadata" + (stsch/encode-metadata {:bucket "file-media-object" + :content-type "image/png" + :other "data"})))) + +(t/deftest encode-rejects-malformed-uuid + (t/is (thrown-with-msg? clojure.lang.ExceptionInfo + #"invalid storage object metadata" + (stsch/encode-metadata {:bucket "file-data" + :content-type "application/octet-stream" + :file-id "not-a-uuid" + :id "83df2f92-6bd4-4e6d-9c9a-3f6d2b1a4c55"})))) + +(t/deftest encode-rejects-missing-content-type + (t/is (thrown-with-msg? clojure.lang.ExceptionInfo + #"invalid storage object metadata" + (stsch/encode-metadata {:bucket "file-media-object"})))) + +(t/deftest encode-rejects-missing-file-data-ids + ;; Both ids are required for file-data; omitting one fails the bucket + ;; requirement check (the file-id here is a native UUID, so the map + ;; itself is valid). + (t/is (thrown-with-msg? clojure.lang.ExceptionInfo + #"invalid storage object metadata" + (stsch/encode-metadata {:bucket "file-data" + :content-type "application/octet-stream" + :file-id (parse-uuid "86907e95-1cb8-8122-8008-4eb7ba07d89d")})))) + +(t/deftest encode-rejects-missing-organization-id + (t/is (thrown-with-msg? clojure.lang.ExceptionInfo + #"invalid storage object metadata" + (stsch/encode-metadata {:bucket "organization" + :content-type "image/svg+xml"})))) + +(t/deftest decode-real-transit-payload + ;; Same legacy shape as above but produced by the real transit + ;; encoder, so the test breaks if the encoding ever drifts. + (let [mdata (stsch/decode-metadata + (db/tjson {:reference :team-font-variant + :content-type "font/woff2"}))] + (t/is (= "team-font-variant" (:bucket mdata))) + (t/is (not (contains? mdata :reference))))) + +(t/deftest transit-and-json-encodings-decode-to-same-shape + (let [mdata {:bucket "tempfile" + :content-type "application/zip" + :profile-id (parse-uuid "86907e95-1cb8-8122-8008-4eb7ba07d89d")} + transit (binding [cf/config (assoc cf/config :storage-metadata-as-json nil)] + (stsch/decode-metadata (stsch/encode-metadata mdata))) + json (binding [cf/config (assoc cf/config :storage-metadata-as-json true)] + (stsch/decode-metadata (stsch/encode-metadata mdata)))] + (t/is (= transit json)) + (t/is (= mdata json)))) + +(t/deftest flag-on-and-off-differ-only-in-encoding-family + ;; The Phase 2 rollback contract: flipping the flag changes how the + ;; same logical metadata hits the disk, never what it means. + (let [mdata {:bucket "file-media-object" + :content-type "image/png" + :hash "blake2b:9f1c2e"} + transit (binding [cf/config (assoc cf/config :storage-metadata-as-json nil)] + (.getValue ^PGobject (stsch/encode-metadata mdata))) + json (binding [cf/config (assoc cf/config :storage-metadata-as-json true)] + (.getValue ^PGobject (stsch/encode-metadata mdata)))] + (t/is (str/includes? transit "\"~:bucket\"")) + (t/is (str/includes? json "\"bucket\"")) + (t/is (not (str/includes? json "\"~:"))) + (t/is (= (stsch/decode-metadata (pgobject transit)) + (stsch/decode-metadata (pgobject json)))))) + +(t/deftest decode-json-keeps-hash-byte-exact + ;; Dedup matches on the hash string; it must survive the JSON + ;; roundtrip untouched. + (let [mdata (stsch/decode-metadata + (pgobject "{\"bucket\":\"file-media-object\",\"content-type\":\"image/png\",\"hash\":\"blake2b:9f1c2e\"}"))] + (t/is (= "blake2b:9f1c2e" (:hash mdata))))) + +(def ^:private bucket-samples + ;; One representative payload per bucket, with every required key and + ;; one optional key where the bucket declares any. + [["file-media-object" + {:bucket "file-media-object" :content-type "image/png" :hash "blake2b:9f1c2e"}] + ["team-font-variant" + {:bucket "team-font-variant" :content-type "font/woff2"}] + ["file-object-thumbnail" + {:bucket "file-object-thumbnail" :content-type "image/png"}] + ["file-thumbnail" + {:bucket "file-thumbnail" :content-type "image/png"}] + ["profile" + {:bucket "profile" :content-type "image/png"}] + ["organization" + {:bucket "organization" :content-type "image/svg+xml" + :organization-id #uuid "11111111-2222-3333-4444-555555555555"}] + ["tempfile" + {:bucket "tempfile" :content-type "application/zip" + :profile-id #uuid "86907e95-1cb8-8122-8008-4eb7ba07d89d"}] + ["upload-session" + {:bucket "upload-session" :content-type "application/octet-stream"}] + ["file-data" + {:bucket "file-data" :content-type "application/octet-stream" + :file-id #uuid "86907e95-1cb8-8122-8008-4eb7ba07d89d" + :id #uuid "83df2f92-6bd4-4e6d-9c9a-3f6d2b1a4c55"}] + ["file-data-fragment" + {:bucket "file-data-fragment" :content-type "application/octet-stream"}] + ["file-change" + {:bucket "file-change" :content-type "application/octet-stream"}]]) + +(t/deftest encode-decode-roundtrips-every-bucket + (binding [cf/config (assoc cf/config :storage-metadata-as-json nil)] + (doseq [[bucket mdata] bucket-samples] + (t/testing bucket + (t/is (= mdata (stsch/decode-metadata (stsch/encode-metadata mdata)))))))) + +(t/deftest json-encoding-roundtrips-every-bucket + (binding [cf/config (assoc cf/config :storage-metadata-as-json true)] + (doseq [[bucket mdata] bucket-samples] + (t/testing bucket + (t/is (= mdata (stsch/decode-metadata (stsch/encode-metadata mdata)))))))) + +(t/deftest metadata-buckets-is-the-canonical-set + (t/is (= #{"file-media-object" "team-font-variant" "file-object-thumbnail" + "file-thumbnail" "profile" "organization" "tempfile" + "upload-session" "file-data" "file-data-fragment" "file-change"} + stsch/metadata-buckets))) + +(t/deftest decode-metadata-returns-nil-for-nil + (t/is (nil? (stsch/decode-metadata nil)))) + +(t/deftest sniff-treats-leading-tilde-value-as-json + ;; A plain-JSON value starting with `~:` must not switch the reader to + ;; the Transit branch: the sniff is anchored to the first key. + (let [encoded (binding [cf/config (assoc cf/config :storage-metadata-as-json true)] + (stsch/encode-metadata {:bucket "tempfile" + :content-type "~:not-transit"})) + decoded (stsch/decode-metadata encoded)] + (t/is (= "~:not-transit" (:content-type decoded))) + (t/is (= "tempfile" (:bucket decoded))))) diff --git a/backend/test/backend_tests/storage_test.clj b/backend/test/backend_tests/storage_test.clj index 54953620a0..abdd8c1503 100644 --- a/backend/test/backend_tests/storage_test.clj +++ b/backend/test/backend_tests/storage_test.clj @@ -9,12 +9,16 @@ [app.common.exceptions :as ex] [app.common.time :as ct] [app.common.uuid :as uuid] + [app.config :as cf] [app.db :as db] + [app.metrics :as mtx] + [app.metrics.definition :as-alias mdef] [app.rpc :as-alias rpc] [app.storage :as sto] [app.storage.fs :as-alias sto.fs] [app.storage.impl :as impl] [app.storage.s3 :as-alias sto.s3] + [app.storage.schema :as stsch] [backend-tests.helpers :as th] [clojure.test :as t] [cuerdas.core :as str] @@ -23,6 +27,11 @@ [mockery.core :refer [with-mocks]] [promesa.core :as p]) (:import + (io.prometheus.client + Counter + Counter$Child) + (org.postgresql.util + PGobject) (software.amazon.awssdk.services.s3 S3AsyncClient) (software.amazon.awssdk.services.s3.model @@ -41,24 +50,51 @@ [storage] (assoc storage ::sto/backend :fs)) +(defn- poison-counter-value + "Read the process-wide gc-poison counter from the test system metrics." + [] + (let [metrics (:app.metrics/metrics th/*system*) + collector (mtx/get-collector metrics :storage-gc-poison) + instance (::mdef/instance collector)] + (.get ^Counter$Child (.labels ^Counter instance (into-array String []))))) + (t/deftest put-and-retrieve-object (let [storage (-> (:app.storage/storage th/*system*) (configure-storage-backend)) content (sto/content "content") object (sto/put-object! storage {::sto/content content - :content-type "text/plain" - :other "data"})] + :content-type "text/plain"})] (t/is (sto/object? object)) (t/is (fs/path? (sto/get-object-path storage object))) (t/is (nil? (:expired-at object))) (t/is (= :fs (:backend object))) - (t/is (= "data" (:other (meta object)))) + (t/is (= "file-media-object" (:bucket (meta object)))) (t/is (= "text/plain" (:content-type (meta object)))) (t/is (= "content" (slurp (sto/get-object-data storage object)))) (t/is (= "content" (slurp (sto/get-object-path storage object)))))) +(t/deftest put-and-retrieve-with-json-metadata-flag + ;; End-to-end through the jsonb column: with the flag on, the row is + ;; stored as plain JSON and still decodes to native types. + (binding [cf/config (assoc cf/config :storage-metadata-as-json true)] + (let [storage (-> (:app.storage/storage th/*system*) + (configure-storage-backend)) + profile (uuid/random) + object (sto/put-object! storage {::sto/content (sto/content "content") + :bucket "tempfile" + :content-type "application/zip" + :profile-id profile}) + row (th/db-exec-one! ["select metadata::text as metadata from storage_object where id = ?" + (:id object)])] + (t/is (not (str/includes? (:metadata row) "\"~:"))) + (t/is (str/includes? (:metadata row) "\"bucket\"")) + (let [loaded (sto/get-object storage (:id object))] + (t/is (= "tempfile" (:bucket (meta loaded)))) + (t/is (= "application/zip" (:content-type (meta loaded)))) + (t/is (= profile (:profile-id (meta loaded)))))))) + (t/deftest tempfile-objects-are-not-deduplicated (let [storage (-> (:app.storage/storage th/*system*) (configure-storage-backend)) @@ -94,8 +130,7 @@ (configure-storage-backend)) content (sto/content "content") object (sto/put-object! storage {::sto/content content - :content-type "text/plain" - :expired-at (ct/in-future {:seconds 1})})] + :content-type "text/plain"})] (t/is (sto/object? object)) (t/is (true? (sto/del-object! storage (:id object)))) @@ -109,6 +144,18 @@ ;; marked as deleted/expired. (t/is (nil? (sto/get-object storage (:id object)))))) +(t/deftest delete-expired-object-returns-false + ;; An object stored with `::sto/expired-at` is born with `deleted_at` + ;; set, so deleting it is a no-op and must report `false`. + (let [storage (-> (:app.storage/storage th/*system*) + (configure-storage-backend)) + content (sto/content "content") + object (sto/put-object! storage {::sto/content content + :content-type "text/plain" + ::sto/expired-at (ct/in-future {:hours 1})})] + (t/is (some? (:expired-at object))) + (t/is (false? (sto/del-object! storage (:id object)))))) + (t/deftest deleted-gc-task (let [storage (-> (:app.storage/storage th/*system*) (configure-storage-backend)) @@ -438,6 +485,146 @@ (let [row (th/db-exec-one! ["select deleted_at from storage_object where id = ?" (:id object1)])] (t/is (ct/is-before-or-equal? (:deleted-at row) (ct/plus now {:seconds 1})))))) +(t/deftest storage-gc-touched-null-metadata + ;; A NULL metadata column (predates any normalization) flows through + ;; the lookup fallback with a warning instead of breaking the GC loop. + (let [storage (-> (:app.storage/storage th/*system*) + (configure-storage-backend)) + content (sto/content "content") + object (sto/put-object! storage {::sto/content content + ::sto/touch true + :content-type "text/plain"})] + (th/db-exec! ["update storage_object set metadata = null where id = ?" (:id object)]) + (let [res (th/run-task! :storage-gc-touched {:skip-delay true})] + (t/is (= 0 (:freeze res))) + (t/is (= 1 (:delete res)))))) + +(t/deftest storage-gc-touched-defers-corrupt-metadata + ;; A non-map metadata row neither blocks the chunk nor gets + ;; collected: it is logged and deferred exactly one day, keeping its + ;; metadata intact for a later repair. + (let [now (ct/now) + before (poison-counter-value) + storage (-> (:app.storage/storage th/*system*) + (configure-storage-backend)) + healthy (sto/put-object! storage {::sto/content (sto/content "healthy") + ::sto/touched-at now + :content-type "text/plain"}) + poison (uuid/random)] + (th/db-exec! ["insert into storage_object (id, backend, metadata, touched_at) values (?, 'fs', '[]'::jsonb, ?)" + poison now]) + (binding [ct/*clock* (ct/fixed-clock now)] + (let [res (th/run-task! :storage-gc-touched {:skip-delay true})] + (t/is (= 0 (:freeze res))) + (t/is (= 1 (:delete res))))) + (let [row (th/db-exec-one! ["select metadata::text as metadata, touched_at from storage_object where id = ?" poison])] + (t/is (= "[]" (:metadata row))) + ;; inst-ms: timestamptz keeps micros, the frozen clock has nanos. + (t/is (= (inst-ms (ct/plus now {:days 1})) + (inst-ms (:touched-at row))))) + ;; One poison row observed, one increment. + (t/is (= (inc before) (poison-counter-value))))) + +(t/deftest storage-gc-touched-poison-only + ;; A chunk made only of poison rows still terminates: the rows are + ;; deferred, the next scan returns nothing, and no delete is counted. + (let [now (ct/now) + poison (uuid/random)] + (th/db-exec! ["insert into storage_object (id, backend, metadata, touched_at) values (?, 'fs', '[]'::jsonb, ?)" + poison now]) + (binding [ct/*clock* (ct/fixed-clock now)] + (let [res (th/run-task! :storage-gc-touched {:skip-delay true})] + (t/is (= 0 (:freeze res))) + (t/is (= 0 (:delete res))))) + (let [row (th/db-exec-one! ["select touched_at from storage_object where id = ?" poison])] + (t/is (= (inst-ms (ct/plus now {:days 1})) + (inst-ms (:touched-at row))))))) + +(t/deftest storage-gc-touched-drains-past-a-poison-only-batch + ;; A full batch of poison rows must not stop the run: they are deferred + ;; and the loop keeps draining, so a healthy row queued behind the + ;; LIMIT (later touched_at) still gets collected. + (let [now (ct/now) + storage (-> (:app.storage/storage th/*system*) + (configure-storage-backend)) + earlier (ct/minus now {:seconds 1}) + healthy (sto/put-object! storage {::sto/content (sto/content "healthy") + ::sto/touched-at now + :content-type "text/plain"}) + poisons (repeatedly 10 uuid/random)] + (doseq [id poisons] + (th/db-exec! ["insert into storage_object (id, backend, metadata, touched_at) values (?, 'fs', '[]'::jsonb, ?)" + id earlier])) + (binding [ct/*clock* (ct/fixed-clock now)] + (let [res (th/run-task! :storage-gc-touched {:skip-delay true})] + (t/is (= 0 (:freeze res))) + (t/is (= 1 (:delete res))))) + (doseq [id poisons] + (let [row (th/db-exec-one! ["select touched_at from storage_object where id = ?" id])] + (t/is (= (inst-ms (ct/plus now {:days 1})) + (inst-ms (:touched-at row)))))))) + +(def ^:private migration-0155-fixtures + ;; [id transit-metadata]: production-shaped legacy rows (nil payload + ;; means a NULL column). + [["11111111-1111-1111-1111-111111111111" + "{\"~:reference\":\"~:file-media-object\",\"~:content-type\":\"image/png\",\"~:hash\":\"blake2b:aaa\"}"] + ["22222222-2222-2222-2222-222222222222" + "{\"~:content-type\":\"image/svg+xml\"}"] + ["33333333-3333-3333-3333-333333333333" + "{\"~:bucket\":\"tempfile\",\"~:reference\":\"~:tempfile\",\"~:content-type\":\"application/zip\"}"] + ["44444444-4444-4444-4444-444444444444" + "{\"~:bucket\":\"tempfile\",\"~:content-type\":\"application/zip\",\"~:upload-id\":\"~u86907e95-1cb8-8122-8008-4eb7ba07d89d\",\"~:chunk-index\":3}"] + ["55555555-5555-5555-5555-555555555555" + nil]]) + +(defn- run-migration-0155! + ;; Re-runs the 0155 statements (not migratus: it already applied at + ;; bootstrap) over the fixture rows above. + [] + (let [sql (-> (io/resource "app/migrations/sql/0155-normalize-storage-object-metadata.sql") + (slurp)) + no-comments (->> (.split ^String sql "\n") + (remove #(.startsWith ^String (str/trim %) "--")) + (str/join "\n"))] + (doseq [stmt (->> (.split ^String no-comments ";") + (map str/trim) + (remove str/blank?))] + (th/db-exec! [stmt])))) + +(defn- get-metadata-by-id + [id] + (:metadata (th/db-exec-one! ["select metadata from storage_object where id = ?" + (parse-uuid id)]))) + +(t/deftest storage-migration-0155-normalizes-legacy-rows + (doseq [[id mdata] migration-0155-fixtures] + (th/db-exec! ["insert into storage_object (id, backend, metadata) values (?, 'fs', ?::jsonb)" + (parse-uuid id) mdata])) + (run-migration-0155!) + (let [mdata (fn [id] (stsch/decode-metadata (get-metadata-by-id id)))] + (t/is (= {:bucket "file-media-object" + :content-type "image/png" + :hash "blake2b:aaa"} + (mdata "11111111-1111-1111-1111-111111111111"))) + (t/is (= {:bucket "file-media-object" + :content-type "image/svg+xml"} + (mdata "22222222-2222-2222-2222-222222222222"))) + (t/is (= {:bucket "tempfile" + :content-type "application/zip"} + (mdata "33333333-3333-3333-3333-333333333333"))) + (t/is (= {:bucket "tempfile" + :content-type "application/zip"} + (mdata "44444444-4444-4444-4444-444444444444"))) + (t/is (= {:bucket "file-media-object"} + (mdata "55555555-5555-5555-5555-555555555555")))) + ;; second run changes nothing (idempotent) + (let [raw (fn [] (mapv #(.getValue ^PGobject (get-metadata-by-id %)) + (map first migration-0155-fixtures))) + before (raw)] + (run-migration-0155!) + (t/is (= before (raw))))) + (t/deftest storage-gc-deleted-immediate (let [storage (-> (:app.storage/storage th/*system*) (configure-storage-backend)) @@ -626,6 +813,43 @@ (let [row (th/db-exec-one! ["select count(*) from storage_object"])] (t/is (= 1 (:count row)))))) +(t/deftest dedup-reuses-json-encoded-blob + ;; With the JSON flag on, the dedup lookup must find the row it just + ;; wrote; a "~:"-only lookup used to miss it and duplicate the blob. + (binding [cf/config (assoc cf/config :storage-metadata-as-json true)] + (let [storage (-> (:app.storage/storage th/*system*) + (configure-storage-backend)) + content (-> (sto/content "json-content") + (sto/wrap-with-hash "json-hash")) + params {::sto/content content + ::sto/deduplicate? true + :bucket "file-media-object" + :content-type "text/plain"} + object1 (sto/put-object! storage params) + object2 (sto/put-object! storage params)] + (t/is (= (:id object1) (:id object2))) + (let [row (th/db-exec-one! ["select count(*) from storage_object"])] + (t/is (= 1 (:count row))))))) + +(t/deftest dedup-json-put-finds-transit-row + ;; Both encodings coexist during the transition: a JSON write must + ;; reuse a blob already stored as Transit. + (let [storage (-> (:app.storage/storage th/*system*) + (configure-storage-backend)) + content (-> (sto/content "mixed-content") + (sto/wrap-with-hash "mixed-hash")) + params {::sto/content content + ::sto/deduplicate? true + :bucket "file-media-object" + :content-type "text/plain"} + object1 (binding [cf/config (assoc cf/config :storage-metadata-as-json nil)] + (sto/put-object! storage params)) + object2 (binding [cf/config (assoc cf/config :storage-metadata-as-json true)] + (sto/put-object! storage params))] + (t/is (= (:id object1) (:id object2))) + (let [row (th/db-exec-one! ["select count(*) from storage_object"])] + (t/is (= 1 (:count row)))))) + (t/deftest dedup-repairs-stale-object (let [storage (-> (:app.storage/storage th/*system*) (configure-storage-backend)) From d851f82678cf77c61c269762985bf8910c621803 Mon Sep 17 00:00:00 2001 From: Alonso Torres Date: Thu, 1 Oct 2026 09:21:05 +0200 Subject: [PATCH 6/8] :bug: Fix crashing happening on hot reloads (#12005) --- frontend/src/app/main.cljs | 9 ++++++++- frontend/src/app/render_wasm/api.cljs | 15 +++++++++++---- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/frontend/src/app/main.cljs b/frontend/src/app/main.cljs index b6fbfb3ca5..c7d87ba8ba 100644 --- a/frontend/src/app/main.cljs +++ b/frontend/src/app/main.cljs @@ -54,8 +54,12 @@ (let [el (dom/get-element "app")] (mf/create-root el))) +;; Root component of the last render; a hot reload of `app.main.ui` replaces it +(defonce ^:private rendered-app (atom nil)) + (defn init-ui [] + (reset! rendered-app ui/app) (mf/render! app-root (mf/element ui/app))) (defn- initialize-rasterizer @@ -146,7 +150,10 @@ (init-ui))) (defn ^:dev/after-load after-load + "Rerender the UI after a hot reload. When the reload replaced the root + component, unmount the whole tree first so its cleanups run before + the new tree mounts." [] - (reinit)) + (reinit (not (identical? @rendered-app ui/app)))) (set! (.-stackTraceLimit js/Error) 50) diff --git a/frontend/src/app/render_wasm/api.cljs b/frontend/src/app/render_wasm/api.cljs index 0c5a4c3d0b..78623aa43e 100644 --- a/frontend/src/app/render_wasm/api.cljs +++ b/frontend/src/app/render_wasm/api.cljs @@ -2492,6 +2492,13 @@ :cause cause) nil))))) +;; Stable listener identities, so `removeEventListener` matches across hot reloads +(defonce ^:private webgl-context-lost-listener + (fn [event] (on-webgl-context-lost event))) + +(defonce ^:private webgl-context-restored-listener + (fn [event] (on-webgl-context-restored event))) + (defn init-canvas-context [canvas] (if-not (wasm/module-ready?) @@ -2536,8 +2543,8 @@ ;; Add event listeners for WebGL context lost (set! wasm/canvas canvas) (when can-listen? - (.addEventListener canvas "webglcontextlost" on-webgl-context-lost) - (.addEventListener canvas "webglcontextrestored" on-webgl-context-restored)) + (.addEventListener canvas "webglcontextlost" webgl-context-lost-listener) + (.addEventListener canvas "webglcontextrestored" webgl-context-restored-listener)) (start-canvas-snapshot-listener!) (reset! wasm/context-lost? false) (set! wasm/context-initialized? true) @@ -2571,8 +2578,8 @@ ;; Remove listener before losing/deleting context. (when wasm/canvas - (.removeEventListener wasm/canvas "webglcontextlost" on-webgl-context-lost) - (.removeEventListener wasm/canvas "webglcontextrestored" on-webgl-context-restored)) + (.removeEventListener wasm/canvas "webglcontextlost" webgl-context-lost-listener) + (.removeEventListener wasm/canvas "webglcontextrestored" webgl-context-restored-listener)) (stop-canvas-snapshot-listener!) (when (wasm/module-ready?) From e6983a1e4ebafc3e22b5824294d0bc6e7d3f6d1c Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Thu, 1 Oct 2026 10:12:05 +0200 Subject: [PATCH 7/8] :sparkles: Enforce idle and absolute session expiration (#11654) * :sparkles: Enforce idle and absolute session expiration Sessions now expire on two server-side conditions: an idle window (PENPOT_AUTH_TOKEN_COOKIE_MAX_AGE, default 7d) and an absolute cap from creation (PENPOT_AUTH_TOKEN_COOKIE_MAX_AGE_ABSOLUTE, default 30d, enforced by the token :exp claim). The daily session-gc task deletes rows that exceed either window, so idle sessions can no longer be replayed and active sessions are not deleted at the idle window. Also remove the legacy v1 HTTP sessions: the http_session table and the string-id / :ver 0 token code paths are gone. Any v1 cookie now requires a fresh login. Document the session expiration configuration in the technical guide and add a backend memory describing the token, renewal and GC model. Closes #11646 AI-assisted-by: deepseek-v4.1-flash * :bug: Address session-expiration review findings F1-F4 Fix the unreadable test (a stray paren broke whole-suite discovery), enforce idle expiration on every request in wrap-authz, fail boot fast when the absolute cap sits below the idle window, and align config defaults with the memory rule while fixing its migration number and stale reference. Closes #11646 AI-assisted-by: muse-spark-1.3-contributor --- .../auth-permissions-product-domains.md | 2 +- .serena/memories/backend/core.md | 1 + .../memories/backend/session-expiration.md | 35 ++ .serena/memories/backend/subtleties.md | 6 +- backend/src/app/config.clj | 1 - backend/src/app/http/debug.clj | 2 +- backend/src/app/http/middleware.clj | 7 +- backend/src/app/http/session.clj | 122 ++++--- backend/src/app/migrations.clj | 3 + .../sql/0155-drop-http-session-table.sql | 4 + backend/src/app/srepl/main.clj | 2 +- .../backend_tests/http_middleware_test.clj | 312 +++++++++++++++--- docs/technical-guide/configuration.md | 38 +++ .../developer/subsystems/authentication.md | 45 ++- 14 files changed, 453 insertions(+), 127 deletions(-) create mode 100644 .serena/memories/backend/session-expiration.md create mode 100644 backend/src/app/migrations/sql/0155-drop-http-session-table.sql diff --git a/.serena/memories/backend/auth-permissions-product-domains.md b/.serena/memories/backend/auth-permissions-product-domains.md index 0423a838a6..02e2e83462 100644 --- a/.serena/memories/backend/auth-permissions-product-domains.md +++ b/.serena/memories/backend/auth-permissions-product-domains.md @@ -13,7 +13,7 @@ - Logout may return an OIDC provider redirect URI when the session claims include provider/session data and the provider has a logout URI. - Invitation tokens are verified through token issuers and only accepted when the token member id/email matches the authenticated profile; otherwise login proceeds without consuming the invitation. - When public registration is disabled, invitation-based password registration also requires a matching, non-expired `team_invitation` row; registration revalidates it under a row lock before creating the profile. -- HTTP/session parsing details such as cookie/header precedence, JWT session token versions, and SameSite behavior are in `mem:backend/subtleties`. +- HTTP/session parsing details such as cookie/header precedence and SameSite behavior are in `mem:backend/subtleties`. ## Permission model diff --git a/.serena/memories/backend/core.md b/.serena/memories/backend/core.md index 8f67530e71..71714d12c3 100644 --- a/.serena/memories/backend/core.md +++ b/.serena/memories/backend/core.md @@ -7,6 +7,7 @@ Backend: JVM Clojure; Integrant; PostgreSQL; Redis/Valkey; RPC; HTTP; storage; m - Cross-cutting backend subtleties (RPC, DB, workers, cron, HTTP/sessions, storage, file data): `mem:backend/subtleties` - Storage abstraction, logical buckets, object lifecycle, deduplication, access, and garbage collection: `mem:backend/storage`. - Embedded Ladybug graph experiment, projection, incremental sync, console, and risks: `mem:backend/graph-experiment` +- Session lifetime config, token `:exp`, and idle/absolute GC: `mem:backend/session-expiration`. - Auth flows, permission model, teams, projects, invitations, comments, webhooks, audit: `mem:backend/auth-permissions-product-domains` - Audit-log event collection (RPC wrapper, frontend ingestion), telemetry duality, webhook fan-out, error reporters, Nexus archival and retention: `mem:backend/audit-log` - Services, task-queue/Pub-Sub topology constraints -> `mem:prod-infra/core`. diff --git a/.serena/memories/backend/session-expiration.md b/.serena/memories/backend/session-expiration.md new file mode 100644 index 0000000000..99c0d6acac --- /dev/null +++ b/.serena/memories/backend/session-expiration.md @@ -0,0 +1,35 @@ +# Backend Session Expiration + +## Config + +- `:auth-token-cookie-name` / `PENPOT_AUTH_TOKEN_COOKIE_NAME` (default `auth-token`): cookie name. +- `:auth-token-cookie-max-age` / `PENPOT_AUTH_TOKEN_COOKIE_MAX_AGE` (default 7d): idle window; drives the sliding cookie `Expires` and the GC idle threshold. +- `:auth-token-cookie-max-age-absolute` / `PENPOT_AUTH_TOKEN_COOKIE_MAX_AGE_ABSOLUTE` (default 30d): hard cap from `created-at`; drives the token `:exp` and the GC absolute threshold. +- Durations decode as `` with hour/minute/second units (`168h`, `30m`); day units like `7d` are rejected by the `Duration/parse` string path, use hours (`168h`, `720h`). +- Session lifetime defaults live only as code constants in `session.clj` (`default-cookie-max-age`, `default-cookie-max-age-absolute`); do not duplicate them in the `config.clj` default map. Call sites always pass the constant as the `cf/get` fallback. + +## Token and session model + +- Sessions live only in `http_session_v2`. Legacy v1 support (`http_session`, string ids, `:ver 0` tokens) was removed and the table dropped (`0155-drop-http-session-table`). +- `assign-token` emits header `{:kid 1 :ver 1}` with claims `:sid`, `:iat` (= `modified-at`) and `:exp` = `created-at + absolute-max-age` (omitted only when `created-at` is nil, which neither manager produces today; the branch is defensive). +- `:exp` is anchored to `created-at`, never `modified-at`, so renewal cannot extend the absolute maximum. +- Tokens issued before `:exp` existed carry no `:exp`; they are still bounded by the GC's `created_at` condition and acquire `:exp` on their next renewal (self-healing, no operator action). +- `wrap-authz` resolves the session by `(:sid claims)` only; there is no fallback to reading a session by the raw token string. +- `middleware/wrap-auth` attaches `::http/auth-data` only for `kid=1`/`ver=1` tokens with a configured decoder; anything else stays unauthenticated. +- Renewal fires when `modified-at` is older than 6h (`default-renewal-max-age`, not configurable). It `UPDATE`s the same row (`modified-at` only) and issues a new token that keeps the original `:exp`. +- `read-session` does not check age. Idle expiration is enforced on every request by `wrap-authz` (sessions past the idle window authenticate like missing ones and are never renewed); the GC only deletes the rows afterwards. + +## GC (`::tasks/gc`, cron `session-gc`, daily) + +- Deletes from `http_session_v2` where `modified_at < now - :auth-token-cookie-max-age` **or** `created_at < now - :auth-token-cookie-max-age-absolute`. +- The two thresholds are separate task params (`::tasks/max-age`, `::tasks/max-age-absolute`) built in `ig/expand-key`; both must be durations. +- Do not collapse the two conditions: `modified_at` alone never collects active sessions; `created_at` alone logs out active users at the idle window. + +## Related lifetimes + +- Organization SSO entries in session `:props` (`:sso {org-id exp}`) last 4h, hardcoded in `app.auth.oidc`. +- Access tokens (`app.rpc.commands.access-token`) have a user-chosen `:expires-at` (Never/30/60/90/180d), independent from HTTP sessions. +- Cookie `SameSite`/`secure` depend on `:cors`, `:strict-session-cookies`, `:secure-session-cookies`; `parse-flags` auto-adds `:disable-secure-session-cookies` for non-localhost HTTP `public-uri`. +- Without `PENPOT_SECRET_KEY`, derived subsystem keys change on restart and all sessions/invitations are invalidated. +- Read-only DB pool: sessions fall back to `inmemory-manager`, lost on restart, no GC. +- HTTP middleware and cookie/header precedence details: `mem:backend/subtleties`. diff --git a/.serena/memories/backend/subtleties.md b/.serena/memories/backend/subtleties.md index 354b9291a9..ad6c4321ae 100644 --- a/.serena/memories/backend/subtleties.md +++ b/.serena/memories/backend/subtleties.md @@ -33,10 +33,10 @@ - The backend sets Clojure `*assert*` globally from the `:backend-asserts` feature flag. Assertion-dependent checks can therefore differ by runtime flags. - Request body parsing is mostly POST-oriented and supports Transit JSON plus plain JSON. Plain JSON request keys are kebab-decoded before being merged into `:params`. - Response formatting negotiates with `Accept` or `_fmt=json`. Transit is the default for collection/boolean bodies; JSON encoding has special pointer-map handling. -- Auth prefers the session cookie token before the `Authorization` header. Headers may be `Token` or `Bearer`; JWTs with `kid=1` and `ver=1` are decoded as v1 session tokens, otherwise they are treated as legacy tokens. +- Auth prefers the session cookie token before the `Authorization` header. Headers may be `Token` or `Bearer`. Only `kid=1`/`ver=1` tokens are decoded as session tokens; anything else is left unauthenticated (legacy v1 tokens were removed). - Shared-key auth requires `x-shared-key` as ` ` and stores the lowercased key id on the request. If no shared keys are configured it always rejects. -- Session management uses DB storage unless the DB pool is read-only, then falls back to the in-memory manager. DB sessions support both legacy string ids and v2 UUID session ids. -- Session cookies are renewed when using a legacy string id or when `modified-at` is older than the renewal interval. SameSite is `none` for CORS, otherwise strict/lax based on config. +- Session management uses DB storage unless the DB pool is read-only, then falls back to the in-memory manager. Sessions use only the v2 UUID model (`http_session_v2`); legacy string ids were removed. +- Session cookies are renewed when `modified-at` is older than the 6h renewal interval. SameSite is `none` for CORS, otherwise strict/lax based on config. Session lifetime config and GC: `mem:backend/session-expiration`. ## HTTP server self-metrics diff --git a/backend/src/app/config.clj b/backend/src/app/config.clj index 08c48476b5..a9bc88e7eb 100644 --- a/backend/src/app/config.clj +++ b/backend/src/app/config.clj @@ -61,7 +61,6 @@ :objects-storage-fs-directory "assets" :auth-token-cookie-name "auth-token" - :auth-token-cookie-max-age-absolute (ct/duration {:days 30}) :assets-path "/internal/assets/" :smtp-default-reply-to "Penpot " diff --git a/backend/src/app/http/debug.clj b/backend/src/app/http/debug.clj index 0a4929b2ec..5884e8e4bd 100644 --- a/backend/src/app/http/debug.clj +++ b/backend/src/app/http/debug.clj @@ -675,7 +675,7 @@ (contains? params :block) (do (db/update! conn :profile {:is-blocked true} {:id (:id profile)}) - (db/delete! conn :http-session {:profile-id (:id profile)}) + (session/invalidate-all cfg (:id profile)) {::yres/status 200 ::yres/headers {"content-type" "text/plain"} diff --git a/backend/src/app/http/middleware.clj b/backend/src/app/http/middleware.clj index 6aa8e9d3a2..45f8abecca 100644 --- a/backend/src/app/http/middleware.clj +++ b/backend/src/app/http/middleware.clj @@ -306,16 +306,15 @@ (let [decode-fn (get decoders type)] (if (or (= type :cookie) (= type :bearer)) (let [metadata (tokens/decode-header token)] - ;; NOTE: we only proceed to decode claims on new - ;; cookie tokens. The old cookies dont need to be - ;; decoded because they use the token string as ID + ;; NOTE: only current (kid=1/ver=1) cookie tokens carry + ;; decodable claims. Anything else is left unauthenticated. (if (and (= (:kid metadata) 1) (= (:ver metadata) 1) (some? decode-fn)) (assoc request ::http/auth-data (assoc auth :claims (decode-fn token) :metadata metadata)) - (assoc request ::http/auth-data (assoc auth :metadata {:ver 0})))) + request)) (if decode-fn (assoc request ::http/auth-data (assoc auth :claims (decode-fn token))) diff --git a/backend/src/app/http/session.clj b/backend/src/app/http/session.clj index 288fdc4396..c5909ca432 100644 --- a/backend/src/app/http/session.clj +++ b/backend/src/app/http/session.clj @@ -81,14 +81,8 @@ [pool] (reify ISessionManager (read-session [_ id] - (if (string? id) - ;; Backward compatibility: http_session (v1) has no props column - (let [session (db/exec-one! pool (sql/select :http-session {:id id}))] - (-> session - (assoc :modified-at (:updated-at session)) - (dissoc :updated-at))) - (some-> (db/exec-one! pool (sql/select :http-session-v2 {:id id})) - (decode-session)))) + (some-> (db/exec-one! pool (sql/select :http-session-v2 {:id id})) + (decode-session))) (create-session [_ params] (assert (valid-params? params) "expect valid session params") @@ -103,23 +97,15 @@ (update-session [_ session] (let [modified-at (ct/now)] - (if (string? (:id session)) - (db/insert! pool :http-session-v2 - (-> session - (assoc :id (uuid/next)) - (assoc :created-at modified-at) - (assoc :modified-at modified-at))) - (db/update! pool :http-session-v2 - (cond-> {:modified-at modified-at} - (some? (:props session)) - (assoc :props (db/tjson (:props session)))) - {:id (:id session)} - {::db/return-keys true})))) + (db/update! pool :http-session-v2 + (cond-> {:modified-at modified-at} + (some? (:props session)) + (assoc :props (db/tjson (:props session)))) + {:id (:id session)} + {::db/return-keys true}))) (delete-session [_ id] - (if (string? id) - (db/delete! pool :http-session {:id id} {::db/return-keys false}) - (db/delete! pool :http-session-v2 {:id id} {::db/return-keys false})) + (db/delete! pool :http-session-v2 {:id id} {::db/return-keys false}) nil))) (defn inmemory-manager @@ -256,25 +242,28 @@ (db/exec! pool [sql:clear-organization-sso-sessions organization-key organization-key]))) (defn- renew-session? - [{:keys [id modified-at] :as session}] - (or (string? id) - (and (ct/inst? modified-at) - (let [elapsed (ct/diff modified-at (ct/now))] - (neg? (compare default-renewal-max-age elapsed)))))) + [{:keys [modified-at]}] + (and (ct/inst? modified-at) + (let [elapsed (ct/diff modified-at (ct/now))] + (neg? (compare default-renewal-max-age elapsed))))) + +(defn- session-expired? + [{:keys [modified-at]}] + (and (ct/inst? modified-at) + (let [max-age (cf/get :auth-token-cookie-max-age default-cookie-max-age)] + (pos? (compare (ct/now) (ct/plus modified-at max-age)))))) (defn- wrap-authz [handler {:keys [::manager] :as cfg}] (assert (manager? manager) "expected valid session manager") (fn [request] - (let [{:keys [type token claims metadata]} (get request ::http/auth-data)] + (let [{:keys [type claims]} (get request ::http/auth-data)] (cond (= type :cookie) - (let [session - (case (:ver metadata) - ;; BACKWARD COMPATIBILITY WITH OLD TOKENS - 0 (read-session manager token) - 1 (some->> (:sid claims) (read-session manager)) - nil) + (let [session (some->> (:sid claims) (read-session manager)) + ;; Idle-expired sessions authenticate like missing ones: + ;; no profile, no renewal. The GC deletes the row later. + session (when-not (session-expired? session) session) request (cond-> request @@ -294,11 +283,8 @@ response)) (= type :bearer) - (let [session (case (:ver metadata) - ;; BACKWARD COMPATIBILITY WITH OLD TOKENS - 0 (read-session manager token) - 1 (some->> (:sid claims) (read-session manager)) - nil) + (let [session (some->> (:sid claims) (read-session manager)) + session (when-not (session-expired? session) session) request (cond-> request (some? session) (-> (assoc ::profile-id (:profile-id session)) @@ -317,9 +303,9 @@ (defn- assign-session-cookie [response {token :token modified-at :modified-at}] (let [max-age (cf/get :auth-token-cookie-max-age default-cookie-max-age) - created-at modified-at - renewal (ct/plus created-at default-renewal-max-age) - expires (ct/plus created-at max-age) + renewal-at modified-at + renewal (ct/plus renewal-at default-renewal-max-age) + expires (ct/plus renewal-at max-age) secure? (contains? cf/flags :secure-session-cookies) strict? (contains? cf/flags :strict-session-cookies) cors? (contains? cf/flags :cors) @@ -346,40 +332,46 @@ (defmethod ig/assert-key ::tasks/gc [_ params] (assert (db/pool? (::db/pool params)) "expected valid database pool") - (assert (ct/duration? (::tasks/max-age params)))) + (assert (ct/duration? (::tasks/max-age params))) + (assert (ct/duration? (::tasks/max-age-absolute params))) + ;; NOTE: explicit throw (not `assert`) so the guard fires even when + ;; the `:backend-asserts` flag leaves `*assert*` false. + (when (pos? (compare (::tasks/max-age params) + (::tasks/max-age-absolute params))) + (throw (IllegalArgumentException. + "absolute session max-age must be greater than or equal to idle max-age")))) (defmethod ig/expand-key ::tasks/gc [k v] - (let [max-age (cf/get :auth-token-cookie-max-age default-cookie-max-age)] - {k (merge {::tasks/max-age max-age} (d/without-nils v))})) - -(def ^:private - sql:delete-expired - "DELETE FROM http_session - WHERE updated_at < ?::timestamptz - or (updated_at is null and - created_at < ?::timestamptz)") + (let [max-age (cf/get :auth-token-cookie-max-age default-cookie-max-age) + max-age-absolute (cf/get :auth-token-cookie-max-age-absolute + default-cookie-max-age-absolute)] + {k (merge {::tasks/max-age max-age + ::tasks/max-age-absolute max-age-absolute} + (d/without-nils v))})) (def ^:private sql:delete-expired-v2 "DELETE FROM http_session_v2 - WHERE created_at < ?::timestamptz") + WHERE modified_at < ?::timestamptz + OR created_at < ?::timestamptz") (defn- collect-expired-tasks - [{:keys [::db/conn ::tasks/max-age]}] - (let [threshold (ct/minus (ct/now) max-age) - result-legacy (-> (db/exec-one! conn [sql:delete-expired threshold threshold]) - (db/get-update-count)) - result-v2 (-> (db/exec-one! conn [sql:delete-expired-v2 threshold]) - (db/get-update-count))] + [{:keys [::db/conn ::tasks/max-age ::tasks/max-age-absolute]}] + (let [idle-threshold (ct/minus (ct/now) max-age) + abs-threshold (ct/minus (ct/now) max-age-absolute) + result (-> (db/exec-one! conn [sql:delete-expired-v2 + idle-threshold abs-threshold]) + (db/get-update-count))] (l/dbg :task "gc" :hint "clean http sessions" - :deleted-legacy result-legacy - :deleted-v2 result-v2) - (+ result-legacy result-v2))) + :deleted result) + result)) (defmethod ig/init-key ::tasks/gc - [_ {:keys [::tasks/max-age] :as cfg}] - (l/dbg :hint "initializing session gc task" :max-age max-age) + [_ {:keys [::tasks/max-age ::tasks/max-age-absolute] :as cfg}] + (l/dbg :hint "initializing session gc task" + :max-age max-age + :max-age-absolute max-age-absolute) (fn [_] (db/tx-run! cfg collect-expired-tasks))) diff --git a/backend/src/app/migrations.clj b/backend/src/app/migrations.clj index c458f45150..ff9b9a4c6f 100644 --- a/backend/src/app/migrations.clj +++ b/backend/src/app/migrations.clj @@ -510,6 +510,9 @@ {:name "0155-normalize-storage-object-metadata" :fn (mg/resource "app/migrations/sql/0155-normalize-storage-object-metadata.sql")} + {:name "0155-drop-http-session-table" + :fn (mg/resource "app/migrations/sql/0155-drop-http-session-table.sql")} + {:name "0156-add-storage-object-json-dedup-index" :fn (mg/resource "app/migrations/sql/0156-add-storage-object-json-dedup-index.sql")}]) diff --git a/backend/src/app/migrations/sql/0155-drop-http-session-table.sql b/backend/src/app/migrations/sql/0155-drop-http-session-table.sql new file mode 100644 index 0000000000..09db78c632 --- /dev/null +++ b/backend/src/app/migrations/sql/0155-drop-http-session-table.sql @@ -0,0 +1,4 @@ +-- Legacy v1 HTTP sessions have been removed from the backend; the +-- http_session table is no longer read or written by any code path. + +DROP TABLE http_session; diff --git a/backend/src/app/srepl/main.clj b/backend/src/app/srepl/main.clj index b8d0a353af..7123917900 100644 --- a/backend/src/app/srepl/main.clj +++ b/backend/src/app/srepl/main.clj @@ -125,7 +125,7 @@ {:columns [:id :email]})] (when-not (:is-blocked profile) (db/update! conn :profile {:is-blocked true} {:id (:id profile)}) - (db/delete! conn :http-session {:profile-id (:id profile)}) + (session/invalidate-all system (:id profile)) :blocked)))))) (defn reset-password! diff --git a/backend/test/backend_tests/http_middleware_test.clj b/backend/test/backend_tests/http_middleware_test.clj index e058e2fb53..1651d92e0a 100644 --- a/backend/test/backend_tests/http_middleware_test.clj +++ b/backend/test/backend_tests/http_middleware_test.clj @@ -8,6 +8,8 @@ (:require [app.common.exceptions :as ex] [app.common.time :as ct] + [app.common.uuid :as uuid] + [app.config :as cf] [app.db :as db] [app.http :as-alias http] [app.http.access-token] @@ -21,6 +23,7 @@ [backend-tests.helpers :as th] [clojure.string :as str] [clojure.test :as t] + [integrant.core :as ig] [mockery.core :refer [with-mocks]] [yetti.request :as yreq] [yetti.response :as yres]) @@ -56,12 +59,11 @@ (handler (th/make-dummy-request {})) (t/is (nil? (::http/auth-data @request))) + ;; A bearer token is only attached when it is a current session + ;; token (kid=1/ver=1) and a decoder is configured. Otherwise the + ;; request stays unauthenticated. (handler (th/make-dummy-request {:headers {"authorization" "Bearer aaaa"}})) - - (let [{:keys [token claims] token-type :type} (get @request ::http/auth-data)] - (t/is (= :bearer token-type)) - (t/is (= "aaaa" token)) - (t/is (nil? claims))))) + (t/is (nil? (::http/auth-data @request))))) (t/deftest auth-middleware-3 (let [request (volatile! nil) @@ -73,11 +75,7 @@ (t/is (nil? (::http/auth-data @request))) (handler (th/make-dummy-request {:cookies {"auth-token" "foobar"}})) - - (let [{:keys [token claims] token-type :type} (get @request ::http/auth-data)] - (t/is (= :cookie token-type)) - (t/is (= "foobar" token)) - (t/is (nil? claims))))) + (t/is (nil? (::http/auth-data @request))))) (t/deftest shared-key-auth (let [handler (#'app.http.middleware/wrap-shared-key-auth @@ -230,56 +228,282 @@ (t/is (ct/inst? exp) "exp should be an instant"))) (t/deftest session-token-exp-based-on-created-at - (let [cfg th/*system* - manager (session/inmemory-manager) - profile (th/create-profile* 1) - session (->> (session/create-session manager {:profile-id (:id profile) - :user-agent "user agent"}) - (#'session/assign-token cfg)) - claims (tokens/decode cfg (:token session)) - expected-exp (ct/plus (:created-at session) (ct/duration {:days 30}))] + (let [cfg th/*system* + manager (session/inmemory-manager) + profile (th/create-profile* 1) + session (->> (session/create-session manager {:profile-id (:id profile) + :user-agent "user agent"}) + (#'session/assign-token cfg)) + claims (tokens/decode cfg (:token session)) + expected-exp (ct/plus (:created-at session) (ct/duration {:days 30}))] (t/is (some? (:exp claims)) "session token should contain :exp claim") (t/is (= (inst-ms (:exp claims)) (inst-ms expected-exp)) "exp should equal created-at + 30 days"))) (t/deftest session-token-past-exp-is-rejected - (let [cfg th/*system* - manager (session/inmemory-manager) - profile (th/create-profile* 1) - session (->> (session/create-session manager {:profile-id (:id profile) - :user-agent "user agent"}) - (#'session/assign-token cfg)) - claims (tokens/decode cfg (:token session)) - ;; Manually create a token with exp in the past + (let [cfg th/*system* + manager (session/inmemory-manager) + profile (th/create-profile* 1) + session (->> (session/create-session manager {:profile-id (:id profile) + :user-agent "user agent"}) + (#'session/assign-token cfg)) + claims (tokens/decode cfg (:token session)) past-claims (assoc claims :exp (ct/minus (ct/now) (ct/duration {:days 1}))) - header {:kid 1 :ver 1} - past-token (tokens/generate cfg past-claims header)] + past-token (tokens/generate cfg past-claims {:kid 1 :ver 1})] (t/is (nil? (session/decode-token cfg past-token)) "token with exp in the past should be rejected"))) (t/deftest session-renewal-preserves-original-exp (let [cfg th/*system* - manager (session/inmemory-manager) profile (th/create-profile* 1) + created (ct/minus (ct/now) (ct/duration {:days 1})) + session {:id (uuid/random) + :profile-id (:id profile) + :user-agent "user agent" + :created-at created + :modified-at (ct/minus (ct/now) (ct/duration {:hours 7}))} + manager (reify session/ISessionManager + (read-session [_ _] session) + (create-session [_ _] session) + (update-session [_ s] (assoc s :modified-at (ct/now))) + (delete-session [_ _] nil)) + + old-token (:token (#'session/assign-token cfg session)) + original-exp (:exp (tokens/decode cfg old-token)) + handler (-> (fn [req] req) + (#'session/wrap-authz (assoc th/*system* ::session/manager manager)) + (#'mw/wrap-auth {:bearer (partial session/decode-token cfg) + :cookie (partial session/decode-token cfg)})) + response (handler (th/make-dummy-request {:cookies {"auth-token" old-token}})) + renewed-token (get-in response [::yres/cookies "auth-token" :value]) + renewed-claims (tokens/decode cfg renewed-token)] + (t/is (some? original-exp) "original token should have :exp") + (t/is (not= old-token renewed-token) "renewal should issue a new token string") + (t/is (= (inst-ms original-exp) (inst-ms (:exp renewed-claims))) + "renewed token should preserve the original :exp, not extend it"))) + +(t/deftest session-renewal-preserves-exp-with-db-manager + (let [cfg th/*system* + manager (::session/manager th/*system*) + profile (th/create-profile* 1) + created (session/create-session manager {:profile-id (:id profile) + :user-agent "user agent"}) + _ (th/db-exec-one! ["UPDATE http_session_v2 + SET modified_at = now() - interval '7 hours' + WHERE id = ?" (:id created)]) + stale (session/read-session manager (:id created)) + old-token (:token (#'session/assign-token cfg stale)) + original-exp (:exp (tokens/decode cfg old-token)) + handler (-> (fn [req] req) + (#'session/wrap-authz cfg) + (#'mw/wrap-auth {:bearer (partial session/decode-token cfg) + :cookie (partial session/decode-token cfg)})) + response (handler (th/make-dummy-request {:cookies {"auth-token" old-token}})) + renewed (get-in response [::yres/cookies "auth-token" :value]) + renewed-exp (:exp (tokens/decode cfg renewed)) + expected-exp (ct/plus (:created-at created) (ct/duration {:days 30})) + current (session/read-session manager (:id created))] + (t/is (some? original-exp) "original token should have :exp") + (t/is (some? renewed) "renewal should issue a new cookie token") + (t/is (not= old-token renewed) "renewal should issue a new token string") + (t/is (= (inst-ms original-exp) (inst-ms renewed-exp)) + "renewed token should preserve the original :exp, not extend it") + (t/is (= (inst-ms expected-exp) (inst-ms renewed-exp)) + "renewed :exp should equal created-at + 30 days") + (t/is (some? current) "session row must still exist after renewal") + (t/is (pos? (compare (:modified-at current) (:modified-at stale))) + "persisted modified_at must move forward on renewal"))) + +(t/deftest session-renewal-cookie-expires-diverges-from-token-exp + (let [cfg th/*system* + manager (::session/manager th/*system*) + profile (th/create-profile* 1) + created (session/create-session manager {:profile-id (:id profile) + :user-agent "user agent"}) + _ (th/db-exec-one! ["UPDATE http_session_v2 + SET created_at = now() - interval '29 days', + modified_at = now() - interval '7 hours' + WHERE id = ?" (:id created)]) + stale (session/read-session manager (:id created)) + old-token (:token (#'session/assign-token cfg stale)) + handler (-> (fn [req] req) + (#'session/wrap-authz cfg) + (#'mw/wrap-auth {:bearer (partial session/decode-token cfg) + :cookie (partial session/decode-token cfg)})) + response (handler (th/make-dummy-request {:cookies {"auth-token" old-token}})) + cookie (get-in response [::yres/cookies "auth-token"]) + renewed (:value cookie) + renewed-exp (:exp (tokens/decode cfg renewed)) + expected-exp (ct/plus (:created-at stale) (ct/duration {:days 30})) + close-to? (fn [a b tolerance-ms] + (<= (Math/abs (- (inst-ms a) (inst-ms b))) tolerance-ms))] + (t/is (some? renewed) "renewal should issue a new cookie token") + (t/is (not= old-token renewed) "renewal should issue a new token string") + (t/is (= (inst-ms expected-exp) (inst-ms renewed-exp)) + "renewed :exp should equal created-at + 30 days") + (t/is (close-to? renewed-exp (ct/plus (ct/now) (ct/duration {:days 1})) + (* 10 60 1000)) + "renewed :exp should be ~1 day out (absolute cap is near)") + (t/is (close-to? (:expires cookie) (ct/plus (ct/now) (ct/duration {:days 7})) + (* 10 60 1000)) + "cookie Expires should slide ~7 days out from now") + (t/is (pos? (compare (:expires cookie) renewed-exp)) + "cookie Expires should stay ahead of the token :exp"))) + +(t/deftest legacy-session-token-is-rejected + (let [cfg th/*system* + manager (session/inmemory-manager) handler (-> (fn [req] req) - (#'session/wrap-authz {::session/manager manager}) + (#'session/wrap-authz {::session/manager manager}) (#'mw/wrap-auth {:bearer (partial session/decode-token cfg) :cookie (partial session/decode-token cfg)})) - session (->> (session/create-session manager {:profile-id (:id profile) - :user-agent "user agent"}) - (#'session/assign-token cfg)) - original-exp (:exp (tokens/decode cfg (:token session))) - ;; Force renewal by setting modified-at to 7 hours ago - old-session (assoc session :modified-at (ct/minus (ct/now) (ct/duration {:hours 7}))) - response (handler (th/make-dummy-request {:cookies {"auth-token" (:token old-session)}})) - {:keys [token claims]} (get response ::http/auth-data) - new-exp (:exp claims)] - (t/is (some? original-exp) "original token should have :exp") - (t/is (some? new-exp) "renewed token should have :exp") - (t/is (= (inst-ms original-exp) - (inst-ms new-exp)) - "renewed token should preserve original :exp, not extend it"))) + token (tokens/generate cfg {:sid "legacy-session-id"} {:kid 0 :ver 0}) + response (handler (th/make-dummy-request {:cookies {"auth-token" token}}))] + (t/is (nil? (get response ::http/auth-data)) + "legacy tokens must not be attached as auth data") + (t/is (nil? (::session/profile-id response)) + "legacy tokens must not authenticate"))) + +(t/deftest session-gc-deletes-idle-and-absolute-expired-rows + (let [profile (th/create-profile* 1) + fresh (uuid/random) + idle (uuid/random) + absolute (uuid/random) + valid (uuid/random)] + + (th/db-exec-one! ["INSERT INTO http_session_v2 (id, profile_id, created_at, modified_at) + VALUES (?, ?, now(), now())" + fresh (:id profile)]) + (th/db-exec-one! ["INSERT INTO http_session_v2 (id, profile_id, created_at, modified_at) + VALUES (?, ?, now() - interval '1 day', now() - interval '8 days')" + idle (:id profile)]) + (th/db-exec-one! ["INSERT INTO http_session_v2 (id, profile_id, created_at, modified_at) + VALUES (?, ?, now() - interval '31 days', now())" + absolute (:id profile)]) + (th/db-exec-one! ["INSERT INTO http_session_v2 (id, profile_id, created_at, modified_at) + VALUES (?, ?, now() - interval '1 day', now() - interval '6 days')" + valid (:id profile)]) + + (db/tx-run! th/*system* + (fn [cfg] + (#'session/collect-expired-tasks + (assoc cfg + :app.http.session.tasks/max-age (ct/duration {:days 7}) + :app.http.session.tasks/max-age-absolute (ct/duration {:days 30}))))) + + (let [ids (->> (th/db-exec! ["SELECT id FROM http_session_v2 WHERE profile_id = ?" (:id profile)]) + (map :id) + (set))] + (t/is (contains? ids fresh) "fresh session must be kept") + (t/is (contains? ids valid) "session within both windows must be kept") + (t/is (not (contains? ids idle)) "idle session must be deleted") + (t/is (not (contains? ids absolute)) "session past the absolute cap must be deleted")))) + +(t/deftest session-gc-config-wiring + (let [idle (ct/duration {:days 3}) + absolute (ct/duration {:days 10})] + (with-redefs [cf/get (fn + ([k] (case k + :auth-token-cookie-max-age idle + :auth-token-cookie-max-age-absolute absolute + nil)) + ([k default] (case k + :auth-token-cookie-max-age idle + :auth-token-cookie-max-age-absolute absolute + default)))] + (let [expanded (ig/expand-key :app.http.session.tasks/gc {})] + (t/is (= idle + (get-in expanded [:app.http.session.tasks/gc + :app.http.session.tasks/max-age])) + "task max-age should carry the configured idle window") + (t/is (= absolute + (get-in expanded [:app.http.session.tasks/gc + :app.http.session.tasks/max-age-absolute])) + "task max-age-absolute should carry the configured absolute cap"))) + (with-redefs [cf/get (fn + ([_k] nil) + ([_k default] default))] + (let [expanded (ig/expand-key :app.http.session.tasks/gc {})] + (t/is (= session/default-cookie-max-age + (get-in expanded [:app.http.session.tasks/gc + :app.http.session.tasks/max-age])) + "task max-age should fall back to the default idle window") + (t/is (= session/default-cookie-max-age-absolute + (get-in expanded [:app.http.session.tasks/gc + :app.http.session.tasks/max-age-absolute])) + "task max-age-absolute should fall back to the default absolute cap"))))) + +(t/deftest session-gc-rejects-absolute-below-idle + (let [pool (:app.db/pool th/*system*) + params {:app.db/pool pool + :app.http.session.tasks/max-age (ct/duration {:days 7}) + :app.http.session.tasks/max-age-absolute (ct/duration {:days 30})}] + (t/is (nil? (ig/assert-key :app.http.session.tasks/gc params)) + "absolute cap above the idle window should pass") + (t/is (nil? (ig/assert-key :app.http.session.tasks/gc + (assoc params + :app.http.session.tasks/max-age-absolute + (ct/duration {:days 7})))) + "absolute cap equal to the idle window should pass") + (t/is (thrown? IllegalArgumentException + (ig/assert-key :app.http.session.tasks/gc + (assoc params + :app.http.session.tasks/max-age-absolute + (ct/duration {:days 3})))) + "absolute cap below the idle window should fail fast"))) + +(t/deftest idle-expired-session-is-rejected + (let [cfg th/*system* + profile (th/create-profile* 1) + updated? (atom false) + session {:id (uuid/random) + :profile-id (:id profile) + :user-agent "user agent" + :created-at (ct/now) + :modified-at (ct/minus (ct/now) (ct/duration {:days 8}))} + manager (reify session/ISessionManager + (read-session [_ _] session) + (create-session [_ _] session) + (update-session [_ s] (reset! updated? true) s) + (delete-session [_ _] nil)) + token (:token (#'session/assign-token cfg session)) + handler (-> (fn [req] req) + (#'session/wrap-authz (assoc cfg ::session/manager manager)) + (#'mw/wrap-auth {:bearer (partial session/decode-token cfg) + :cookie (partial session/decode-token cfg)})) + cookie-r (handler (th/make-dummy-request {:cookies {"auth-token" token}})) + bearer-r (handler (th/make-dummy-request {:headers {"authorization" (str "Bearer " token)}}))] + (t/is (nil? (::session/profile-id cookie-r)) + "idle-expired session must not authenticate via cookie") + (t/is (nil? (::session/session cookie-r)) + "idle-expired session must not be attached via cookie") + (t/is (nil? (::session/profile-id bearer-r)) + "idle-expired session must not authenticate via bearer") + (t/is (false? @updated?) + "idle-expired session must not be renewed"))) + +(t/deftest idle-session-within-window-is-accepted + (let [cfg th/*system* + profile (th/create-profile* 1) + session {:id (uuid/random) + :profile-id (:id profile) + :user-agent "user agent" + :created-at (ct/now) + :modified-at (ct/minus (ct/now) (ct/duration {:days 6}))} + manager (reify session/ISessionManager + (read-session [_ _] session) + (create-session [_ _] session) + (update-session [_ s] (assoc s :modified-at (ct/now))) + (delete-session [_ _] nil)) + token (:token (#'session/assign-token cfg session)) + handler (-> (fn [req] req) + (#'session/wrap-authz (assoc cfg ::session/manager manager)) + (#'mw/wrap-auth {:bearer (partial session/decode-token cfg) + :cookie (partial session/decode-token cfg)})) + response (handler (th/make-dummy-request {:cookies {"auth-token" token}}))] + (t/is (= (:id profile) (::session/profile-id response)) + "session within the idle window must authenticate"))) (t/deftest parse-request-illegal-argument-exception ;; clojure.data.json raises IllegalArgumentException (case diff --git a/docs/technical-guide/configuration.md b/docs/technical-guide/configuration.md index 9fd0282761..d730abd8fc 100644 --- a/docs/technical-guide/configuration.md +++ b/docs/technical-guide/configuration.md @@ -643,6 +643,44 @@ And configure it: PENPOT_SECRET_KEY: my-super-secure-key ``` +### Session expiration + +__Since version 2.19.0__ + +User sessions are stored server-side and expire on two independent conditions: an +**idle timeout** and an **absolute maximum lifetime**. Both are backend only. + +```bash +# Backend +# Idle timeout: the session stops working after this much inactivity. +# Default: 168h +PENPOT_AUTH_TOKEN_COOKIE_MAX_AGE: 168h + +# Absolute maximum lifetime from the moment the session was created, +# regardless of activity. Default: 720h +PENPOT_AUTH_TOKEN_COOKIE_MAX_AGE_ABSOLUTE: 720h +``` + +Durations use the `` form with hour, minute or second units, +for example `168h`, `30m` or `90s` (day units like `7d` are not accepted). + +While a user is active the session is automatically renewed every 6 hours (not +configurable). Renewal extends the cookie, but never the absolute maximum. The +backend checks the idle timeout on every request, measured from the last +renewal: once it passes, the session stops working at once, even for a copied +token. A daily task (`session-gc`) then deletes the sessions that have exceeded +either window. Legacy v1 sessions and the old `http_session` table are no +longer used. + +Sessions created before 2.19.0 carry no `:exp` in their token; they are still +removed by the 30-day `created_at` cleanup and acquire `:exp` on their next +renewal. + +The `secure` and `same-site` attributes of the session cookie are controlled by +the `disable-secure-session-cookies`, `strict-session-cookies` and `enable-cors` +flags, and by whether `PENPOT_PUBLIC_URI` is served over HTTPS. See +[Penpot URI](#penpot-uri). + ### Database Penpot only supports PostgreSQL and we highly recommend >=13 version. If you are using official diff --git a/docs/technical-guide/developer/subsystems/authentication.md b/docs/technical-guide/developer/subsystems/authentication.md index da437194ee..b5946696ac 100644 --- a/docs/technical-guide/developer/subsystems/authentication.md +++ b/docs/technical-guide/developer/subsystems/authentication.md @@ -94,14 +94,45 @@ Similarly as the OIDC backend, it checks if the profile exists, and calls ## Sessions -User sessions are created when a user logs in via any one of the backends. A -session token is generated (a JWT token that does not currently contain any data) -and returned to frontend as a cookie. +User sessions are created when a user logs in via any one of the backends. The +backend generates a signed JWT token and returns it to the frontend as an +auth-token cookie. A matching row is stored in +the http_session_v2 table with the profile id +and the session timestamps. -Normally the session is stored in a DB table with the information of the user -profile and the session expiration. But if a frontend connects to the backend in -"read only" mode (for example, to debug something in production with the local -devenv), sessions are stored in memory (may be lost if the backend restarts). +A request is authenticated only when the token verifies, its session row still +exists and the session is within the idle timeout. The token claims carry the session row id (:sid), the last activity instant (:iat) and an absolute expiration (:exp). The server enforces two independent +limits: + +* **Idle timeout:** a session that is not renewed within + PENPOT_AUTH_TOKEN_COOKIE_MAX_AGE (default 7 + days) stops working at once. The backend checks the session's last renewal + time on every request, so the row does not need to be deleted first. +* **Absolute maximum:** a session cannot live longer than + PENPOT_AUTH_TOKEN_COOKIE_MAX_AGE_ABSOLUTE + (default 30 days) from its creation, no matter how much it is renewed. The + :exp claim enforces this even when the + cookie is still present. + +Sessions are automatically renewed every 6 hours of use (not configurable). +Renewal issues a new token but keeps the same session row, so the absolute +maximum is not extended. A daily garbage collector +(session-gc) deletes rows that exceed either +the idle window or the absolute maximum. + +Sessions created before 2.19.0 carry no :exp in their token; they are still removed by +the 30-day created_at cleanup and acquire :exp on their next renewal. + +The normal storage is the database. When the backend uses a read-only database +pool (for example, to debug something in production with the local devenv), +sessions are kept in memory and are lost when the backend restarts. The +organization SSO gate keeps an additional 4-hour entry inside the same session +row, separate from the session token lifetime. ## Team invitations From 423cb411324d8c0de39032b2244e1a3c96c23dba Mon Sep 17 00:00:00 2001 From: Andrey Antukh Date: Thu, 1 Oct 2026 10:58:44 +0200 Subject: [PATCH 8/8] :books: Update changelog --- CHANGES.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index ecec05d735..720cbc0fb9 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,14 @@ # CHANGELOG +## 2.18.1 + +### :bug: Bugs fixed + +- Fix workspace freezing with a stack overflow when opening the context menu on a shape that isn't on the current page [#11882](https://github.com/penpot/penpot/issues/11882) (PR: [#11891](https://github.com/penpot/penpot/pull/11891)) +- Fix workspace showing repeated error toasts when a third-party script injected into the page throws [#11813](https://github.com/penpot/penpot/issues/11813) (PR: [#11816](https://github.com/penpot/penpot/pull/11816)) +- Fix register flow from workspace URL not working [#11753](https://github.com/penpot/penpot/issues/11753) (PR: [#11756](https://github.com/penpot/penpot/pull/11756)) +- Fix empty response when exporting bin using API (by @filipsajdak) [#7649](https://github.com/penpot/penpot/issues/7649) (PR: [#10341](https://github.com/penpot/penpot/pull/10341)) + ## 2.18.0 ### :rocket: Epics and highlights