diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000000..f0c1a8ddff --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,16 @@ +# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file +version: 2 +updates: + # Actions are pinned to a commit SHA with a `# vX.Y.Z` comment; Dependabot + # bumps both. All updates are grouped into a single monthly PR. + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "monthly" + open-pull-requests-limit: 5 + commit-message: + prefix: ":arrow_up:" + groups: + github-actions: + patterns: + - "*" diff --git a/.github/workflows/auto-label.yml b/.github/workflows/auto-label.yml index 491f422d1f..6b32b61c52 100644 --- a/.github/workflows/auto-label.yml +++ b/.github/workflows/auto-label.yml @@ -12,14 +12,14 @@ jobs: steps: - name: Generate GitHub App token id: triage-app-token - uses: actions/create-github-app-token@v1 + uses: actions/create-github-app-token@d72941d797fd3113feb6b93fd0dec494b13a2547 # v1.12.0 with: app-id: ${{ secrets.TRIAGE_APP_ID }} private-key: ${{ secrets.TRIAGE_APP_PRIVATE_KEY }} owner: penpot - name: Process Issue or PR - uses: actions/github-script@v7 + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0 with: github-token: ${{ steps.triage-app-token.outputs.token }} script: | diff --git a/.github/workflows/build-adhoc.yml b/.github/workflows/build-adhoc.yml index abfe01d5a1..f5f9b88854 100644 --- a/.github/workflows/build-adhoc.yml +++ b/.github/workflows/build-adhoc.yml @@ -23,7 +23,12 @@ on: jobs: build-bundle: uses: ./.github/workflows/build-bundle.yml - secrets: inherit + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_REGION: ${{ secrets.AWS_REGION }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }} + S3_BUCKET: ${{ secrets.S3_BUCKET }} with: gh_ref: ${{ inputs.gh_ref }} force: ${{ inputs.force }} @@ -31,7 +36,17 @@ jobs: build-docker: needs: build-bundle uses: ./.github/workflows/build-docker.yml - secrets: inherit + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_REGION: ${{ secrets.AWS_REGION }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + DOCKER_REGISTRY: ${{ secrets.DOCKER_REGISTRY }} + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }} + PUB_DOCKER_PASSWORD: ${{ secrets.PUB_DOCKER_PASSWORD }} + PUB_DOCKER_USERNAME: ${{ secrets.PUB_DOCKER_USERNAME }} + S3_BUCKET: ${{ secrets.S3_BUCKET }} with: gh_ref: ${{ inputs.gh_ref }} # Pin build-docker to the commit build-bundle actually bundled. @@ -40,7 +55,8 @@ jobs: build-docker-admin-console: uses: ./.github/workflows/build-docker-admin-console.yml - secrets: inherit + secrets: + ORG_WORKFLOW_TOKEN: ${{ secrets.ORG_WORKFLOW_TOKEN }} with: gh_ref: ${{ inputs.nitrate_ref || inputs.gh_ref }} force: ${{ inputs.force }} diff --git a/.github/workflows/build-bundle.yml b/.github/workflows/build-bundle.yml index 43ce74c1b6..752ff096f6 100644 --- a/.github/workflows/build-bundle.yml +++ b/.github/workflows/build-bundle.yml @@ -29,6 +29,17 @@ on: sha: description: 'Bundled commit, full 40-char SHA.' value: ${{ jobs.check.outputs.sha }} + secrets: + AWS_ACCESS_KEY_ID: + required: false + AWS_REGION: + required: false + AWS_SECRET_ACCESS_KEY: + required: false + MATTERMOST_WEBHOOK: + required: false + S3_BUCKET: + required: false # Literal group name: under `workflow_call`, `github.workflow` resolves to the # caller's workflow, which put this workflow and the other reusable one called @@ -54,8 +65,9 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: + persist-credentials: false fetch-depth: 0 ref: ${{ inputs.gh_ref }} @@ -121,8 +133,9 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: + persist-credentials: false fetch-depth: 0 ref: ${{ inputs.gh_ref }} diff --git a/.github/workflows/build-develop.yml b/.github/workflows/build-develop.yml index 8f12163965..942f840576 100644 --- a/.github/workflows/build-develop.yml +++ b/.github/workflows/build-develop.yml @@ -21,7 +21,12 @@ concurrency: jobs: build-bundle: uses: ./.github/workflows/build-bundle.yml - secrets: inherit + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_REGION: ${{ secrets.AWS_REGION }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }} + S3_BUCKET: ${{ secrets.S3_BUCKET }} with: gh_ref: "develop" force: ${{ inputs.force || false }} @@ -29,7 +34,17 @@ jobs: build-docker: needs: build-bundle uses: ./.github/workflows/build-docker.yml - secrets: inherit + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_REGION: ${{ secrets.AWS_REGION }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + DOCKER_REGISTRY: ${{ secrets.DOCKER_REGISTRY }} + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }} + PUB_DOCKER_PASSWORD: ${{ secrets.PUB_DOCKER_PASSWORD }} + PUB_DOCKER_USERNAME: ${{ secrets.PUB_DOCKER_USERNAME }} + S3_BUCKET: ${{ secrets.S3_BUCKET }} with: gh_ref: "develop" # Pin build-docker to the commit build-bundle actually bundled. @@ -38,7 +53,8 @@ jobs: build-docker-admin-console: uses: ./.github/workflows/build-docker-admin-console.yml - secrets: inherit + secrets: + ORG_WORKFLOW_TOKEN: ${{ secrets.ORG_WORKFLOW_TOKEN }} with: gh_ref: "develop" force: ${{ inputs.force || false }} diff --git a/.github/workflows/build-docker-devenv.yml b/.github/workflows/build-docker-devenv.yml index 2eab6be51e..39d3c2594f 100644 --- a/.github/workflows/build-docker-devenv.yml +++ b/.github/workflows/build-docker-devenv.yml @@ -15,26 +15,28 @@ jobs: echo "DOCKER_CONFIG=${{ runner.temp }}/.docker-${{ github.run_id }}-${{ github.job }}" >> $GITHUB_ENV - name: Checkout code - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: + persist-credentials: false - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v4 + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 - name: Login to Docker Registry (push destination) - uses: docker/login-action@v4 + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 with: username: ${{ secrets.PUB_DOCKER_USERNAME }} password: ${{ secrets.PUB_DOCKER_PASSWORD }} - name: Login to Docker Hardened Images registry (base image pull) - uses: docker/login-action@v4 + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 with: registry: dhi.io username: ${{ secrets.PUB_DOCKER_USERNAME }} password: ${{ secrets.PUB_DOCKER_PASSWORD }} - name: Build and push DevEnv Docker image - uses: docker/build-push-action@v7 + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 env: DOCKER_IMAGE: 'penpotapp/devenv' with: diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 271042700a..f7348a0998 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -32,6 +32,27 @@ on: type: boolean required: false default: false + secrets: + AWS_ACCESS_KEY_ID: + required: false + AWS_REGION: + required: false + AWS_SECRET_ACCESS_KEY: + required: false + DOCKER_PASSWORD: + required: false + DOCKER_REGISTRY: + required: false + DOCKER_USERNAME: + required: false + MATTERMOST_WEBHOOK: + required: false + PUB_DOCKER_PASSWORD: + required: false + PUB_DOCKER_USERNAME: + required: false + S3_BUCKET: + required: false # Literal group name: under `workflow_call`, `github.workflow` resolves to the # caller's workflow, which put this workflow and the other reusable one called @@ -65,8 +86,9 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: + persist-credentials: false fetch-depth: 0 ref: ${{ inputs.sha != '' && inputs.sha || inputs.gh_ref }} @@ -159,12 +181,13 @@ jobs: echo "DOCKER_CONFIG=${{ runner.temp }}/.docker-${{ github.run_id }}-${{ github.job }}-${{ matrix.image }}" >> $GITHUB_ENV - name: Checkout code - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: + persist-credentials: false ref: ${{ inputs.sha != '' && inputs.sha || inputs.gh_ref }} - name: Login to Docker Registry - uses: docker/login-action@v4 + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 with: registry: ${{ secrets.DOCKER_REGISTRY }} username: ${{ secrets.DOCKER_USERNAME }} @@ -174,7 +197,7 @@ jobs: # images from DockerHub for unregistered users. # https://docs.docker.com/docker-hub/usage/ - name: Login to DockerHub Registry - uses: docker/login-action@v4 + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 with: username: ${{ secrets.PUB_DOCKER_USERNAME }} password: ${{ secrets.PUB_DOCKER_PASSWORD }} @@ -186,7 +209,7 @@ jobs: # reuses the same PUB_DOCKER_* credentials as the DockerHub # login above. - name: Login to Docker Hardened Images registry (base image pull) - uses: docker/login-action@v4 + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 with: registry: dhi.io username: ${{ secrets.PUB_DOCKER_USERNAME }} @@ -218,23 +241,23 @@ jobs: popd - name: Set up QEMU (stable) - uses: docker/setup-qemu-action@v4 + uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4.4.0 with: platforms: linux/amd64,linux/arm64 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v4 + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 - name: Extract metadata (tags, labels) id: meta - uses: docker/metadata-action@v6 + uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0 with: images: ${{ matrix.image }} labels: | bundle_version=${{ needs.prepare.outputs.bundle_version }} - name: Build and push Docker image - uses: docker/build-push-action@v7 + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 with: context: ./docker/images/ file: ./docker/images/Dockerfile.${{ matrix.image }} @@ -262,10 +285,10 @@ jobs: echo "DOCKER_CONFIG=${{ runner.temp }}/.docker-${{ github.run_id }}-${{ github.job }}" >> $GITHUB_ENV - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v4 + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 - name: Login to Docker Registry - uses: docker/login-action@v4 + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 with: registry: ${{ secrets.DOCKER_REGISTRY }} username: ${{ secrets.DOCKER_USERNAME }} @@ -331,10 +354,10 @@ jobs: echo "DOCKER_CONFIG=${{ runner.temp }}/.docker-${{ github.run_id }}-${{ github.job }}" >> $GITHUB_ENV - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v4 + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 - name: Login to Docker Registry - uses: docker/login-action@v4 + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 with: registry: ${{ secrets.DOCKER_REGISTRY }} username: ${{ secrets.DOCKER_USERNAME }} diff --git a/.github/workflows/build-staging.yml b/.github/workflows/build-staging.yml index 249ba39e40..3fb417c801 100644 --- a/.github/workflows/build-staging.yml +++ b/.github/workflows/build-staging.yml @@ -21,7 +21,12 @@ concurrency: jobs: build-bundle: uses: ./.github/workflows/build-bundle.yml - secrets: inherit + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_REGION: ${{ secrets.AWS_REGION }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }} + S3_BUCKET: ${{ secrets.S3_BUCKET }} with: gh_ref: "staging" force: ${{ inputs.force || false }} @@ -29,7 +34,17 @@ jobs: build-docker: needs: build-bundle uses: ./.github/workflows/build-docker.yml - secrets: inherit + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_REGION: ${{ secrets.AWS_REGION }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + DOCKER_REGISTRY: ${{ secrets.DOCKER_REGISTRY }} + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }} + PUB_DOCKER_PASSWORD: ${{ secrets.PUB_DOCKER_PASSWORD }} + PUB_DOCKER_USERNAME: ${{ secrets.PUB_DOCKER_USERNAME }} + S3_BUCKET: ${{ secrets.S3_BUCKET }} with: gh_ref: "staging" # Pin build-docker to the commit build-bundle actually bundled. @@ -38,7 +53,8 @@ jobs: build-docker-admin-console: uses: ./.github/workflows/build-docker-admin-console.yml - secrets: inherit + secrets: + ORG_WORKFLOW_TOKEN: ${{ secrets.ORG_WORKFLOW_TOKEN }} with: gh_ref: "staging" force: ${{ inputs.force || false }} diff --git a/.github/workflows/build-tag.yml b/.github/workflows/build-tag.yml index ed79d398f9..9b0b0fcbb3 100644 --- a/.github/workflows/build-tag.yml +++ b/.github/workflows/build-tag.yml @@ -24,7 +24,12 @@ concurrency: jobs: build-bundle: uses: ./.github/workflows/build-bundle.yml - secrets: inherit + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_REGION: ${{ secrets.AWS_REGION }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }} + S3_BUCKET: ${{ secrets.S3_BUCKET }} with: gh_ref: ${{ github.ref_name }} force: ${{ inputs.force || false }} @@ -32,14 +37,25 @@ jobs: build-docker: needs: build-bundle uses: ./.github/workflows/build-docker.yml - secrets: inherit + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_REGION: ${{ secrets.AWS_REGION }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + DOCKER_REGISTRY: ${{ secrets.DOCKER_REGISTRY }} + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }} + PUB_DOCKER_PASSWORD: ${{ secrets.PUB_DOCKER_PASSWORD }} + PUB_DOCKER_USERNAME: ${{ secrets.PUB_DOCKER_USERNAME }} + S3_BUCKET: ${{ secrets.S3_BUCKET }} with: gh_ref: ${{ github.ref_name }} force: ${{ inputs.force || false }} build-docker-admin-console: uses: ./.github/workflows/build-docker-admin-console.yml - secrets: inherit + secrets: + ORG_WORKFLOW_TOKEN: ${{ secrets.ORG_WORKFLOW_TOKEN }} with: gh_ref: ${{ github.ref_name }} force: ${{ inputs.force || false }} @@ -67,6 +83,13 @@ jobs: - build-docker - build-docker-admin-console uses: ./.github/workflows/release.yml - secrets: inherit + secrets: + AWS_REGION: ${{ secrets.AWS_REGION }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + DOCKER_REGISTRY: ${{ secrets.DOCKER_REGISTRY }} + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }} + PUB_DOCKER_PASSWORD: ${{ secrets.PUB_DOCKER_PASSWORD }} + PUB_DOCKER_USERNAME: ${{ secrets.PUB_DOCKER_USERNAME }} with: gh_ref: ${{ github.ref_name }} diff --git a/.github/workflows/build-tmp-tokens.yml b/.github/workflows/build-tmp-tokens.yml index e9e4183bad..68eeae1706 100644 --- a/.github/workflows/build-tmp-tokens.yml +++ b/.github/workflows/build-tmp-tokens.yml @@ -12,13 +12,28 @@ concurrency: jobs: build-bundle: uses: ./.github/workflows/build-bundle.yml - secrets: inherit + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_REGION: ${{ secrets.AWS_REGION }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }} + S3_BUCKET: ${{ secrets.S3_BUCKET }} with: gh_ref: "hiru-tokens-in-libs" build-docker: needs: build-bundle uses: ./.github/workflows/build-docker.yml - secrets: inherit + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_REGION: ${{ secrets.AWS_REGION }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + DOCKER_REGISTRY: ${{ secrets.DOCKER_REGISTRY }} + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }} + PUB_DOCKER_PASSWORD: ${{ secrets.PUB_DOCKER_PASSWORD }} + PUB_DOCKER_USERNAME: ${{ secrets.PUB_DOCKER_USERNAME }} + S3_BUCKET: ${{ secrets.S3_BUCKET }} with: gh_ref: "hiru-tokens-in-libs" diff --git a/.github/workflows/commit-checker.yml b/.github/workflows/commit-checker.yml index a80e6e4cc0..c876d10160 100644 --- a/.github/workflows/commit-checker.yml +++ b/.github/workflows/commit-checker.yml @@ -27,7 +27,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check Commit Type - uses: gsactions/commit-message-checker@v2 + uses: gsactions/commit-message-checker@16fa2d5de096ae0d35626443bcd24f1e756cafee # v2.0.0 with: pattern: '^(((:(lipstick|globe_with_meridians|wrench|books|arrow_up|arrow_down|zap|ambulance|construction|boom|fire|whale|bug|sparkles|paperclip|tada|recycle|rewind|construction_worker):)\s[A-Z].*[^.])|(Merge|Revert|Reapply).+[^.])$' flags: 'gm' diff --git a/.github/workflows/plugins-deploy-api-doc.yml b/.github/workflows/plugins-deploy-api-doc.yml index dc74f61fdd..a3598af4bd 100644 --- a/.github/workflows/plugins-deploy-api-doc.yml +++ b/.github/workflows/plugins-deploy-api-doc.yml @@ -39,14 +39,15 @@ jobs: echo "gh_ref=$GH_REF" >> $GITHUB_OUTPUT - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: + persist-credentials: false fetch-depth: 0 ref: ${{ steps.vars.outputs.gh_ref }} # START: Setup Node and PNPM enabling cache - name: Setup pnpm + Node.js - uses: pnpm/setup@v2 + uses: pnpm/setup@84cb39b217b10273981911c288cd62326dc7c6d2 # v2.0.2 with: working-directory: plugins runtime: node@24.21.0 @@ -59,7 +60,7 @@ jobs: run: echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_OUTPUT - name: Cache pnpm store - uses: actions/cache@v5 + uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: ${{ steps.pnpm-store.outputs.STORE_PATH }} key: ${{ runner.os }}-pnpm-${{ hashFiles('plugins/pnpm-lock.yaml') }} diff --git a/.github/workflows/plugins-deploy-package.yml b/.github/workflows/plugins-deploy-package.yml index eec741a009..7c83df9f5d 100644 --- a/.github/workflows/plugins-deploy-package.yml +++ b/.github/workflows/plugins-deploy-package.yml @@ -27,6 +27,13 @@ on: description: 'Publig name (from plugins/apps/-plugin)' type: string required: true + secrets: + CLOUDFLARE_ACCOUNT_ID: + required: false + CLOUDFLARE_API_TOKEN: + required: false + MATTERMOST_WEBHOOK: + required: false permissions: contents: read @@ -58,8 +65,9 @@ jobs: fi - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: + persist-credentials: false fetch-depth: 0 ref: ${{ inputs.gh_ref }} diff --git a/.github/workflows/plugins-deploy-packages.yml b/.github/workflows/plugins-deploy-packages.yml index 943e4b790d..218620983f 100644 --- a/.github/workflows/plugins-deploy-packages.yml +++ b/.github/workflows/plugins-deploy-packages.yml @@ -36,9 +36,11 @@ jobs: # [For new plugins] # Add more outputs here steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: + persist-credentials: false - id: filter - uses: dorny/paths-filter@v4 + uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4.0.3 with: filters: | colors_to_tokens: @@ -72,7 +74,10 @@ jobs: needs: detect-changes if: github.event_name == 'workflow_dispatch' || needs.detect-changes.outputs.colors_to_tokens == 'true' uses: ./.github/workflows/plugins-deploy-package.yml - secrets: inherit + secrets: + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }} with: gh_ref: "${{ inputs.gh_ref || github.ref_name }}" plugin_name: colors-to-tokens @@ -81,7 +86,10 @@ jobs: needs: detect-changes if: github.event_name == 'workflow_dispatch' || needs.detect-changes.outputs.contrast == 'true' uses: ./.github/workflows/plugins-deploy-package.yml - secrets: inherit + secrets: + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }} with: gh_ref: "${{ inputs.gh_ref || github.ref_name }}" plugin_name: contrast @@ -90,7 +98,10 @@ jobs: needs: detect-changes if: github.event_name == 'workflow_dispatch' || needs.detect-changes.outputs.create_palette == 'true' uses: ./.github/workflows/plugins-deploy-package.yml - secrets: inherit + secrets: + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }} with: gh_ref: "${{ inputs.gh_ref || github.ref_name }}" plugin_name: create-palette @@ -99,7 +110,10 @@ jobs: needs: detect-changes if: github.event_name == 'workflow_dispatch' || needs.detect-changes.outputs.icons == 'true' uses: ./.github/workflows/plugins-deploy-package.yml - secrets: inherit + secrets: + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }} with: gh_ref: "${{ inputs.gh_ref || github.ref_name }}" plugin_name: icons @@ -108,7 +122,10 @@ jobs: needs: detect-changes if: github.event_name == 'workflow_dispatch' || needs.detect-changes.outputs.lorem_ipsum == 'true' uses: ./.github/workflows/plugins-deploy-package.yml - secrets: inherit + secrets: + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }} with: gh_ref: "${{ inputs.gh_ref || github.ref_name }}" plugin_name: lorem-ipsum @@ -117,7 +134,10 @@ jobs: needs: detect-changes if: github.event_name == 'workflow_dispatch' || needs.detect-changes.outputs.rename_layers == 'true' uses: ./.github/workflows/plugins-deploy-package.yml - secrets: inherit + secrets: + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }} with: gh_ref: "${{ inputs.gh_ref || github.ref_name }}" plugin_name: rename-layers @@ -126,7 +146,10 @@ jobs: needs: detect-changes if: github.event_name == 'workflow_dispatch' || needs.detect-changes.outputs.table == 'true' uses: ./.github/workflows/plugins-deploy-package.yml - secrets: inherit + secrets: + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }} with: gh_ref: "${{ inputs.gh_ref || github.ref_name }}" plugin_name: table diff --git a/.github/workflows/plugins-deploy-styles-doc.yml b/.github/workflows/plugins-deploy-styles-doc.yml index 2d356962bf..3c04eb6b58 100644 --- a/.github/workflows/plugins-deploy-styles-doc.yml +++ b/.github/workflows/plugins-deploy-styles-doc.yml @@ -37,14 +37,15 @@ jobs: echo "gh_ref=$GH_REF" >> $GITHUB_OUTPUT - name: Checkout - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: + persist-credentials: false fetch-depth: 0 ref: ${{ steps.vars.outputs.gh_ref }} # START: Setup Node and PNPM enabling cache - name: Setup pnpm + Node.js - uses: pnpm/setup@v2 + uses: pnpm/setup@84cb39b217b10273981911c288cd62326dc7c6d2 # v2.0.2 with: working-directory: plugins runtime: node@24.21.0 @@ -57,7 +58,7 @@ jobs: run: echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_OUTPUT - name: Cache pnpm store - uses: actions/cache@v5 + uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: ${{ steps.pnpm-store.outputs.STORE_PATH }} key: ${{ runner.os }}-pnpm-${{ hashFiles('plugins/pnpm-lock.yaml') }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7468f8d4d9..e956f2310d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,6 +13,21 @@ on: description: 'Tag to release' type: string required: true + secrets: + AWS_REGION: + required: false + DOCKER_PASSWORD: + required: false + DOCKER_REGISTRY: + required: false + DOCKER_USERNAME: + required: false + MATTERMOST_WEBHOOK: + required: false + PUB_DOCKER_PASSWORD: + required: false + PUB_DOCKER_USERNAME: + required: false permissions: contents: write @@ -32,14 +47,15 @@ jobs: echo "gh_ref=$GH_REF" >> $GITHUB_OUTPUT - name: Checkout code - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: + persist-credentials: false fetch-depth: 0 ref: ${{ steps.vars.outputs.gh_ref }} # --- Publicly release the docker images --- - name: Configure ECR credentials - uses: aws-actions/configure-aws-credentials@v4 + uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1 with: aws-access-key-id: ${{ secrets.DOCKER_USERNAME }} aws-secret-access-key: ${{ secrets.DOCKER_PASSWORD }} @@ -94,14 +110,19 @@ jobs: echo "EOF" >> $GITHUB_OUTPUT # --- Create GitHub release --- + # Updates the release if it already exists, so re-running is safe. - name: Create GitHub release - uses: softprops/action-gh-release@v2 env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - tag_name: ${{ steps.vars.outputs.gh_ref }} - name: ${{ steps.vars.outputs.gh_ref }} - body: ${{ steps.extract_release_notes.outputs.release_notes }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + TAG: ${{ steps.vars.outputs.gh_ref }} + NOTES: ${{ steps.extract_release_notes.outputs.release_notes }} + run: | + if gh release view "$TAG" > /dev/null 2>&1; then + gh release edit "$TAG" --title "$TAG" --notes "$NOTES" + else + gh release create "$TAG" --verify-tag --title "$TAG" --notes "$NOTES" + fi - name: Notify Mattermost if: failure() diff --git a/.github/workflows/tests-backend.yml b/.github/workflows/tests-backend.yml index 02f839c062..1557a8685a 100644 --- a/.github/workflows/tests-backend.yml +++ b/.github/workflows/tests-backend.yml @@ -60,7 +60,9 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: + persist-credentials: false - name: Fmt working-directory: ./backend diff --git a/.github/workflows/tests-common.yml b/.github/workflows/tests-common.yml index e513efc54f..b622958277 100644 --- a/.github/workflows/tests-common.yml +++ b/.github/workflows/tests-common.yml @@ -39,7 +39,9 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: + persist-credentials: false - name: Lint working-directory: ./common diff --git a/.github/workflows/tests-e2e.yml b/.github/workflows/tests-e2e.yml index ccbde28970..07e1cc47a9 100644 --- a/.github/workflows/tests-e2e.yml +++ b/.github/workflows/tests-e2e.yml @@ -107,8 +107,9 @@ jobs: # An empty `ref` makes checkout fall back to its default (the PR merge # ref on pull_request, the pushed ref on push). - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: + persist-credentials: false ref: ${{ inputs.gh_ref }} # The cache key must come from the SHA actually checked out: on a manual @@ -120,7 +121,7 @@ jobs: - name: Restore Bundle Cache id: restore - uses: actions/cache/restore@v5 + uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: key: ${{ steps.vars.outputs.bundle_key }} path: frontend/resources/public @@ -133,7 +134,7 @@ jobs: - name: Store Bundle Cache if: steps.restore.outputs.cache-hit != 'true' - uses: actions/cache/save@v5 + uses: actions/cache/save@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: key: ${{ steps.vars.outputs.bundle_key }} path: frontend/resources/public @@ -163,12 +164,13 @@ jobs: steps: - name: Checkout Repository - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: + persist-credentials: false ref: ${{ inputs.gh_ref }} - name: Restore Cache - uses: actions/cache/restore@v5 + uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: key: ${{ needs.build-bundle.outputs.bundle_key }} path: frontend/resources/public @@ -201,7 +203,7 @@ jobs: --reporter=blob - name: Upload blob report - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 if: always() with: name: integration-blob-report-${{ matrix.shard }} @@ -210,7 +212,7 @@ jobs: retention-days: 3 - name: Upload test result - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 if: always() with: name: integration-tests-result-${{ matrix.shard }} @@ -234,11 +236,13 @@ jobs: - /var/cache/github-runner/gitlib:/root/.gitlibs steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: + persist-credentials: false # The driver serves the prebuilt bundle from frontend/resources/public. - name: Restore Cache - uses: actions/cache/restore@v5 + uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: key: ${{ needs.build-bundle.outputs.bundle_key }} path: frontend/resources/public @@ -272,11 +276,13 @@ jobs: - /var/cache/github-runner/gitlib:/root/.gitlibs steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: + persist-credentials: false # Mocked mode serves the prebuilt bundle from frontend/resources/public. - name: Restore Cache - uses: actions/cache/restore@v5 + uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: key: ${{ needs.build-bundle.outputs.bundle_key }} path: frontend/resources/public @@ -318,8 +324,9 @@ jobs: steps: - name: Checkout Repository - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: + persist-credentials: false ref: ${{ inputs.gh_ref }} - name: Install deps @@ -328,7 +335,7 @@ jobs: pnpm install --frozen-lockfile; - name: Download blob reports - uses: actions/download-artifact@v7 + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: path: frontend/all-blob-reports pattern: integration-blob-report-* @@ -357,7 +364,7 @@ jobs: >> "$GITHUB_STEP_SUMMARY" - name: Upload JSON report - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 if: always() with: name: integration-json-report @@ -367,7 +374,7 @@ jobs: retention-days: 7 - name: Upload HTML report - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 if: always() with: name: integration-html-report diff --git a/.github/workflows/tests-exporter.yml b/.github/workflows/tests-exporter.yml index aefbfc1dbc..8441af3a75 100644 --- a/.github/workflows/tests-exporter.yml +++ b/.github/workflows/tests-exporter.yml @@ -43,7 +43,9 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: + persist-credentials: false - name: Lint working-directory: ./exporter diff --git a/.github/workflows/tests-frontend.yml b/.github/workflows/tests-frontend.yml index f08f34054d..8fcfdc8066 100644 --- a/.github/workflows/tests-frontend.yml +++ b/.github/workflows/tests-frontend.yml @@ -43,7 +43,9 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: + persist-credentials: false - name: Lint working-directory: ./frontend diff --git a/.github/workflows/tests-library.yml b/.github/workflows/tests-library.yml index 84ab5c9dc9..df2c478ca1 100644 --- a/.github/workflows/tests-library.yml +++ b/.github/workflows/tests-library.yml @@ -41,7 +41,9 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: + persist-credentials: false - name: Lint working-directory: ./library diff --git a/.github/workflows/tests-mcp.yml b/.github/workflows/tests-mcp.yml index f51c56b8a3..68ff0584bd 100644 --- a/.github/workflows/tests-mcp.yml +++ b/.github/workflows/tests-mcp.yml @@ -33,7 +33,9 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: + persist-credentials: false - name: Setup working-directory: ./mcp diff --git a/.github/workflows/tests-plugins.yml b/.github/workflows/tests-plugins.yml index 9f6c461da2..180f125489 100644 --- a/.github/workflows/tests-plugins.yml +++ b/.github/workflows/tests-plugins.yml @@ -38,7 +38,9 @@ jobs: - /var/cache/github-runner/gitlib:/root/.gitlibs steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: + persist-credentials: false - name: Install deps working-directory: ./plugins diff --git a/.github/workflows/tests-wasm.yml b/.github/workflows/tests-wasm.yml index 9f26175554..fdbc8820ff 100644 --- a/.github/workflows/tests-wasm.yml +++ b/.github/workflows/tests-wasm.yml @@ -39,7 +39,9 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: + persist-credentials: false - name: Format working-directory: ./render-wasm diff --git a/.github/zizmor.yml b/.github/zizmor.yml index bdaa8777e5..d4681e9f17 100644 --- a/.github/zizmor.yml +++ b/.github/zizmor.yml @@ -12,26 +12,6 @@ rules: - plugins-deploy-api-doc.yml - plugins-deploy-package.yml - plugins-deploy-styles-doc.yml - # Use `persist-credentials: false` in actions/checkout. - artipacked: - ignore: - - build-bundle.yml - - build-docker-devenv.yml - - build-docker.yml - - plugins-deploy-api-doc.yml - - plugins-deploy-package.yml - - plugins-deploy-packages.yml - - plugins-deploy-styles-doc.yml - - release.yml - - tests-backend.yml - - tests-common.yml - - tests-e2e.yml - - tests-exporter.yml - - tests-frontend.yml - - tests-library.yml - - tests-mcp.yml - - tests-plugins.yml - - tests-wasm.yml # Avoid restoring caches in release/publish workflows. cache-poisoning: ignore: @@ -70,22 +50,9 @@ rules: github-app: ignore: - auto-label.yml - # Pass only the secrets each reusable workflow needs. - secrets-inherit: - ignore: - - build-adhoc.yml - - build-develop.yml - - build-staging.yml - - build-tag.yml - - build-tmp-tokens.yml - - plugins-deploy-packages.yml # Style nudge towards the `$/...` syntax; not worth enforcing. self-repository: disable: true - # Replace actions that duplicate built-in runner tools. - superfluous-actions: - ignore: - - release.yml # Pin container images to a digest. unpinned-images: ignore: @@ -99,25 +66,3 @@ rules: - tests-mcp.yml - tests-plugins.yml - tests-wasm.yml - # Pin actions to a full commit SHA. - unpinned-uses: - ignore: - - auto-label.yml - - build-bundle.yml - - build-docker-devenv.yml - - build-docker.yml - - commit-checker.yml - - plugins-deploy-api-doc.yml - - plugins-deploy-package.yml - - plugins-deploy-packages.yml - - plugins-deploy-styles-doc.yml - - release.yml - - tests-backend.yml - - tests-common.yml - - tests-e2e.yml - - tests-exporter.yml - - tests-frontend.yml - - tests-library.yml - - tests-mcp.yml - - tests-plugins.yml - - tests-wasm.yml diff --git a/.serena/memories/backend/auth-permissions-product-domains.md b/.serena/memories/backend/auth-permissions-product-domains.md index 0423a838a6..02e2e83462 100644 --- a/.serena/memories/backend/auth-permissions-product-domains.md +++ b/.serena/memories/backend/auth-permissions-product-domains.md @@ -13,7 +13,7 @@ - Logout may return an OIDC provider redirect URI when the session claims include provider/session data and the provider has a logout URI. - Invitation tokens are verified through token issuers and only accepted when the token member id/email matches the authenticated profile; otherwise login proceeds without consuming the invitation. - When public registration is disabled, invitation-based password registration also requires a matching, non-expired `team_invitation` row; registration revalidates it under a row lock before creating the profile. -- HTTP/session parsing details such as cookie/header precedence, JWT session token versions, and SameSite behavior are in `mem:backend/subtleties`. +- HTTP/session parsing details such as cookie/header precedence and SameSite behavior are in `mem:backend/subtleties`. ## Permission model diff --git a/.serena/memories/backend/core.md b/.serena/memories/backend/core.md index 8f67530e71..71714d12c3 100644 --- a/.serena/memories/backend/core.md +++ b/.serena/memories/backend/core.md @@ -7,6 +7,7 @@ Backend: JVM Clojure; Integrant; PostgreSQL; Redis/Valkey; RPC; HTTP; storage; m - Cross-cutting backend subtleties (RPC, DB, workers, cron, HTTP/sessions, storage, file data): `mem:backend/subtleties` - Storage abstraction, logical buckets, object lifecycle, deduplication, access, and garbage collection: `mem:backend/storage`. - Embedded Ladybug graph experiment, projection, incremental sync, console, and risks: `mem:backend/graph-experiment` +- Session lifetime config, token `:exp`, and idle/absolute GC: `mem:backend/session-expiration`. - Auth flows, permission model, teams, projects, invitations, comments, webhooks, audit: `mem:backend/auth-permissions-product-domains` - Audit-log event collection (RPC wrapper, frontend ingestion), telemetry duality, webhook fan-out, error reporters, Nexus archival and retention: `mem:backend/audit-log` - Services, task-queue/Pub-Sub topology constraints -> `mem:prod-infra/core`. diff --git a/.serena/memories/backend/session-expiration.md b/.serena/memories/backend/session-expiration.md new file mode 100644 index 0000000000..99c0d6acac --- /dev/null +++ b/.serena/memories/backend/session-expiration.md @@ -0,0 +1,35 @@ +# Backend Session Expiration + +## Config + +- `:auth-token-cookie-name` / `PENPOT_AUTH_TOKEN_COOKIE_NAME` (default `auth-token`): cookie name. +- `:auth-token-cookie-max-age` / `PENPOT_AUTH_TOKEN_COOKIE_MAX_AGE` (default 7d): idle window; drives the sliding cookie `Expires` and the GC idle threshold. +- `:auth-token-cookie-max-age-absolute` / `PENPOT_AUTH_TOKEN_COOKIE_MAX_AGE_ABSOLUTE` (default 30d): hard cap from `created-at`; drives the token `:exp` and the GC absolute threshold. +- Durations decode as `` with hour/minute/second units (`168h`, `30m`); day units like `7d` are rejected by the `Duration/parse` string path, use hours (`168h`, `720h`). +- Session lifetime defaults live only as code constants in `session.clj` (`default-cookie-max-age`, `default-cookie-max-age-absolute`); do not duplicate them in the `config.clj` default map. Call sites always pass the constant as the `cf/get` fallback. + +## Token and session model + +- Sessions live only in `http_session_v2`. Legacy v1 support (`http_session`, string ids, `:ver 0` tokens) was removed and the table dropped (`0155-drop-http-session-table`). +- `assign-token` emits header `{:kid 1 :ver 1}` with claims `:sid`, `:iat` (= `modified-at`) and `:exp` = `created-at + absolute-max-age` (omitted only when `created-at` is nil, which neither manager produces today; the branch is defensive). +- `:exp` is anchored to `created-at`, never `modified-at`, so renewal cannot extend the absolute maximum. +- Tokens issued before `:exp` existed carry no `:exp`; they are still bounded by the GC's `created_at` condition and acquire `:exp` on their next renewal (self-healing, no operator action). +- `wrap-authz` resolves the session by `(:sid claims)` only; there is no fallback to reading a session by the raw token string. +- `middleware/wrap-auth` attaches `::http/auth-data` only for `kid=1`/`ver=1` tokens with a configured decoder; anything else stays unauthenticated. +- Renewal fires when `modified-at` is older than 6h (`default-renewal-max-age`, not configurable). It `UPDATE`s the same row (`modified-at` only) and issues a new token that keeps the original `:exp`. +- `read-session` does not check age. Idle expiration is enforced on every request by `wrap-authz` (sessions past the idle window authenticate like missing ones and are never renewed); the GC only deletes the rows afterwards. + +## GC (`::tasks/gc`, cron `session-gc`, daily) + +- Deletes from `http_session_v2` where `modified_at < now - :auth-token-cookie-max-age` **or** `created_at < now - :auth-token-cookie-max-age-absolute`. +- The two thresholds are separate task params (`::tasks/max-age`, `::tasks/max-age-absolute`) built in `ig/expand-key`; both must be durations. +- Do not collapse the two conditions: `modified_at` alone never collects active sessions; `created_at` alone logs out active users at the idle window. + +## Related lifetimes + +- Organization SSO entries in session `:props` (`:sso {org-id exp}`) last 4h, hardcoded in `app.auth.oidc`. +- Access tokens (`app.rpc.commands.access-token`) have a user-chosen `:expires-at` (Never/30/60/90/180d), independent from HTTP sessions. +- Cookie `SameSite`/`secure` depend on `:cors`, `:strict-session-cookies`, `:secure-session-cookies`; `parse-flags` auto-adds `:disable-secure-session-cookies` for non-localhost HTTP `public-uri`. +- Without `PENPOT_SECRET_KEY`, derived subsystem keys change on restart and all sessions/invitations are invalidated. +- Read-only DB pool: sessions fall back to `inmemory-manager`, lost on restart, no GC. +- HTTP middleware and cookie/header precedence details: `mem:backend/subtleties`. diff --git a/.serena/memories/backend/storage.md b/.serena/memories/backend/storage.md index 43c925eeec..4a48cd8c71 100644 --- a/.serena/memories/backend/storage.md +++ b/.serena/memories/backend/storage.md @@ -54,8 +54,7 @@ ### Key Warning (from function notes): -The improved note in `import-storage-objects` and `handle-persistence` warns: -**Do not reuse the main database connection for storage operations within a transaction.** The storage upload process can fail mid-operation, leaving orphaned objects on the backend. If the outer transaction aborts, pending storage objects become unreconciliable because the storage subsystem registers its pending state in separate transactions. +The improved note in `import-storage-objects` and `handle-persistence` warns: **Do not reuse the main database connection for storage operations within a transaction.** The storage upload process can fail mid-operation, leaving orphaned objects on the backend. If the outer transaction aborts, pending storage objects become unreconciliable because the storage subsystem registers its pending state in separate transactions. ### Rule of Thumb for `sto/put-object!`: @@ -66,10 +65,7 @@ Since `put-object!` uses backend-specific operations (`impl/resolve-backend` + ` - Deduplication requires `::sto/deduplicate?`, a content hash, and bucket metadata. - The lookup matches hash, bucket, backend, and `deleted_at IS NULL`. - The lookup only considers rows with `status='valid'`; pending rows are invisible. -- A hit whose blob is missing is repaired in place: the same row/id is kept, - and `put-object!` rewrites the blob under that id. This heals all existing - references to the object. If the rewrite fails, the row is left live and - valid for a later retry. +- A hit whose blob is missing is repaired in place: the same row/id is kept, and `put-object!` rewrites the blob under that id. This heals all existing references to the object. If the rewrite fails, the row is left live and valid for a later retry. - The lookup does not include file ID, profile ID, team ID, or organization ID. - Objects can therefore share content across users and files within one bucket. - Deleted objects are not reused. @@ -130,6 +126,8 @@ Since `put-object!` uses backend-specific operations (`impl/resolve-backend` + ` - Logical storage operations (`app.storage`, `::mtx/metrics` required by the schema): - `penpot_storage_operations_total{op,bucket,backend}` — `put`, `repair`, `get-data`, `get-bytes`, `del`, `touch`, `exists`. All ops are success-only: `put`/`repair` emit after the backend write, `get-*` after the backend fetch opens, `touch`/`del` only when a row actually changed. `touch-object!`/`del-object!` take the object id (UUID) only — no object overload. Labels come from the updated row itself via `UPDATE ... RETURNING id, backend, metadata` (no extra `SELECT`); with no row matched they emit nothing. `del-object!` only matches live rows (`deleted_at IS NULL`): a repeated del returns `false` and emits nothing. Post-open stream read errors stay counted as attempts. `del` only marks `deleted_at`; physical deletion is a GC concern. `exists` is emitted per deduplication-hit probe, always paired with a `hit`/`repair` outcome (never on probe failure), not per user-facing existence check. - `penpot_storage_dedup_total{result,bucket}` — `hit`, `miss`, `repair`, `skip`. +- Storage GC poison (`app.storage.gc-touched`, `::mtx/metrics` required in the handler cfg): + - `penpot_storage_gc_poison_total` — touched rows whose metadata fails to decode. Each one is logged at error level and deferred exactly one day instead of aborting the chunk. No labels; incremented once per row per scan. - Asset serving (`app.http.assets`, `::mtx/metrics` required in the handler cfg): - `penpot_storage_asset_requests_total{route,backend,bucket,result}` — `route` is `by-id`, `by-file-media-id`, or `thumbnail`; `result` is `served` (<400), `not-found` (404), `unauthorized` (401/403), or `error` (everything else, including a nil/non-number status: every serve path must set `::yres/status`). Serve-path exceptions are counted by `serve-object-measured` and then rethrown. Permission-denied file-media requests and tempfile ownership mismatches both answer HTTP 404 (to avoid leaking existence) but are counted as `unauthorized`. Malformed UUIDs raise before any emission point and are never counted. Counts backend requests that trigger a browser GET to the object store (one per cache miss), so it is a proxy for object GETs, not an exact count. - The physical and logical counters intentionally overlap in coverage but differ in meaning; do not sum them. diff --git a/.serena/memories/backend/subtleties.md b/.serena/memories/backend/subtleties.md index 354b9291a9..ad6c4321ae 100644 --- a/.serena/memories/backend/subtleties.md +++ b/.serena/memories/backend/subtleties.md @@ -33,10 +33,10 @@ - The backend sets Clojure `*assert*` globally from the `:backend-asserts` feature flag. Assertion-dependent checks can therefore differ by runtime flags. - Request body parsing is mostly POST-oriented and supports Transit JSON plus plain JSON. Plain JSON request keys are kebab-decoded before being merged into `:params`. - Response formatting negotiates with `Accept` or `_fmt=json`. Transit is the default for collection/boolean bodies; JSON encoding has special pointer-map handling. -- Auth prefers the session cookie token before the `Authorization` header. Headers may be `Token` or `Bearer`; JWTs with `kid=1` and `ver=1` are decoded as v1 session tokens, otherwise they are treated as legacy tokens. +- Auth prefers the session cookie token before the `Authorization` header. Headers may be `Token` or `Bearer`. Only `kid=1`/`ver=1` tokens are decoded as session tokens; anything else is left unauthenticated (legacy v1 tokens were removed). - Shared-key auth requires `x-shared-key` as ` ` and stores the lowercased key id on the request. If no shared keys are configured it always rejects. -- Session management uses DB storage unless the DB pool is read-only, then falls back to the in-memory manager. DB sessions support both legacy string ids and v2 UUID session ids. -- Session cookies are renewed when using a legacy string id or when `modified-at` is older than the renewal interval. SameSite is `none` for CORS, otherwise strict/lax based on config. +- Session management uses DB storage unless the DB pool is read-only, then falls back to the in-memory manager. Sessions use only the v2 UUID model (`http_session_v2`); legacy string ids were removed. +- Session cookies are renewed when `modified-at` is older than the 6h renewal interval. SameSite is `none` for CORS, otherwise strict/lax based on config. Session lifetime config and GC: `mem:backend/session-expiration`. ## HTTP server self-metrics diff --git a/CHANGES.md b/CHANGES.md index 327d5c8dcb..75a9c866b4 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -47,6 +47,15 @@ - Export multiple fills to SVG [#11466](https://github.com/penpot/penpot/issues/11466) (PR: [#11467](https://github.com/penpot/penpot/pull/11467)) - Add Penpot-specific board size presets (file thumbnail, template cover, plugin icon/cover) [#11561](https://github.com/penpot/penpot/issues/11561) (PR: [#11565](https://github.com/penpot/penpot/pull/11565)) +## 2.18.1 + +### :bug: Bugs fixed + +- Fix workspace freezing with a stack overflow when opening the context menu on a shape that isn't on the current page [#11882](https://github.com/penpot/penpot/issues/11882) (PR: [#11891](https://github.com/penpot/penpot/pull/11891)) +- Fix workspace showing repeated error toasts when a third-party script injected into the page throws [#11813](https://github.com/penpot/penpot/issues/11813) (PR: [#11816](https://github.com/penpot/penpot/pull/11816)) +- Fix register flow from workspace URL not working [#11753](https://github.com/penpot/penpot/issues/11753) (PR: [#11756](https://github.com/penpot/penpot/pull/11756)) +- Fix empty response when exporting bin using API (by @filipsajdak) [#7649](https://github.com/penpot/penpot/issues/7649) (PR: [#10341](https://github.com/penpot/penpot/pull/10341)) + ## 2.18.0 ### :rocket: Epics and highlights diff --git a/backend/src/app/config.clj b/backend/src/app/config.clj index 48081d6430..a9bc88e7eb 100644 --- a/backend/src/app/config.clj +++ b/backend/src/app/config.clj @@ -61,7 +61,6 @@ :objects-storage-fs-directory "assets" :auth-token-cookie-name "auth-token" - :auth-token-cookie-max-age-absolute (ct/duration {:days 30}) :assets-path "/internal/assets/" :smtp-default-reply-to "Penpot " @@ -304,6 +303,10 @@ [:objects-storage-s3-region {:optional true} :keyword] [:objects-storage-s3-endpoint {:optional true} ::sm/uri] + ;; Write storage_object.metadata as plain JSON instead of + ;; Transit-JSON. Unset by default (Phase 1: keep writing Transit). + [:storage-metadata-as-json {:optional true} ::sm/boolean] + ;; SSRF protection [:ssrf-allowed-hosts {:optional true} [::sm/set :string]] [:ssrf-extra-blocked-cidrs {:optional true} [::sm/set :string]]])) diff --git a/backend/src/app/http/debug.clj b/backend/src/app/http/debug.clj index 0a4929b2ec..5884e8e4bd 100644 --- a/backend/src/app/http/debug.clj +++ b/backend/src/app/http/debug.clj @@ -675,7 +675,7 @@ (contains? params :block) (do (db/update! conn :profile {:is-blocked true} {:id (:id profile)}) - (db/delete! conn :http-session {:profile-id (:id profile)}) + (session/invalidate-all cfg (:id profile)) {::yres/status 200 ::yres/headers {"content-type" "text/plain"} diff --git a/backend/src/app/http/middleware.clj b/backend/src/app/http/middleware.clj index 6aa8e9d3a2..45f8abecca 100644 --- a/backend/src/app/http/middleware.clj +++ b/backend/src/app/http/middleware.clj @@ -306,16 +306,15 @@ (let [decode-fn (get decoders type)] (if (or (= type :cookie) (= type :bearer)) (let [metadata (tokens/decode-header token)] - ;; NOTE: we only proceed to decode claims on new - ;; cookie tokens. The old cookies dont need to be - ;; decoded because they use the token string as ID + ;; NOTE: only current (kid=1/ver=1) cookie tokens carry + ;; decodable claims. Anything else is left unauthenticated. (if (and (= (:kid metadata) 1) (= (:ver metadata) 1) (some? decode-fn)) (assoc request ::http/auth-data (assoc auth :claims (decode-fn token) :metadata metadata)) - (assoc request ::http/auth-data (assoc auth :metadata {:ver 0})))) + request)) (if decode-fn (assoc request ::http/auth-data (assoc auth :claims (decode-fn token))) diff --git a/backend/src/app/http/session.clj b/backend/src/app/http/session.clj index 288fdc4396..c5909ca432 100644 --- a/backend/src/app/http/session.clj +++ b/backend/src/app/http/session.clj @@ -81,14 +81,8 @@ [pool] (reify ISessionManager (read-session [_ id] - (if (string? id) - ;; Backward compatibility: http_session (v1) has no props column - (let [session (db/exec-one! pool (sql/select :http-session {:id id}))] - (-> session - (assoc :modified-at (:updated-at session)) - (dissoc :updated-at))) - (some-> (db/exec-one! pool (sql/select :http-session-v2 {:id id})) - (decode-session)))) + (some-> (db/exec-one! pool (sql/select :http-session-v2 {:id id})) + (decode-session))) (create-session [_ params] (assert (valid-params? params) "expect valid session params") @@ -103,23 +97,15 @@ (update-session [_ session] (let [modified-at (ct/now)] - (if (string? (:id session)) - (db/insert! pool :http-session-v2 - (-> session - (assoc :id (uuid/next)) - (assoc :created-at modified-at) - (assoc :modified-at modified-at))) - (db/update! pool :http-session-v2 - (cond-> {:modified-at modified-at} - (some? (:props session)) - (assoc :props (db/tjson (:props session)))) - {:id (:id session)} - {::db/return-keys true})))) + (db/update! pool :http-session-v2 + (cond-> {:modified-at modified-at} + (some? (:props session)) + (assoc :props (db/tjson (:props session)))) + {:id (:id session)} + {::db/return-keys true}))) (delete-session [_ id] - (if (string? id) - (db/delete! pool :http-session {:id id} {::db/return-keys false}) - (db/delete! pool :http-session-v2 {:id id} {::db/return-keys false})) + (db/delete! pool :http-session-v2 {:id id} {::db/return-keys false}) nil))) (defn inmemory-manager @@ -256,25 +242,28 @@ (db/exec! pool [sql:clear-organization-sso-sessions organization-key organization-key]))) (defn- renew-session? - [{:keys [id modified-at] :as session}] - (or (string? id) - (and (ct/inst? modified-at) - (let [elapsed (ct/diff modified-at (ct/now))] - (neg? (compare default-renewal-max-age elapsed)))))) + [{:keys [modified-at]}] + (and (ct/inst? modified-at) + (let [elapsed (ct/diff modified-at (ct/now))] + (neg? (compare default-renewal-max-age elapsed))))) + +(defn- session-expired? + [{:keys [modified-at]}] + (and (ct/inst? modified-at) + (let [max-age (cf/get :auth-token-cookie-max-age default-cookie-max-age)] + (pos? (compare (ct/now) (ct/plus modified-at max-age)))))) (defn- wrap-authz [handler {:keys [::manager] :as cfg}] (assert (manager? manager) "expected valid session manager") (fn [request] - (let [{:keys [type token claims metadata]} (get request ::http/auth-data)] + (let [{:keys [type claims]} (get request ::http/auth-data)] (cond (= type :cookie) - (let [session - (case (:ver metadata) - ;; BACKWARD COMPATIBILITY WITH OLD TOKENS - 0 (read-session manager token) - 1 (some->> (:sid claims) (read-session manager)) - nil) + (let [session (some->> (:sid claims) (read-session manager)) + ;; Idle-expired sessions authenticate like missing ones: + ;; no profile, no renewal. The GC deletes the row later. + session (when-not (session-expired? session) session) request (cond-> request @@ -294,11 +283,8 @@ response)) (= type :bearer) - (let [session (case (:ver metadata) - ;; BACKWARD COMPATIBILITY WITH OLD TOKENS - 0 (read-session manager token) - 1 (some->> (:sid claims) (read-session manager)) - nil) + (let [session (some->> (:sid claims) (read-session manager)) + session (when-not (session-expired? session) session) request (cond-> request (some? session) (-> (assoc ::profile-id (:profile-id session)) @@ -317,9 +303,9 @@ (defn- assign-session-cookie [response {token :token modified-at :modified-at}] (let [max-age (cf/get :auth-token-cookie-max-age default-cookie-max-age) - created-at modified-at - renewal (ct/plus created-at default-renewal-max-age) - expires (ct/plus created-at max-age) + renewal-at modified-at + renewal (ct/plus renewal-at default-renewal-max-age) + expires (ct/plus renewal-at max-age) secure? (contains? cf/flags :secure-session-cookies) strict? (contains? cf/flags :strict-session-cookies) cors? (contains? cf/flags :cors) @@ -346,40 +332,46 @@ (defmethod ig/assert-key ::tasks/gc [_ params] (assert (db/pool? (::db/pool params)) "expected valid database pool") - (assert (ct/duration? (::tasks/max-age params)))) + (assert (ct/duration? (::tasks/max-age params))) + (assert (ct/duration? (::tasks/max-age-absolute params))) + ;; NOTE: explicit throw (not `assert`) so the guard fires even when + ;; the `:backend-asserts` flag leaves `*assert*` false. + (when (pos? (compare (::tasks/max-age params) + (::tasks/max-age-absolute params))) + (throw (IllegalArgumentException. + "absolute session max-age must be greater than or equal to idle max-age")))) (defmethod ig/expand-key ::tasks/gc [k v] - (let [max-age (cf/get :auth-token-cookie-max-age default-cookie-max-age)] - {k (merge {::tasks/max-age max-age} (d/without-nils v))})) - -(def ^:private - sql:delete-expired - "DELETE FROM http_session - WHERE updated_at < ?::timestamptz - or (updated_at is null and - created_at < ?::timestamptz)") + (let [max-age (cf/get :auth-token-cookie-max-age default-cookie-max-age) + max-age-absolute (cf/get :auth-token-cookie-max-age-absolute + default-cookie-max-age-absolute)] + {k (merge {::tasks/max-age max-age + ::tasks/max-age-absolute max-age-absolute} + (d/without-nils v))})) (def ^:private sql:delete-expired-v2 "DELETE FROM http_session_v2 - WHERE created_at < ?::timestamptz") + WHERE modified_at < ?::timestamptz + OR created_at < ?::timestamptz") (defn- collect-expired-tasks - [{:keys [::db/conn ::tasks/max-age]}] - (let [threshold (ct/minus (ct/now) max-age) - result-legacy (-> (db/exec-one! conn [sql:delete-expired threshold threshold]) - (db/get-update-count)) - result-v2 (-> (db/exec-one! conn [sql:delete-expired-v2 threshold]) - (db/get-update-count))] + [{:keys [::db/conn ::tasks/max-age ::tasks/max-age-absolute]}] + (let [idle-threshold (ct/minus (ct/now) max-age) + abs-threshold (ct/minus (ct/now) max-age-absolute) + result (-> (db/exec-one! conn [sql:delete-expired-v2 + idle-threshold abs-threshold]) + (db/get-update-count))] (l/dbg :task "gc" :hint "clean http sessions" - :deleted-legacy result-legacy - :deleted-v2 result-v2) - (+ result-legacy result-v2))) + :deleted result) + result)) (defmethod ig/init-key ::tasks/gc - [_ {:keys [::tasks/max-age] :as cfg}] - (l/dbg :hint "initializing session gc task" :max-age max-age) + [_ {:keys [::tasks/max-age ::tasks/max-age-absolute] :as cfg}] + (l/dbg :hint "initializing session gc task" + :max-age max-age + :max-age-absolute max-age-absolute) (fn [_] (db/tx-run! cfg collect-expired-tasks))) diff --git a/backend/src/app/main.clj b/backend/src/app/main.clj index 4133cc3a8b..de72b9a0ef 100644 --- a/backend/src/app/main.clj +++ b/backend/src/app/main.clj @@ -180,6 +180,12 @@ ::mdef/labels ["route" "backend" "bucket" "result"] ::mdef/type :counter} + :storage-gc-poison + {::mdef/name "penpot_storage_gc_poison_total" + ::mdef/help "Storage objects deferred by storage-gc-touched because their metadata is corrupt." + ::mdef/labels [] + ::mdef/type :counter} + :http-server-dispatch-timing {::mdef/name "penpot_http_server_dispatch_timing" ::mdef/help "Histogram of dispatch handler" @@ -271,7 +277,8 @@ ::sto/storage (ig/ref ::sto/storage)} ::sto.gc-touched/handler - {::db/pool (ig/ref ::db/pool)} + {::db/pool (ig/ref ::db/pool) + ::mtx/metrics (ig/ref ::mtx/metrics)} ::sto.pending-gc/handler {::db/pool (ig/ref ::db/pool) diff --git a/backend/src/app/migrations.clj b/backend/src/app/migrations.clj index fad1f91168..ff9b9a4c6f 100644 --- a/backend/src/app/migrations.clj +++ b/backend/src/app/migrations.clj @@ -505,7 +505,16 @@ :fn (mg/resource "app/migrations/sql/0153-add-storage-object-status-and-deletion-attempts.sql")} {:name "0154-add-upload-session-chunk-table" - :fn (mg/resource "app/migrations/sql/0154-add-upload-session-chunk-table.sql")}]) + :fn (mg/resource "app/migrations/sql/0154-add-upload-session-chunk-table.sql")} + + {:name "0155-normalize-storage-object-metadata" + :fn (mg/resource "app/migrations/sql/0155-normalize-storage-object-metadata.sql")} + + {:name "0155-drop-http-session-table" + :fn (mg/resource "app/migrations/sql/0155-drop-http-session-table.sql")} + + {:name "0156-add-storage-object-json-dedup-index" + :fn (mg/resource "app/migrations/sql/0156-add-storage-object-json-dedup-index.sql")}]) (defn apply-migrations! [pool name migrations] diff --git a/backend/src/app/migrations/sql/0155-drop-http-session-table.sql b/backend/src/app/migrations/sql/0155-drop-http-session-table.sql new file mode 100644 index 0000000000..09db78c632 --- /dev/null +++ b/backend/src/app/migrations/sql/0155-drop-http-session-table.sql @@ -0,0 +1,4 @@ +-- Legacy v1 HTTP sessions have been removed from the backend; the +-- http_session table is no longer read or written by any code path. + +DROP TABLE http_session; diff --git a/backend/src/app/migrations/sql/0155-normalize-storage-object-metadata.sql b/backend/src/app/migrations/sql/0155-normalize-storage-object-metadata.sql new file mode 100644 index 0000000000..36ee9b2c97 --- /dev/null +++ b/backend/src/app/migrations/sql/0155-normalize-storage-object-metadata.sql @@ -0,0 +1,52 @@ +--- Normalize storage_object.metadata inside Transit (Phase 1). +--- +--- Brings every row to a canonical shape WITHOUT changing the encoding +--- family (still Transit JSON, "~:" keys), so old and new code keep +--- reading while it runs. Each statement below documents its own rule. +--- +--- NOTE: key-existence is tested with `-> ... IS [NOT] NULL` instead of +--- the `?` operator because the migration runner prepares statements +--- and a bare `?` is parsed as a bind placeholder. +--- +--- Idempotent: re-running changes zero rows. +--- NOTE for large instances (e.g. our production, ~33M rows): DO NOT run +--- this inline. Mark the migration as executed (fake) and run the batched +--- equivalent by hand: .agents/plans/2026-09-11-storage-normalize-phase1.sql + +-- Drop leftovers from the old chunked-upload metadata: no code reads +-- "~:upload-id" / "~:chunk-index" anymore (#11651 moved chunks to +-- upload_session_chunk and objects-gc purges the orphan rows), and the +-- Phase 1 readers already ignore them on decode. +UPDATE storage_object + SET metadata = metadata - '~:upload-id' - '~:chunk-index' + WHERE (metadata -> '~:upload-id') IS NOT NULL + OR (metadata -> '~:chunk-index') IS NOT NULL; + +-- Rows that carry only the legacy "~:reference": promote it to "~:bucket" +-- with the same value. The value comes as a Transit keyword ("~:xxx"), so +-- the "~:" prefix is stripped; plain strings are kept as they are. +UPDATE storage_object + SET metadata = (metadata - '~:reference') + || jsonb_build_object('~:bucket', + CASE WHEN metadata->>'~:reference' LIKE '~:%' + THEN substr(metadata->>'~:reference', 3) + ELSE metadata->>'~:reference' + END) + WHERE (metadata -> '~:reference') IS NOT NULL + AND (metadata -> '~:bucket') IS NULL; + +-- Rows that carry both keys: "~:bucket" wins and the residual +-- "~:reference" is dropped. +UPDATE storage_object + SET metadata = metadata - '~:reference' + WHERE (metadata -> '~:reference') IS NOT NULL + AND (metadata -> '~:bucket') IS NOT NULL; + +-- Rows with neither key get the historical default bucket +-- "file-media-object" (the value `lookup-bucket` used to fall back to). +-- NULL columns match this rule too (`->` on NULL is NULL) and are +-- backfilled by coalesce. +UPDATE storage_object + SET metadata = coalesce(metadata, '{}') || '{"~:bucket": "file-media-object"}' + WHERE (metadata -> '~:bucket') IS NULL + AND (metadata -> '~:reference') IS NULL; diff --git a/backend/src/app/migrations/sql/0156-add-storage-object-json-dedup-index.sql b/backend/src/app/migrations/sql/0156-add-storage-object-json-dedup-index.sql new file mode 100644 index 0000000000..0576318f50 --- /dev/null +++ b/backend/src/app/migrations/sql/0156-add-storage-object-json-dedup-index.sql @@ -0,0 +1,11 @@ +-- Dedup lookup index for the plain-JSON encoding of +-- storage_object.metadata ("hash" / "bucket"). +-- +-- The dedup lookup matches both metadata encodings while Transit rows +-- still exist (a UNION ALL of two indexable branches). The legacy "~:" +-- index (storage_object__hash_backend_bucket__idx, migration 0068) keeps +-- covering the Transit branch until Transit support is removed. + +CREATE INDEX storage_object__hash_backend_bucket_json__idx + ON storage_object ((metadata->>'hash'), (metadata->>'bucket'), backend) + WHERE deleted_at IS NULL AND status = 'valid'; diff --git a/backend/src/app/srepl/main.clj b/backend/src/app/srepl/main.clj index b8d0a353af..7123917900 100644 --- a/backend/src/app/srepl/main.clj +++ b/backend/src/app/srepl/main.clj @@ -125,7 +125,7 @@ {:columns [:id :email]})] (when-not (:is-blocked profile) (db/update! conn :profile {:is-blocked true} {:id (:id profile)}) - (db/delete! conn :http-session {:profile-id (:id profile)}) + (session/invalidate-all system (:id profile)) :blocked)))))) (defn reset-password! diff --git a/backend/src/app/storage.clj b/backend/src/app/storage.clj index 52163bfc95..25c2d3482a 100644 --- a/backend/src/app/storage.clj +++ b/backend/src/app/storage.clj @@ -20,6 +20,7 @@ [app.storage.fs :as sfs] [app.storage.impl :as impl] [app.storage.s3 :as ss3] + [app.storage.schema :as stsch] [cuerdas.core :as str] [datoteka.fs :as fs] [integrant.core :as ig]) @@ -37,28 +38,18 @@ nil))) (def default-bucket - "file-media-object") + stsch/default-bucket) (def tempfile-bucket "Bucket name for temporary file uploads (10-minute expiry)." - "tempfile") + stsch/tempfile-bucket) (def upload-session-bucket "Bucket name for chunked-upload chunks." - "upload-session") + stsch/upload-session-bucket) (def valid-buckets - #{"file-media-object" - "team-font-variant" - "file-object-thumbnail" - "file-thumbnail" - "profile" - "organization" - tempfile-bucket - upload-session-bucket - "file-data" - "file-data-fragment" - "file-change"}) + stsch/metadata-buckets) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Storage Module State @@ -118,18 +109,37 @@ params params)) +;; Matches both metadata encodings while Transit rows still exist. The +;; UNION ALL keeps each branch indexable: migration 0068 covers the legacy +;; "~:" keys and 0156 covers the plain JSON keys. Phase 3 drops the legacy +;; branch once no "~:" rows remain. +(def ^:private sql:get-database-object-by-hash + "( + select * from storage_object + where (metadata->>'~:hash') = ? + and (metadata->>'~:bucket') = ? + and backend = ? + and deleted_at is null + and status = 'valid' + limit 1 + ) union all ( + select * from storage_object + where (metadata->>'hash') = ? + and (metadata->>'bucket') = ? + and backend = ? + and deleted_at is null + and status = 'valid' + limit 1 + ) limit 1") + +;; NOTE: metadata is left encoded; row->storage-object is responsible for +;; decoding it. (defn- get-database-object-by-hash [connectable backend bucket hash] - (let [sql (str "select * from storage_object " - " where (metadata->>'~:hash') = ? " - " and (metadata->>'~:bucket') = ? " - " and backend = ?" - " and deleted_at is null" - " and status = 'valid'" - " limit 1")] - ;; NOTE: metadata is left encoded; row->storage-object is - ;; responsible for decoding it. - (db/exec-one! connectable [sql hash bucket (name backend)]))) + (let [backend (name backend)] + (db/exec-one! connectable [sql:get-database-object-by-hash + hash bucket backend + hash bucket backend]))) (defn- promote-object! [storage object] @@ -147,7 +157,7 @@ res)) (defn row->storage-object [res] - (let [mdata (or (some-> (:metadata res) (db/decode-transit-pgobject)) {})] + (let [mdata (or (some-> (:metadata res) (stsch/decode-metadata)) {})] (impl/storage-object (:id res) (:size res) @@ -289,7 +299,7 @@ {:id id :size (impl/get-size content) :backend (name backend) - :metadata (db/tjson mdata) + :metadata (stsch/encode-metadata mdata) :deleted-at expired-at :touched-at touched-at :status "pending"}) diff --git a/backend/src/app/storage/gc_deleted.clj b/backend/src/app/storage/gc_deleted.clj index d9dc908e0a..7048bd4d2e 100644 --- a/backend/src/app/storage/gc_deleted.clj +++ b/backend/src/app/storage/gc_deleted.clj @@ -51,7 +51,7 @@ "DELETE FROM storage_object WHERE id = ANY(?::uuid[])") -(defn- delete-sobjects! +(defn- delete-sobjects [conn ids] (let [ids (db/create-array conn "uuid" ids)] (-> (db/exec-one! conn [sql:delete-sobjects ids]) @@ -61,7 +61,7 @@ "DELETE FROM upload_session_chunk WHERE object_id = ANY(?::uuid[])") -(defn- delete-upload-session-chunks! +(defn- delete-upload-session-chunks "Remove the chunk mappings for the given storage object ids. This must run before the storage_object rows are deleted: the upload_session_chunk foreign keys are ON DELETE NO ACTION." @@ -75,7 +75,7 @@ deleted_at = NOW() + INTERVAL '1 day' WHERE id = ANY(?::uuid[])") -(defn- increment-attempts-and-defer! +(defn- increment-attempts-and-defer [conn ids] (let [ids (db/create-array conn "uuid" ids)] (db/exec-one! conn [sql:increment-attempts-and-defer ids]))) @@ -85,7 +85,7 @@ WHERE id = ANY(?::uuid[]) AND deletion_attempts >= ?") -(defn- delete-give-up! +(defn- delete-give-up [conn ids] (let [ids (db/create-array conn "uuid" ids)] (db/exec-one! conn [sql:delete-give-up ids max-attempts]))) @@ -93,7 +93,7 @@ (defn- process-chunk "Attempt to delete a chunk of storage objects from a specific backend. - This function runs inside the caller's transaction (clean-deleted!) — + This function runs inside the caller's transaction (clean-deleted) — it does NOT open its own transaction. The caller is responsible for ensuring the rows are locked via FOR UPDATE SKIP LOCKED before calling. @@ -121,18 +121,18 @@ ;; storage_object rows (NO ACTION foreign keys). It only affects ;; objects of the upload-session bucket; for any other bucket the ;; delete matches no rows. - (delete-upload-session-chunks! conn ok-ids) - (delete-sobjects! conn ok-ids)) + (delete-upload-session-chunks conn ok-ids) + (delete-sobjects conn ok-ids)) (when (seq fail-ids) - (increment-attempts-and-defer! conn fail-ids) + (increment-attempts-and-defer conn fail-ids) ;; NOTE: same NO ACTION ordering as above: the give-up DELETE below ;; removes storage_object rows, so chunk mappings must go first. ;; Deferred objects keep their rows; only the mapping of a ;; permanently given-up object disappears early, and that object is ;; already deleted-marked. - (delete-upload-session-chunks! conn fail-ids) - (let [given-up (delete-give-up! conn fail-ids)] + (delete-upload-session-chunks conn fail-ids) + (let [given-up (delete-give-up conn fail-ids)] (when (pos? (db/get-update-count given-up)) (l/wrn :hint "giving up on orphan blob after max attempts" :ids fail-ids @@ -160,7 +160,7 @@ [conn size] (db/exec! conn [sql:get-deleted-chunk (ct/now) size])) -(defn- clean-deleted! +(defn- clean-deleted [cfg] (loop [total 0] (let [deleted (db/tx-run! cfg @@ -184,6 +184,6 @@ (defmethod ig/init-key ::handler [_ cfg] (fn [_] - (let [total (clean-deleted! cfg)] + (let [total (clean-deleted cfg)] (l/inf :hint "task finished" :total total) {:deleted total}))) diff --git a/backend/src/app/storage/gc_touched.clj b/backend/src/app/storage/gc_touched.clj index 260739b02f..f73b6024f1 100644 --- a/backend/src/app/storage/gc_touched.clj +++ b/backend/src/app/storage/gc_touched.clj @@ -24,6 +24,7 @@ [app.common.logging :as l] [app.common.time :as ct] [app.db :as db] + [app.metrics :as mtx] [app.storage :as sto] [app.storage.impl :as impl] [integrant.core :as ig])) @@ -95,7 +96,7 @@ SET touched_at = NULL WHERE id = ANY(?::uuid[])") -(defn- mark-freeze-in-bulk! +(defn- mark-freeze-in-bulk [conn ids] (let [ids (db/create-array conn "uuid" ids)] (db/exec-one! conn [sql:mark-freeze-in-bulk ids]))) @@ -106,11 +107,21 @@ touched_at = NULL WHERE id = ANY(?::uuid[])") -(defn- mark-delete-in-bulk! +(defn- mark-delete-in-bulk [conn ids] (let [ids (db/create-array conn "uuid" ids)] (db/exec-one! conn [sql:mark-delete-in-bulk (ct/now) ids]))) +(def ^:private sql:defer-in-bulk + "UPDATE storage_object + SET touched_at = ? + WHERE id = ANY(?::uuid[])") + +(defn- defer-in-bulk + [conn ids timestamp] + (let [ids (db/create-array conn "uuid" ids)] + (db/exec-one! conn [sql:defer-in-bulk timestamp ids]))) + ;; NOTE: A getter that retrieves the key which will be used for group ;; ids; previously we have no value, then we introduced the ;; `:reference` prop, and then it is renamed to `:bucket` and now is @@ -125,12 +136,17 @@ ;; have value, it means :file-media-object. (defn- lookup-bucket - [{:keys [metadata]}] + [{:keys [id metadata]}] (or (some-> metadata :bucket) - (some-> metadata :reference d/name) - sto/default-bucket)) + (do + ;; Only reachable when the metadata column is NULL: the decode + ;; always sets :bucket on non-nil metadata (0155 also backfills + ;; NULL columns). Keep working, but make it visible. + (l/wrn :hint "storage object without bucket metadata, using fallback" + :id (str id)) + sto/default-bucket))) -(defn- process-objects! +(defn- process-objects [conn has-refs? bucket objects] (loop [to-freeze #{} to-delete #{} @@ -148,31 +164,37 @@ :bucket bucket) (recur to-freeze (conj to-delete id) (rest objects)))) (do - (some->> (seq to-freeze) (mark-freeze-in-bulk! conn)) - (some->> (seq to-delete) (mark-delete-in-bulk! conn)) + (some->> (seq to-freeze) (mark-freeze-in-bulk conn)) + (some->> (seq to-delete) (mark-delete-in-bulk conn)) [(count to-freeze) (count to-delete)])))) -(defn- process-bucket! +(defn- process-bucket [conn bucket objects] (cond - (= bucket "file-media-object") (process-objects! conn has-file-media-object-refs? bucket objects) - (= bucket "team-font-variant") (process-objects! conn has-team-font-variant-refs? bucket objects) - (= bucket "file-object-thumbnail") (process-objects! conn has-file-object-thumbnails-refs? bucket objects) - (= bucket "file-thumbnail") (process-objects! conn has-file-thumbnails-refs? bucket objects) - (= bucket "profile") (process-objects! conn has-profile-refs? bucket objects) - (= bucket "file-data") (process-objects! conn has-file-data-refs? bucket objects) - (= bucket sto/tempfile-bucket) (process-objects! conn (constantly false) sto/tempfile-bucket objects) - (= bucket sto/upload-session-bucket) (process-objects! conn (constantly false) sto/upload-session-bucket objects) - (= bucket "organization") (process-objects! conn (constantly false) bucket objects) + (= bucket "file-media-object") (process-objects conn has-file-media-object-refs? bucket objects) + (= bucket "team-font-variant") (process-objects conn has-team-font-variant-refs? bucket objects) + (= bucket "file-object-thumbnail") (process-objects conn has-file-object-thumbnails-refs? bucket objects) + (= bucket "file-thumbnail") (process-objects conn has-file-thumbnails-refs? bucket objects) + (= bucket "profile") (process-objects conn has-profile-refs? bucket objects) + (= bucket "file-data") (process-objects conn has-file-data-refs? bucket objects) + (= bucket sto/tempfile-bucket) (process-objects conn (constantly false) sto/tempfile-bucket objects) + (= bucket sto/upload-session-bucket) (process-objects conn (constantly false) sto/upload-session-bucket objects) + (= bucket "organization") (process-objects conn (constantly false) bucket objects) :else (ex/raise :type :internal :code :unexpected-unknown-reference :hint (dm/fmt "unknown reference '%'" bucket)))) -(defn process-chunk! +(defn- defer-poison + "Defer corrupt rows by one day in their own transaction, separate from + the healthy-chunk work." + [{:keys [::db/conn]} poison-ids] + (defer-in-bulk conn poison-ids (ct/plus (ct/now) {:days 1}))) + +(defn process-chunk [{:keys [::db/conn]} chunk] (reduce-kv (fn [[nfo ndo] bucket objects] - (let [[nfo' ndo'] (process-bucket! conn bucket objects)] + (let [[nfo' ndo'] (process-bucket conn bucket objects)] [(+ nfo nfo') (+ ndo ndo')])) [0 0] @@ -189,21 +211,45 @@ SKIP LOCKED LIMIT 10") +(defn- try-decode-row + "Decode a touched row, capturing corrupt metadata as poison instead + of aborting the whole chunk. Poison rows are deferred by the caller." + [row] + (try + [:ok (impl/decode-row row)] + ;; Exception, not Throwable: JVM Errors (OOM, StackOverflow) must + ;; not be swallowed as a deferrable data problem. + (catch Exception cause + (l/err :hint "storage object with corrupt metadata, deferring evaluation" + :id (str (:id row)) + :cause cause) + [:poison (:id row)]))) + (defn get-chunk [conn timestamp] - (->> (db/exec! conn [sql:get-touched-storage-objects timestamp]) - (map impl/decode-row) - (not-empty))) + (let [grouped (->> (db/exec! conn [sql:get-touched-storage-objects timestamp]) + (map try-decode-row) + (group-by first))] + {:chunk (not-empty (mapv second (:ok grouped))) + :poison (not-empty (mapv second (:poison grouped)))})) -(defn- process-touched! - [{:keys [::db/pool ::timestamp] :as cfg}] +(defn- process-touched + [{:keys [::db/pool ::mtx/metrics ::timestamp] :as cfg}] (loop [freezed 0 deleted 0] - (if-let [chunk (get-chunk pool timestamp)] - (let [[nfo ndo] (db/tx-run! cfg process-chunk! chunk)] - (recur (long (+ freezed nfo)) - (long (+ deleted ndo)))) - {:freeze freezed :delete deleted}))) + (let [{:keys [chunk poison]} (get-chunk pool timestamp)] + (when (seq poison) + (mtx/run! metrics :id :storage-gc-poison :inc (count poison)) + (db/tx-run! cfg defer-poison poison)) + ;; Keep draining after a poison-only batch: the deferred rows leave + ;; the selection and the next batch may hold healthy objects. + (if (or (seq chunk) (seq poison)) + (let [[nfo ndo] (if (seq chunk) + (db/tx-run! cfg process-chunk chunk) + [0 0])] + (recur (long (+ freezed nfo)) + (long (+ deleted ndo)))) + {:freeze freezed :delete deleted})))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; HANDLER @@ -211,7 +257,8 @@ (defmethod ig/assert-key ::handler [_ params] - (assert (db/pool? (::db/pool params)) "expect valid storage")) + (assert (db/pool? (::db/pool params)) "expect valid storage") + (assert (mtx/metrics? (::mtx/metrics params)) "expect valid metrics")) (defmethod ig/expand-key ::handler [k v] @@ -223,5 +270,5 @@ (let [threshold (if (:skip-delay props) (ct/now) (ct/minus (ct/now) min-age))] - (process-touched! (assoc cfg ::timestamp threshold))))) + (process-touched (assoc cfg ::timestamp threshold))))) diff --git a/backend/src/app/storage/impl.clj b/backend/src/app/storage/impl.clj index 76fafe2cac..a6b3d2ca09 100644 --- a/backend/src/app/storage/impl.clj +++ b/backend/src/app/storage/impl.clj @@ -9,8 +9,8 @@ (:require [app.common.data.macros :as dm] [app.common.exceptions :as ex] - [app.db :as db] [app.storage :as-alias sto] + [app.storage.schema :as stsch] [buddy.core.codecs :as bc] [buddy.core.hash :as bh] [clojure.java.io :as jio] @@ -26,7 +26,7 @@ [{:keys [metadata] :as row}] (cond-> row (some? metadata) - (assoc :metadata (db/decode-transit-pgobject metadata)))) + (assoc :metadata (stsch/decode-metadata metadata)))) ;; --- API Definition diff --git a/backend/src/app/storage/schema.clj b/backend/src/app/storage/schema.clj new file mode 100644 index 0000000000..2ded471780 --- /dev/null +++ b/backend/src/app/storage/schema.clj @@ -0,0 +1,149 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns app.storage.schema + "Schema, validation and encode/decode helpers for + `storage_object.metadata`. + + Phase 1: reads accept both the legacy Transit encoding and plain + JSON (sniffed by the `\"~:` marker); writes validate + normalize + but still serialize as Transit unless the + `:storage-metadata-as-json` config flag is set." + (:require + [app.common.data :as d] + [app.common.exceptions :as ex] + [app.common.schema :as sm] + [app.config :as cf] + [app.db :as db] + [cuerdas.core :as str]) + (:import + org.postgresql.util.PGobject)) + +(def default-bucket + "file-media-object") + +(def tempfile-bucket + "Bucket name for temporary file uploads (10-minute expiry)." + "tempfile") + +(def upload-session-bucket + "Bucket name for chunked-upload chunks." + "upload-session") + +(def bucket-requirements + "Canonical buckets and the extra keys each one must carry, so a new + bucket and its contract live in one place. Listed keys are declared as + optional in `schema:metadata` and only checked for presence; their type + is enforced by the map. Only buckets whose keys are read somewhere need + an entry: `file-data` ids are resolved by the GC, `organization-id` is + the logo's owner." + {"file-media-object" #{} + "team-font-variant" #{} + "file-object-thumbnail" #{} + "file-thumbnail" #{} + "profile" #{} + "organization" #{:organization-id} + tempfile-bucket #{} + upload-session-bucket #{} + "file-data" #{:file-id :id} + "file-data-fragment" #{} + "file-change" #{}}) + +(def metadata-buckets + "Canonical bucket set. `app.storage/valid-buckets` aliases it so the + list lives in exactly one place." + (set (keys bucket-requirements))) + +(defn- bucket-requirements-present? + [{:keys [bucket] :as mdata}] + (every? #(some? (get mdata %)) (get bucket-requirements bucket #{}))) + +(def schema:metadata + "Closed schema for `storage_object.metadata`. A single shape shared by + every bucket: `:bucket` is checked against `metadata-buckets` and the + rest are typed optional keys. `bucket-requirements` adds the per-bucket + presence checks." + [:and + [:map {:closed true} + [:bucket [::sm/one-of {:format :string} metadata-buckets]] + [:content-type :string] + [:hash {:optional true} :string] + [:profile-id {:optional true} ::sm/uuid] + [:organization-id {:optional true} ::sm/uuid] + [:file-id {:optional true} ::sm/uuid] + [:id {:optional true} ::sm/uuid]] + [:fn {:error/message "storage metadata is missing a required key for its bucket"} + bucket-requirements-present?]]) + +(defn- ->bucket + [v] + (cond + (string? v) v + (keyword? v) (d/name v) + :else (str v))) + +(defn- normalize-metadata + "Bring decoded (or incoming) metadata to its canonical shape: + `:bucket` is always present (`:reference` is mapped to it, keyword + or string, and defaults to `default-bucket`), and the legacy + `:reference`, `:upload-id` and `:chunk-index` keys are dropped." + [mdata] + (let [bucket (or (not-empty (->bucket (:bucket mdata))) + (not-empty (->bucket (:reference mdata))) + default-bucket)] + (-> mdata + (dissoc :reference :upload-id :chunk-index) + (assoc :bucket bucket)))) + +;; Decoder and encoder are built once per process: creating them +;; compiles the closed schema, and `decode-metadata` runs on every read +;; path (get-object, dedup probes, GC batches). +(def ^:private decode-metadata-fn + (sm/decode-fn schema:metadata (sm/json-transformer))) + +(def ^:private encode-metadata-fn + (sm/encoder schema:metadata (sm/json-transformer))) + +(defn- transit-encoded? + "Legacy metadata is Transit json-verbose, where every top-level key is + a keyword, so the document starts with `{\"~:`. Anchoring the check to + the start avoids reading a plain-JSON document as Transit just because + one of its values contains `\"~:`." + [value] + (and (string? value) + (str/starts-with? (str/trim value) "{\"~:"))) + +(defn decode-metadata + "Decode storage metadata from a PGobject, accepting both the legacy + Transit encoding (sniffed by the `\"~:` marker) and plain JSON. + Always returns the normalized shape with id fields as UUIDs." + [o] + (when (some? o) + (let [raw (if (transit-encoded? (.getValue ^PGobject o)) + (db/decode-transit-pgobject o) + (db/decode-json-pgobject o))] + (when-not (map? raw) + (ex/raise :type :internal + :code :invalid-storage-metadata + :hint "expected a map on storage object metadata")) + (decode-metadata-fn (normalize-metadata raw))))) + +(def ^:private check-metadata + (sm/check-fn schema:metadata + :hint "invalid storage object metadata" + :type :validation + :code :invalid-storage-metadata)) + +(defn encode-metadata + "Validate, normalize and encode storage metadata into a PGobject + ready for the `metadata` column. Serializes as Transit while the + `:storage-metadata-as-json` config flag is unset, as plain JSON + when it is set." + [mdata] + (let [mdata (check-metadata (normalize-metadata mdata))] + (if (cf/get :storage-metadata-as-json) + (db/json (encode-metadata-fn mdata)) + (db/tjson mdata)))) diff --git a/backend/test/backend_tests/http_assets_test.clj b/backend/test/backend_tests/http_assets_test.clj index 095d04f3e7..a5e60e9965 100644 --- a/backend/test/backend_tests/http_assets_test.clj +++ b/backend/test/backend_tests/http_assets_test.clj @@ -56,7 +56,17 @@ :bucket bucket :content-type "text/plain"} (some? profile-id) - (assoc :profile-id profile-id))))) + (assoc :profile-id profile-id) + + ;; file-data objects require file/id + ;; references for GC (has-file-data-refs?). + (= bucket "file-data") + (assoc :file-id (uuid/random) + :id (uuid/random)) + + ;; organization objects require their owner id. + (= bucket "organization") + (assoc :organization-id (uuid/random)))))) (defn- make-metrics [] diff --git a/backend/test/backend_tests/http_middleware_test.clj b/backend/test/backend_tests/http_middleware_test.clj index e058e2fb53..1651d92e0a 100644 --- a/backend/test/backend_tests/http_middleware_test.clj +++ b/backend/test/backend_tests/http_middleware_test.clj @@ -8,6 +8,8 @@ (:require [app.common.exceptions :as ex] [app.common.time :as ct] + [app.common.uuid :as uuid] + [app.config :as cf] [app.db :as db] [app.http :as-alias http] [app.http.access-token] @@ -21,6 +23,7 @@ [backend-tests.helpers :as th] [clojure.string :as str] [clojure.test :as t] + [integrant.core :as ig] [mockery.core :refer [with-mocks]] [yetti.request :as yreq] [yetti.response :as yres]) @@ -56,12 +59,11 @@ (handler (th/make-dummy-request {})) (t/is (nil? (::http/auth-data @request))) + ;; A bearer token is only attached when it is a current session + ;; token (kid=1/ver=1) and a decoder is configured. Otherwise the + ;; request stays unauthenticated. (handler (th/make-dummy-request {:headers {"authorization" "Bearer aaaa"}})) - - (let [{:keys [token claims] token-type :type} (get @request ::http/auth-data)] - (t/is (= :bearer token-type)) - (t/is (= "aaaa" token)) - (t/is (nil? claims))))) + (t/is (nil? (::http/auth-data @request))))) (t/deftest auth-middleware-3 (let [request (volatile! nil) @@ -73,11 +75,7 @@ (t/is (nil? (::http/auth-data @request))) (handler (th/make-dummy-request {:cookies {"auth-token" "foobar"}})) - - (let [{:keys [token claims] token-type :type} (get @request ::http/auth-data)] - (t/is (= :cookie token-type)) - (t/is (= "foobar" token)) - (t/is (nil? claims))))) + (t/is (nil? (::http/auth-data @request))))) (t/deftest shared-key-auth (let [handler (#'app.http.middleware/wrap-shared-key-auth @@ -230,56 +228,282 @@ (t/is (ct/inst? exp) "exp should be an instant"))) (t/deftest session-token-exp-based-on-created-at - (let [cfg th/*system* - manager (session/inmemory-manager) - profile (th/create-profile* 1) - session (->> (session/create-session manager {:profile-id (:id profile) - :user-agent "user agent"}) - (#'session/assign-token cfg)) - claims (tokens/decode cfg (:token session)) - expected-exp (ct/plus (:created-at session) (ct/duration {:days 30}))] + (let [cfg th/*system* + manager (session/inmemory-manager) + profile (th/create-profile* 1) + session (->> (session/create-session manager {:profile-id (:id profile) + :user-agent "user agent"}) + (#'session/assign-token cfg)) + claims (tokens/decode cfg (:token session)) + expected-exp (ct/plus (:created-at session) (ct/duration {:days 30}))] (t/is (some? (:exp claims)) "session token should contain :exp claim") (t/is (= (inst-ms (:exp claims)) (inst-ms expected-exp)) "exp should equal created-at + 30 days"))) (t/deftest session-token-past-exp-is-rejected - (let [cfg th/*system* - manager (session/inmemory-manager) - profile (th/create-profile* 1) - session (->> (session/create-session manager {:profile-id (:id profile) - :user-agent "user agent"}) - (#'session/assign-token cfg)) - claims (tokens/decode cfg (:token session)) - ;; Manually create a token with exp in the past + (let [cfg th/*system* + manager (session/inmemory-manager) + profile (th/create-profile* 1) + session (->> (session/create-session manager {:profile-id (:id profile) + :user-agent "user agent"}) + (#'session/assign-token cfg)) + claims (tokens/decode cfg (:token session)) past-claims (assoc claims :exp (ct/minus (ct/now) (ct/duration {:days 1}))) - header {:kid 1 :ver 1} - past-token (tokens/generate cfg past-claims header)] + past-token (tokens/generate cfg past-claims {:kid 1 :ver 1})] (t/is (nil? (session/decode-token cfg past-token)) "token with exp in the past should be rejected"))) (t/deftest session-renewal-preserves-original-exp (let [cfg th/*system* - manager (session/inmemory-manager) profile (th/create-profile* 1) + created (ct/minus (ct/now) (ct/duration {:days 1})) + session {:id (uuid/random) + :profile-id (:id profile) + :user-agent "user agent" + :created-at created + :modified-at (ct/minus (ct/now) (ct/duration {:hours 7}))} + manager (reify session/ISessionManager + (read-session [_ _] session) + (create-session [_ _] session) + (update-session [_ s] (assoc s :modified-at (ct/now))) + (delete-session [_ _] nil)) + + old-token (:token (#'session/assign-token cfg session)) + original-exp (:exp (tokens/decode cfg old-token)) + handler (-> (fn [req] req) + (#'session/wrap-authz (assoc th/*system* ::session/manager manager)) + (#'mw/wrap-auth {:bearer (partial session/decode-token cfg) + :cookie (partial session/decode-token cfg)})) + response (handler (th/make-dummy-request {:cookies {"auth-token" old-token}})) + renewed-token (get-in response [::yres/cookies "auth-token" :value]) + renewed-claims (tokens/decode cfg renewed-token)] + (t/is (some? original-exp) "original token should have :exp") + (t/is (not= old-token renewed-token) "renewal should issue a new token string") + (t/is (= (inst-ms original-exp) (inst-ms (:exp renewed-claims))) + "renewed token should preserve the original :exp, not extend it"))) + +(t/deftest session-renewal-preserves-exp-with-db-manager + (let [cfg th/*system* + manager (::session/manager th/*system*) + profile (th/create-profile* 1) + created (session/create-session manager {:profile-id (:id profile) + :user-agent "user agent"}) + _ (th/db-exec-one! ["UPDATE http_session_v2 + SET modified_at = now() - interval '7 hours' + WHERE id = ?" (:id created)]) + stale (session/read-session manager (:id created)) + old-token (:token (#'session/assign-token cfg stale)) + original-exp (:exp (tokens/decode cfg old-token)) + handler (-> (fn [req] req) + (#'session/wrap-authz cfg) + (#'mw/wrap-auth {:bearer (partial session/decode-token cfg) + :cookie (partial session/decode-token cfg)})) + response (handler (th/make-dummy-request {:cookies {"auth-token" old-token}})) + renewed (get-in response [::yres/cookies "auth-token" :value]) + renewed-exp (:exp (tokens/decode cfg renewed)) + expected-exp (ct/plus (:created-at created) (ct/duration {:days 30})) + current (session/read-session manager (:id created))] + (t/is (some? original-exp) "original token should have :exp") + (t/is (some? renewed) "renewal should issue a new cookie token") + (t/is (not= old-token renewed) "renewal should issue a new token string") + (t/is (= (inst-ms original-exp) (inst-ms renewed-exp)) + "renewed token should preserve the original :exp, not extend it") + (t/is (= (inst-ms expected-exp) (inst-ms renewed-exp)) + "renewed :exp should equal created-at + 30 days") + (t/is (some? current) "session row must still exist after renewal") + (t/is (pos? (compare (:modified-at current) (:modified-at stale))) + "persisted modified_at must move forward on renewal"))) + +(t/deftest session-renewal-cookie-expires-diverges-from-token-exp + (let [cfg th/*system* + manager (::session/manager th/*system*) + profile (th/create-profile* 1) + created (session/create-session manager {:profile-id (:id profile) + :user-agent "user agent"}) + _ (th/db-exec-one! ["UPDATE http_session_v2 + SET created_at = now() - interval '29 days', + modified_at = now() - interval '7 hours' + WHERE id = ?" (:id created)]) + stale (session/read-session manager (:id created)) + old-token (:token (#'session/assign-token cfg stale)) + handler (-> (fn [req] req) + (#'session/wrap-authz cfg) + (#'mw/wrap-auth {:bearer (partial session/decode-token cfg) + :cookie (partial session/decode-token cfg)})) + response (handler (th/make-dummy-request {:cookies {"auth-token" old-token}})) + cookie (get-in response [::yres/cookies "auth-token"]) + renewed (:value cookie) + renewed-exp (:exp (tokens/decode cfg renewed)) + expected-exp (ct/plus (:created-at stale) (ct/duration {:days 30})) + close-to? (fn [a b tolerance-ms] + (<= (Math/abs (- (inst-ms a) (inst-ms b))) tolerance-ms))] + (t/is (some? renewed) "renewal should issue a new cookie token") + (t/is (not= old-token renewed) "renewal should issue a new token string") + (t/is (= (inst-ms expected-exp) (inst-ms renewed-exp)) + "renewed :exp should equal created-at + 30 days") + (t/is (close-to? renewed-exp (ct/plus (ct/now) (ct/duration {:days 1})) + (* 10 60 1000)) + "renewed :exp should be ~1 day out (absolute cap is near)") + (t/is (close-to? (:expires cookie) (ct/plus (ct/now) (ct/duration {:days 7})) + (* 10 60 1000)) + "cookie Expires should slide ~7 days out from now") + (t/is (pos? (compare (:expires cookie) renewed-exp)) + "cookie Expires should stay ahead of the token :exp"))) + +(t/deftest legacy-session-token-is-rejected + (let [cfg th/*system* + manager (session/inmemory-manager) handler (-> (fn [req] req) - (#'session/wrap-authz {::session/manager manager}) + (#'session/wrap-authz {::session/manager manager}) (#'mw/wrap-auth {:bearer (partial session/decode-token cfg) :cookie (partial session/decode-token cfg)})) - session (->> (session/create-session manager {:profile-id (:id profile) - :user-agent "user agent"}) - (#'session/assign-token cfg)) - original-exp (:exp (tokens/decode cfg (:token session))) - ;; Force renewal by setting modified-at to 7 hours ago - old-session (assoc session :modified-at (ct/minus (ct/now) (ct/duration {:hours 7}))) - response (handler (th/make-dummy-request {:cookies {"auth-token" (:token old-session)}})) - {:keys [token claims]} (get response ::http/auth-data) - new-exp (:exp claims)] - (t/is (some? original-exp) "original token should have :exp") - (t/is (some? new-exp) "renewed token should have :exp") - (t/is (= (inst-ms original-exp) - (inst-ms new-exp)) - "renewed token should preserve original :exp, not extend it"))) + token (tokens/generate cfg {:sid "legacy-session-id"} {:kid 0 :ver 0}) + response (handler (th/make-dummy-request {:cookies {"auth-token" token}}))] + (t/is (nil? (get response ::http/auth-data)) + "legacy tokens must not be attached as auth data") + (t/is (nil? (::session/profile-id response)) + "legacy tokens must not authenticate"))) + +(t/deftest session-gc-deletes-idle-and-absolute-expired-rows + (let [profile (th/create-profile* 1) + fresh (uuid/random) + idle (uuid/random) + absolute (uuid/random) + valid (uuid/random)] + + (th/db-exec-one! ["INSERT INTO http_session_v2 (id, profile_id, created_at, modified_at) + VALUES (?, ?, now(), now())" + fresh (:id profile)]) + (th/db-exec-one! ["INSERT INTO http_session_v2 (id, profile_id, created_at, modified_at) + VALUES (?, ?, now() - interval '1 day', now() - interval '8 days')" + idle (:id profile)]) + (th/db-exec-one! ["INSERT INTO http_session_v2 (id, profile_id, created_at, modified_at) + VALUES (?, ?, now() - interval '31 days', now())" + absolute (:id profile)]) + (th/db-exec-one! ["INSERT INTO http_session_v2 (id, profile_id, created_at, modified_at) + VALUES (?, ?, now() - interval '1 day', now() - interval '6 days')" + valid (:id profile)]) + + (db/tx-run! th/*system* + (fn [cfg] + (#'session/collect-expired-tasks + (assoc cfg + :app.http.session.tasks/max-age (ct/duration {:days 7}) + :app.http.session.tasks/max-age-absolute (ct/duration {:days 30}))))) + + (let [ids (->> (th/db-exec! ["SELECT id FROM http_session_v2 WHERE profile_id = ?" (:id profile)]) + (map :id) + (set))] + (t/is (contains? ids fresh) "fresh session must be kept") + (t/is (contains? ids valid) "session within both windows must be kept") + (t/is (not (contains? ids idle)) "idle session must be deleted") + (t/is (not (contains? ids absolute)) "session past the absolute cap must be deleted")))) + +(t/deftest session-gc-config-wiring + (let [idle (ct/duration {:days 3}) + absolute (ct/duration {:days 10})] + (with-redefs [cf/get (fn + ([k] (case k + :auth-token-cookie-max-age idle + :auth-token-cookie-max-age-absolute absolute + nil)) + ([k default] (case k + :auth-token-cookie-max-age idle + :auth-token-cookie-max-age-absolute absolute + default)))] + (let [expanded (ig/expand-key :app.http.session.tasks/gc {})] + (t/is (= idle + (get-in expanded [:app.http.session.tasks/gc + :app.http.session.tasks/max-age])) + "task max-age should carry the configured idle window") + (t/is (= absolute + (get-in expanded [:app.http.session.tasks/gc + :app.http.session.tasks/max-age-absolute])) + "task max-age-absolute should carry the configured absolute cap"))) + (with-redefs [cf/get (fn + ([_k] nil) + ([_k default] default))] + (let [expanded (ig/expand-key :app.http.session.tasks/gc {})] + (t/is (= session/default-cookie-max-age + (get-in expanded [:app.http.session.tasks/gc + :app.http.session.tasks/max-age])) + "task max-age should fall back to the default idle window") + (t/is (= session/default-cookie-max-age-absolute + (get-in expanded [:app.http.session.tasks/gc + :app.http.session.tasks/max-age-absolute])) + "task max-age-absolute should fall back to the default absolute cap"))))) + +(t/deftest session-gc-rejects-absolute-below-idle + (let [pool (:app.db/pool th/*system*) + params {:app.db/pool pool + :app.http.session.tasks/max-age (ct/duration {:days 7}) + :app.http.session.tasks/max-age-absolute (ct/duration {:days 30})}] + (t/is (nil? (ig/assert-key :app.http.session.tasks/gc params)) + "absolute cap above the idle window should pass") + (t/is (nil? (ig/assert-key :app.http.session.tasks/gc + (assoc params + :app.http.session.tasks/max-age-absolute + (ct/duration {:days 7})))) + "absolute cap equal to the idle window should pass") + (t/is (thrown? IllegalArgumentException + (ig/assert-key :app.http.session.tasks/gc + (assoc params + :app.http.session.tasks/max-age-absolute + (ct/duration {:days 3})))) + "absolute cap below the idle window should fail fast"))) + +(t/deftest idle-expired-session-is-rejected + (let [cfg th/*system* + profile (th/create-profile* 1) + updated? (atom false) + session {:id (uuid/random) + :profile-id (:id profile) + :user-agent "user agent" + :created-at (ct/now) + :modified-at (ct/minus (ct/now) (ct/duration {:days 8}))} + manager (reify session/ISessionManager + (read-session [_ _] session) + (create-session [_ _] session) + (update-session [_ s] (reset! updated? true) s) + (delete-session [_ _] nil)) + token (:token (#'session/assign-token cfg session)) + handler (-> (fn [req] req) + (#'session/wrap-authz (assoc cfg ::session/manager manager)) + (#'mw/wrap-auth {:bearer (partial session/decode-token cfg) + :cookie (partial session/decode-token cfg)})) + cookie-r (handler (th/make-dummy-request {:cookies {"auth-token" token}})) + bearer-r (handler (th/make-dummy-request {:headers {"authorization" (str "Bearer " token)}}))] + (t/is (nil? (::session/profile-id cookie-r)) + "idle-expired session must not authenticate via cookie") + (t/is (nil? (::session/session cookie-r)) + "idle-expired session must not be attached via cookie") + (t/is (nil? (::session/profile-id bearer-r)) + "idle-expired session must not authenticate via bearer") + (t/is (false? @updated?) + "idle-expired session must not be renewed"))) + +(t/deftest idle-session-within-window-is-accepted + (let [cfg th/*system* + profile (th/create-profile* 1) + session {:id (uuid/random) + :profile-id (:id profile) + :user-agent "user agent" + :created-at (ct/now) + :modified-at (ct/minus (ct/now) (ct/duration {:days 6}))} + manager (reify session/ISessionManager + (read-session [_ _] session) + (create-session [_ _] session) + (update-session [_ s] (assoc s :modified-at (ct/now))) + (delete-session [_ _] nil)) + token (:token (#'session/assign-token cfg session)) + handler (-> (fn [req] req) + (#'session/wrap-authz (assoc cfg ::session/manager manager)) + (#'mw/wrap-auth {:bearer (partial session/decode-token cfg) + :cookie (partial session/decode-token cfg)})) + response (handler (th/make-dummy-request {:cookies {"auth-token" token}}))] + (t/is (= (:id profile) (::session/profile-id response)) + "session within the idle window must authenticate"))) (t/deftest parse-request-illegal-argument-exception ;; clojure.data.json raises IllegalArgumentException (case diff --git a/backend/test/backend_tests/rpc_management_test.clj b/backend/test/backend_tests/rpc_management_test.clj index 570b858aaf..4b449af993 100644 --- a/backend/test/backend_tests/rpc_management_test.clj +++ b/backend/test/backend_tests/rpc_management_test.clj @@ -85,8 +85,7 @@ (configure-storage-backend)) sobject (sto/put-object! storage {::sto/content (sto/content "content") - :content-type "text/plain" - :other "data"}) + :content-type "text/plain"}) profile (th/create-profile* 1 {:is-active true}) project (th/create-project* 1 {:team-id (:default-team-id profile) :profile-id (:id profile)}) @@ -157,8 +156,7 @@ (let [storage (-> (:app.storage/storage th/*system*) (configure-storage-backend)) sobject (sto/put-object! storage {::sto/content (sto/content "content") - :content-type "text/plain" - :other "data"}) + :content-type "text/plain"}) profile (th/create-profile* 1 {:is-active true}) project (th/create-project* 1 {:team-id (:default-team-id profile) @@ -214,8 +212,7 @@ (configure-storage-backend)) sobject (sto/put-object! storage {::sto/content (sto/content "content") - :content-type "text/plain" - :other "data"}) + :content-type "text/plain"}) profile (th/create-profile* 1 {:is-active true}) project (th/create-project* 1 {:team-id (:default-team-id profile) @@ -278,8 +275,7 @@ (let [storage (-> (:app.storage/storage th/*system*) (configure-storage-backend)) sobject (sto/put-object! storage {::sto/content (sto/content "content") - :content-type "text/plain" - :other "data"}) + :content-type "text/plain"}) profile (th/create-profile* 1 {:is-active true}) project (th/create-project* 1 {:team-id (:default-team-id profile) :profile-id (:id profile)}) diff --git a/backend/test/backend_tests/storage_metadata_test.clj b/backend/test/backend_tests/storage_metadata_test.clj new file mode 100644 index 0000000000..4efb4c80df --- /dev/null +++ b/backend/test/backend_tests/storage_metadata_test.clj @@ -0,0 +1,276 @@ +;; This Source Code Form is subject to the terms of the Mozilla Public +;; License, v. 2.0. If a copy of the MPL was not distributed with this +;; file, You can obtain one at http://mozilla.org/MPL/2.0/. +;; +;; Copyright (c) KALEIDOS SUBSIDIARY SL + +(ns backend-tests.storage-metadata-test + (:require + [app.config :as cf] + [app.db :as db] + [app.storage.schema :as stsch] + [clojure.string :as str] + [clojure.test :as t]) + (:import + org.postgresql.util.PGobject)) + +(defn- pgobject + [^String value] + (doto (PGobject.) + (.setType "jsonb") + (.setValue value))) + +;; Raw Transit payloads shaped like the production rows sampled on +;; 2026-09-11 (verbose transit: "~:key" keys, "~u" uuid values, +;; "~:xxx" keyword values). +(def ^:private transit-media + "{\"~:hash\":\"blake2b:9f1c2e\",\"~:bucket\":\"file-media-object\",\"~:content-type\":\"image/png\"}") + +(def ^:private transit-tempfile + "{\"~:hash\":\"blake2b:9f1c2e\",\"~:bucket\":\"tempfile\",\"~:content-type\":\"application/zip\",\"~:profile-id\":\"~u86907e95-1cb8-8122-8008-4eb7ba07d89d\"}") + +(def ^:private transit-legacy-reference + "{\"~:reference\":\"~:file-media-object\",\"~:content-type\":\"image/png\",\"~:hash\":\"blake2b:9f1c2e\"}") + +(def ^:private transit-no-bucket + "{\"~:content-type\":\"image/svg+xml\"}") + +(def ^:private transit-chunk-leftovers + "{\"~:bucket\":\"tempfile\",\"~:content-type\":\"application/zip\",\"~:upload-id\":\"~u86907e95-1cb8-8122-8008-4eb7ba07d89d\",\"~:chunk-index\":3}") + +(def ^:private json-file-data + "{\"bucket\":\"file-data\",\"content-type\":\"application/octet-stream\",\"file-id\":\"86907e95-1cb8-8122-8008-4eb7ba07d89d\",\"id\":\"83df2f92-6bd4-4e6d-9c9a-3f6d2b1a4c55\"}") + +(t/deftest decode-transit-returns-native-types + (let [mdata (stsch/decode-metadata (pgobject transit-tempfile))] + (t/is (= "tempfile" (:bucket mdata))) + (t/is (= "application/zip" (:content-type mdata))) + (t/is (= "blake2b:9f1c2e" (:hash mdata))) + (t/is (uuid? (:profile-id mdata))) + (t/is (= (parse-uuid "86907e95-1cb8-8122-8008-4eb7ba07d89d") + (:profile-id mdata))))) + +(t/deftest decode-transit-maps-reference-to-bucket + (let [mdata (stsch/decode-metadata (pgobject transit-legacy-reference))] + (t/is (= "file-media-object" (:bucket mdata))) + (t/is (= "image/png" (:content-type mdata))) + (t/is (not (contains? mdata :reference))))) + +(t/deftest decode-transit-defaults-missing-bucket + (let [mdata (stsch/decode-metadata (pgobject transit-no-bucket))] + (t/is (= "file-media-object" (:bucket mdata))) + (t/is (= "image/svg+xml" (:content-type mdata))))) + +(t/deftest decode-transit-drops-chunk-leftovers + (let [mdata (stsch/decode-metadata (pgobject transit-chunk-leftovers))] + (t/is (= "tempfile" (:bucket mdata))) + (t/is (not (contains? mdata :upload-id))) + (t/is (not (contains? mdata :chunk-index))))) + +(t/deftest decode-json-coerces-uuids + (let [mdata (stsch/decode-metadata (pgobject json-file-data))] + (t/is (= "file-data" (:bucket mdata))) + (t/is (uuid? (:file-id mdata))) + (t/is (uuid? (:id mdata))) + (t/is (= (parse-uuid "86907e95-1cb8-8122-8008-4eb7ba07d89d") + (:file-id mdata))))) + +(t/deftest sniff-has-no-false-positives-on-values + ;; "~:" inside a value (not anchored to a quote) must not trigger + ;; the transit branch. + (let [mdata (stsch/decode-metadata + (pgobject "{\"bucket\":\"tempfile\",\"content-type\":\"text/plain;~:x\"}"))] + (t/is (= "tempfile" (:bucket mdata))) + (t/is (= "text/plain;~:x" (:content-type mdata))))) + +(t/deftest encode-writes-transit-by-default + ;; Pinned off: without the binding this test inherits the ambient + ;; config and proves nothing where the flag is set. + (binding [cf/config (assoc cf/config :storage-metadata-as-json nil)] + (let [encoded (stsch/encode-metadata {:bucket "file-media-object" + :content-type "image/png" + :hash "blake2b:9f1c2e"}) + value (.getValue ^PGobject encoded)] + (t/is (string? value)) + (t/is (str/includes? value "\"~:bucket\"")) + (t/is (not (str/includes? value "\"~:reference\"")))))) + +(t/deftest encode-normalizes-legacy-input + (binding [cf/config (assoc cf/config :storage-metadata-as-json nil)] + (let [encoded (stsch/encode-metadata {:reference :file-media-object + :content-type "image/png"}) + value (.getValue ^PGobject encoded) + decoded (stsch/decode-metadata encoded)] + (t/is (= "file-media-object" (:bucket decoded))) + (t/is (not (str/includes? value "\"~:reference\"")))))) + +(t/deftest encode-writes-plain-json-with-flag + (binding [cf/config (assoc cf/config :storage-metadata-as-json true)] + (let [file-id (parse-uuid "86907e95-1cb8-8122-8008-4eb7ba07d89d") + encoded (stsch/encode-metadata {:bucket "file-data" + :content-type "application/octet-stream" + :file-id file-id + :id (parse-uuid "83df2f92-6bd4-4e6d-9c9a-3f6d2b1a4c55")}) + value (.getValue ^PGobject encoded)] + (t/is (str/includes? value "\"bucket\"")) + (t/is (not (str/includes? value "\"~:"))) + ;; uuids travel as plain strings on the JSON encoding + (t/is (str/includes? value (str file-id))) + (let [decoded (stsch/decode-metadata encoded)] + (t/is (uuid? (:file-id decoded))) + (t/is (= file-id (:file-id decoded))))))) + +(t/deftest encode-rejects-string-uuids + ;; Encoding does not coerce input types: callers must pass native UUIDs + ;; (every producer does; reads still coerce on the way back). + (t/is (thrown-with-msg? clojure.lang.ExceptionInfo + #"invalid storage object metadata" + (stsch/encode-metadata {:bucket "tempfile" + :content-type "application/zip" + :profile-id "86907e95-1cb8-8122-8008-4eb7ba07d89d"})))) + +(t/deftest encode-rejects-unknown-bucket + (t/is (thrown-with-msg? clojure.lang.ExceptionInfo + #"invalid storage object metadata" + (stsch/encode-metadata {:bucket "no-such-bucket" + :content-type "image/png"})))) + +(t/deftest encode-rejects-extra-key + (t/is (thrown-with-msg? clojure.lang.ExceptionInfo + #"invalid storage object metadata" + (stsch/encode-metadata {:bucket "file-media-object" + :content-type "image/png" + :other "data"})))) + +(t/deftest encode-rejects-malformed-uuid + (t/is (thrown-with-msg? clojure.lang.ExceptionInfo + #"invalid storage object metadata" + (stsch/encode-metadata {:bucket "file-data" + :content-type "application/octet-stream" + :file-id "not-a-uuid" + :id "83df2f92-6bd4-4e6d-9c9a-3f6d2b1a4c55"})))) + +(t/deftest encode-rejects-missing-content-type + (t/is (thrown-with-msg? clojure.lang.ExceptionInfo + #"invalid storage object metadata" + (stsch/encode-metadata {:bucket "file-media-object"})))) + +(t/deftest encode-rejects-missing-file-data-ids + ;; Both ids are required for file-data; omitting one fails the bucket + ;; requirement check (the file-id here is a native UUID, so the map + ;; itself is valid). + (t/is (thrown-with-msg? clojure.lang.ExceptionInfo + #"invalid storage object metadata" + (stsch/encode-metadata {:bucket "file-data" + :content-type "application/octet-stream" + :file-id (parse-uuid "86907e95-1cb8-8122-8008-4eb7ba07d89d")})))) + +(t/deftest encode-rejects-missing-organization-id + (t/is (thrown-with-msg? clojure.lang.ExceptionInfo + #"invalid storage object metadata" + (stsch/encode-metadata {:bucket "organization" + :content-type "image/svg+xml"})))) + +(t/deftest decode-real-transit-payload + ;; Same legacy shape as above but produced by the real transit + ;; encoder, so the test breaks if the encoding ever drifts. + (let [mdata (stsch/decode-metadata + (db/tjson {:reference :team-font-variant + :content-type "font/woff2"}))] + (t/is (= "team-font-variant" (:bucket mdata))) + (t/is (not (contains? mdata :reference))))) + +(t/deftest transit-and-json-encodings-decode-to-same-shape + (let [mdata {:bucket "tempfile" + :content-type "application/zip" + :profile-id (parse-uuid "86907e95-1cb8-8122-8008-4eb7ba07d89d")} + transit (binding [cf/config (assoc cf/config :storage-metadata-as-json nil)] + (stsch/decode-metadata (stsch/encode-metadata mdata))) + json (binding [cf/config (assoc cf/config :storage-metadata-as-json true)] + (stsch/decode-metadata (stsch/encode-metadata mdata)))] + (t/is (= transit json)) + (t/is (= mdata json)))) + +(t/deftest flag-on-and-off-differ-only-in-encoding-family + ;; The Phase 2 rollback contract: flipping the flag changes how the + ;; same logical metadata hits the disk, never what it means. + (let [mdata {:bucket "file-media-object" + :content-type "image/png" + :hash "blake2b:9f1c2e"} + transit (binding [cf/config (assoc cf/config :storage-metadata-as-json nil)] + (.getValue ^PGobject (stsch/encode-metadata mdata))) + json (binding [cf/config (assoc cf/config :storage-metadata-as-json true)] + (.getValue ^PGobject (stsch/encode-metadata mdata)))] + (t/is (str/includes? transit "\"~:bucket\"")) + (t/is (str/includes? json "\"bucket\"")) + (t/is (not (str/includes? json "\"~:"))) + (t/is (= (stsch/decode-metadata (pgobject transit)) + (stsch/decode-metadata (pgobject json)))))) + +(t/deftest decode-json-keeps-hash-byte-exact + ;; Dedup matches on the hash string; it must survive the JSON + ;; roundtrip untouched. + (let [mdata (stsch/decode-metadata + (pgobject "{\"bucket\":\"file-media-object\",\"content-type\":\"image/png\",\"hash\":\"blake2b:9f1c2e\"}"))] + (t/is (= "blake2b:9f1c2e" (:hash mdata))))) + +(def ^:private bucket-samples + ;; One representative payload per bucket, with every required key and + ;; one optional key where the bucket declares any. + [["file-media-object" + {:bucket "file-media-object" :content-type "image/png" :hash "blake2b:9f1c2e"}] + ["team-font-variant" + {:bucket "team-font-variant" :content-type "font/woff2"}] + ["file-object-thumbnail" + {:bucket "file-object-thumbnail" :content-type "image/png"}] + ["file-thumbnail" + {:bucket "file-thumbnail" :content-type "image/png"}] + ["profile" + {:bucket "profile" :content-type "image/png"}] + ["organization" + {:bucket "organization" :content-type "image/svg+xml" + :organization-id #uuid "11111111-2222-3333-4444-555555555555"}] + ["tempfile" + {:bucket "tempfile" :content-type "application/zip" + :profile-id #uuid "86907e95-1cb8-8122-8008-4eb7ba07d89d"}] + ["upload-session" + {:bucket "upload-session" :content-type "application/octet-stream"}] + ["file-data" + {:bucket "file-data" :content-type "application/octet-stream" + :file-id #uuid "86907e95-1cb8-8122-8008-4eb7ba07d89d" + :id #uuid "83df2f92-6bd4-4e6d-9c9a-3f6d2b1a4c55"}] + ["file-data-fragment" + {:bucket "file-data-fragment" :content-type "application/octet-stream"}] + ["file-change" + {:bucket "file-change" :content-type "application/octet-stream"}]]) + +(t/deftest encode-decode-roundtrips-every-bucket + (binding [cf/config (assoc cf/config :storage-metadata-as-json nil)] + (doseq [[bucket mdata] bucket-samples] + (t/testing bucket + (t/is (= mdata (stsch/decode-metadata (stsch/encode-metadata mdata)))))))) + +(t/deftest json-encoding-roundtrips-every-bucket + (binding [cf/config (assoc cf/config :storage-metadata-as-json true)] + (doseq [[bucket mdata] bucket-samples] + (t/testing bucket + (t/is (= mdata (stsch/decode-metadata (stsch/encode-metadata mdata)))))))) + +(t/deftest metadata-buckets-is-the-canonical-set + (t/is (= #{"file-media-object" "team-font-variant" "file-object-thumbnail" + "file-thumbnail" "profile" "organization" "tempfile" + "upload-session" "file-data" "file-data-fragment" "file-change"} + stsch/metadata-buckets))) + +(t/deftest decode-metadata-returns-nil-for-nil + (t/is (nil? (stsch/decode-metadata nil)))) + +(t/deftest sniff-treats-leading-tilde-value-as-json + ;; A plain-JSON value starting with `~:` must not switch the reader to + ;; the Transit branch: the sniff is anchored to the first key. + (let [encoded (binding [cf/config (assoc cf/config :storage-metadata-as-json true)] + (stsch/encode-metadata {:bucket "tempfile" + :content-type "~:not-transit"})) + decoded (stsch/decode-metadata encoded)] + (t/is (= "~:not-transit" (:content-type decoded))) + (t/is (= "tempfile" (:bucket decoded))))) diff --git a/backend/test/backend_tests/storage_test.clj b/backend/test/backend_tests/storage_test.clj index 54953620a0..abdd8c1503 100644 --- a/backend/test/backend_tests/storage_test.clj +++ b/backend/test/backend_tests/storage_test.clj @@ -9,12 +9,16 @@ [app.common.exceptions :as ex] [app.common.time :as ct] [app.common.uuid :as uuid] + [app.config :as cf] [app.db :as db] + [app.metrics :as mtx] + [app.metrics.definition :as-alias mdef] [app.rpc :as-alias rpc] [app.storage :as sto] [app.storage.fs :as-alias sto.fs] [app.storage.impl :as impl] [app.storage.s3 :as-alias sto.s3] + [app.storage.schema :as stsch] [backend-tests.helpers :as th] [clojure.test :as t] [cuerdas.core :as str] @@ -23,6 +27,11 @@ [mockery.core :refer [with-mocks]] [promesa.core :as p]) (:import + (io.prometheus.client + Counter + Counter$Child) + (org.postgresql.util + PGobject) (software.amazon.awssdk.services.s3 S3AsyncClient) (software.amazon.awssdk.services.s3.model @@ -41,24 +50,51 @@ [storage] (assoc storage ::sto/backend :fs)) +(defn- poison-counter-value + "Read the process-wide gc-poison counter from the test system metrics." + [] + (let [metrics (:app.metrics/metrics th/*system*) + collector (mtx/get-collector metrics :storage-gc-poison) + instance (::mdef/instance collector)] + (.get ^Counter$Child (.labels ^Counter instance (into-array String []))))) + (t/deftest put-and-retrieve-object (let [storage (-> (:app.storage/storage th/*system*) (configure-storage-backend)) content (sto/content "content") object (sto/put-object! storage {::sto/content content - :content-type "text/plain" - :other "data"})] + :content-type "text/plain"})] (t/is (sto/object? object)) (t/is (fs/path? (sto/get-object-path storage object))) (t/is (nil? (:expired-at object))) (t/is (= :fs (:backend object))) - (t/is (= "data" (:other (meta object)))) + (t/is (= "file-media-object" (:bucket (meta object)))) (t/is (= "text/plain" (:content-type (meta object)))) (t/is (= "content" (slurp (sto/get-object-data storage object)))) (t/is (= "content" (slurp (sto/get-object-path storage object)))))) +(t/deftest put-and-retrieve-with-json-metadata-flag + ;; End-to-end through the jsonb column: with the flag on, the row is + ;; stored as plain JSON and still decodes to native types. + (binding [cf/config (assoc cf/config :storage-metadata-as-json true)] + (let [storage (-> (:app.storage/storage th/*system*) + (configure-storage-backend)) + profile (uuid/random) + object (sto/put-object! storage {::sto/content (sto/content "content") + :bucket "tempfile" + :content-type "application/zip" + :profile-id profile}) + row (th/db-exec-one! ["select metadata::text as metadata from storage_object where id = ?" + (:id object)])] + (t/is (not (str/includes? (:metadata row) "\"~:"))) + (t/is (str/includes? (:metadata row) "\"bucket\"")) + (let [loaded (sto/get-object storage (:id object))] + (t/is (= "tempfile" (:bucket (meta loaded)))) + (t/is (= "application/zip" (:content-type (meta loaded)))) + (t/is (= profile (:profile-id (meta loaded)))))))) + (t/deftest tempfile-objects-are-not-deduplicated (let [storage (-> (:app.storage/storage th/*system*) (configure-storage-backend)) @@ -94,8 +130,7 @@ (configure-storage-backend)) content (sto/content "content") object (sto/put-object! storage {::sto/content content - :content-type "text/plain" - :expired-at (ct/in-future {:seconds 1})})] + :content-type "text/plain"})] (t/is (sto/object? object)) (t/is (true? (sto/del-object! storage (:id object)))) @@ -109,6 +144,18 @@ ;; marked as deleted/expired. (t/is (nil? (sto/get-object storage (:id object)))))) +(t/deftest delete-expired-object-returns-false + ;; An object stored with `::sto/expired-at` is born with `deleted_at` + ;; set, so deleting it is a no-op and must report `false`. + (let [storage (-> (:app.storage/storage th/*system*) + (configure-storage-backend)) + content (sto/content "content") + object (sto/put-object! storage {::sto/content content + :content-type "text/plain" + ::sto/expired-at (ct/in-future {:hours 1})})] + (t/is (some? (:expired-at object))) + (t/is (false? (sto/del-object! storage (:id object)))))) + (t/deftest deleted-gc-task (let [storage (-> (:app.storage/storage th/*system*) (configure-storage-backend)) @@ -438,6 +485,146 @@ (let [row (th/db-exec-one! ["select deleted_at from storage_object where id = ?" (:id object1)])] (t/is (ct/is-before-or-equal? (:deleted-at row) (ct/plus now {:seconds 1})))))) +(t/deftest storage-gc-touched-null-metadata + ;; A NULL metadata column (predates any normalization) flows through + ;; the lookup fallback with a warning instead of breaking the GC loop. + (let [storage (-> (:app.storage/storage th/*system*) + (configure-storage-backend)) + content (sto/content "content") + object (sto/put-object! storage {::sto/content content + ::sto/touch true + :content-type "text/plain"})] + (th/db-exec! ["update storage_object set metadata = null where id = ?" (:id object)]) + (let [res (th/run-task! :storage-gc-touched {:skip-delay true})] + (t/is (= 0 (:freeze res))) + (t/is (= 1 (:delete res)))))) + +(t/deftest storage-gc-touched-defers-corrupt-metadata + ;; A non-map metadata row neither blocks the chunk nor gets + ;; collected: it is logged and deferred exactly one day, keeping its + ;; metadata intact for a later repair. + (let [now (ct/now) + before (poison-counter-value) + storage (-> (:app.storage/storage th/*system*) + (configure-storage-backend)) + healthy (sto/put-object! storage {::sto/content (sto/content "healthy") + ::sto/touched-at now + :content-type "text/plain"}) + poison (uuid/random)] + (th/db-exec! ["insert into storage_object (id, backend, metadata, touched_at) values (?, 'fs', '[]'::jsonb, ?)" + poison now]) + (binding [ct/*clock* (ct/fixed-clock now)] + (let [res (th/run-task! :storage-gc-touched {:skip-delay true})] + (t/is (= 0 (:freeze res))) + (t/is (= 1 (:delete res))))) + (let [row (th/db-exec-one! ["select metadata::text as metadata, touched_at from storage_object where id = ?" poison])] + (t/is (= "[]" (:metadata row))) + ;; inst-ms: timestamptz keeps micros, the frozen clock has nanos. + (t/is (= (inst-ms (ct/plus now {:days 1})) + (inst-ms (:touched-at row))))) + ;; One poison row observed, one increment. + (t/is (= (inc before) (poison-counter-value))))) + +(t/deftest storage-gc-touched-poison-only + ;; A chunk made only of poison rows still terminates: the rows are + ;; deferred, the next scan returns nothing, and no delete is counted. + (let [now (ct/now) + poison (uuid/random)] + (th/db-exec! ["insert into storage_object (id, backend, metadata, touched_at) values (?, 'fs', '[]'::jsonb, ?)" + poison now]) + (binding [ct/*clock* (ct/fixed-clock now)] + (let [res (th/run-task! :storage-gc-touched {:skip-delay true})] + (t/is (= 0 (:freeze res))) + (t/is (= 0 (:delete res))))) + (let [row (th/db-exec-one! ["select touched_at from storage_object where id = ?" poison])] + (t/is (= (inst-ms (ct/plus now {:days 1})) + (inst-ms (:touched-at row))))))) + +(t/deftest storage-gc-touched-drains-past-a-poison-only-batch + ;; A full batch of poison rows must not stop the run: they are deferred + ;; and the loop keeps draining, so a healthy row queued behind the + ;; LIMIT (later touched_at) still gets collected. + (let [now (ct/now) + storage (-> (:app.storage/storage th/*system*) + (configure-storage-backend)) + earlier (ct/minus now {:seconds 1}) + healthy (sto/put-object! storage {::sto/content (sto/content "healthy") + ::sto/touched-at now + :content-type "text/plain"}) + poisons (repeatedly 10 uuid/random)] + (doseq [id poisons] + (th/db-exec! ["insert into storage_object (id, backend, metadata, touched_at) values (?, 'fs', '[]'::jsonb, ?)" + id earlier])) + (binding [ct/*clock* (ct/fixed-clock now)] + (let [res (th/run-task! :storage-gc-touched {:skip-delay true})] + (t/is (= 0 (:freeze res))) + (t/is (= 1 (:delete res))))) + (doseq [id poisons] + (let [row (th/db-exec-one! ["select touched_at from storage_object where id = ?" id])] + (t/is (= (inst-ms (ct/plus now {:days 1})) + (inst-ms (:touched-at row)))))))) + +(def ^:private migration-0155-fixtures + ;; [id transit-metadata]: production-shaped legacy rows (nil payload + ;; means a NULL column). + [["11111111-1111-1111-1111-111111111111" + "{\"~:reference\":\"~:file-media-object\",\"~:content-type\":\"image/png\",\"~:hash\":\"blake2b:aaa\"}"] + ["22222222-2222-2222-2222-222222222222" + "{\"~:content-type\":\"image/svg+xml\"}"] + ["33333333-3333-3333-3333-333333333333" + "{\"~:bucket\":\"tempfile\",\"~:reference\":\"~:tempfile\",\"~:content-type\":\"application/zip\"}"] + ["44444444-4444-4444-4444-444444444444" + "{\"~:bucket\":\"tempfile\",\"~:content-type\":\"application/zip\",\"~:upload-id\":\"~u86907e95-1cb8-8122-8008-4eb7ba07d89d\",\"~:chunk-index\":3}"] + ["55555555-5555-5555-5555-555555555555" + nil]]) + +(defn- run-migration-0155! + ;; Re-runs the 0155 statements (not migratus: it already applied at + ;; bootstrap) over the fixture rows above. + [] + (let [sql (-> (io/resource "app/migrations/sql/0155-normalize-storage-object-metadata.sql") + (slurp)) + no-comments (->> (.split ^String sql "\n") + (remove #(.startsWith ^String (str/trim %) "--")) + (str/join "\n"))] + (doseq [stmt (->> (.split ^String no-comments ";") + (map str/trim) + (remove str/blank?))] + (th/db-exec! [stmt])))) + +(defn- get-metadata-by-id + [id] + (:metadata (th/db-exec-one! ["select metadata from storage_object where id = ?" + (parse-uuid id)]))) + +(t/deftest storage-migration-0155-normalizes-legacy-rows + (doseq [[id mdata] migration-0155-fixtures] + (th/db-exec! ["insert into storage_object (id, backend, metadata) values (?, 'fs', ?::jsonb)" + (parse-uuid id) mdata])) + (run-migration-0155!) + (let [mdata (fn [id] (stsch/decode-metadata (get-metadata-by-id id)))] + (t/is (= {:bucket "file-media-object" + :content-type "image/png" + :hash "blake2b:aaa"} + (mdata "11111111-1111-1111-1111-111111111111"))) + (t/is (= {:bucket "file-media-object" + :content-type "image/svg+xml"} + (mdata "22222222-2222-2222-2222-222222222222"))) + (t/is (= {:bucket "tempfile" + :content-type "application/zip"} + (mdata "33333333-3333-3333-3333-333333333333"))) + (t/is (= {:bucket "tempfile" + :content-type "application/zip"} + (mdata "44444444-4444-4444-4444-444444444444"))) + (t/is (= {:bucket "file-media-object"} + (mdata "55555555-5555-5555-5555-555555555555")))) + ;; second run changes nothing (idempotent) + (let [raw (fn [] (mapv #(.getValue ^PGobject (get-metadata-by-id %)) + (map first migration-0155-fixtures))) + before (raw)] + (run-migration-0155!) + (t/is (= before (raw))))) + (t/deftest storage-gc-deleted-immediate (let [storage (-> (:app.storage/storage th/*system*) (configure-storage-backend)) @@ -626,6 +813,43 @@ (let [row (th/db-exec-one! ["select count(*) from storage_object"])] (t/is (= 1 (:count row)))))) +(t/deftest dedup-reuses-json-encoded-blob + ;; With the JSON flag on, the dedup lookup must find the row it just + ;; wrote; a "~:"-only lookup used to miss it and duplicate the blob. + (binding [cf/config (assoc cf/config :storage-metadata-as-json true)] + (let [storage (-> (:app.storage/storage th/*system*) + (configure-storage-backend)) + content (-> (sto/content "json-content") + (sto/wrap-with-hash "json-hash")) + params {::sto/content content + ::sto/deduplicate? true + :bucket "file-media-object" + :content-type "text/plain"} + object1 (sto/put-object! storage params) + object2 (sto/put-object! storage params)] + (t/is (= (:id object1) (:id object2))) + (let [row (th/db-exec-one! ["select count(*) from storage_object"])] + (t/is (= 1 (:count row))))))) + +(t/deftest dedup-json-put-finds-transit-row + ;; Both encodings coexist during the transition: a JSON write must + ;; reuse a blob already stored as Transit. + (let [storage (-> (:app.storage/storage th/*system*) + (configure-storage-backend)) + content (-> (sto/content "mixed-content") + (sto/wrap-with-hash "mixed-hash")) + params {::sto/content content + ::sto/deduplicate? true + :bucket "file-media-object" + :content-type "text/plain"} + object1 (binding [cf/config (assoc cf/config :storage-metadata-as-json nil)] + (sto/put-object! storage params)) + object2 (binding [cf/config (assoc cf/config :storage-metadata-as-json true)] + (sto/put-object! storage params))] + (t/is (= (:id object1) (:id object2))) + (let [row (th/db-exec-one! ["select count(*) from storage_object"])] + (t/is (= 1 (:count row)))))) + (t/deftest dedup-repairs-stale-object (let [storage (-> (:app.storage/storage th/*system*) (configure-storage-backend)) diff --git a/docs/technical-guide/configuration.md b/docs/technical-guide/configuration.md index 9fd0282761..d730abd8fc 100644 --- a/docs/technical-guide/configuration.md +++ b/docs/technical-guide/configuration.md @@ -643,6 +643,44 @@ And configure it: PENPOT_SECRET_KEY: my-super-secure-key ``` +### Session expiration + +__Since version 2.19.0__ + +User sessions are stored server-side and expire on two independent conditions: an +**idle timeout** and an **absolute maximum lifetime**. Both are backend only. + +```bash +# Backend +# Idle timeout: the session stops working after this much inactivity. +# Default: 168h +PENPOT_AUTH_TOKEN_COOKIE_MAX_AGE: 168h + +# Absolute maximum lifetime from the moment the session was created, +# regardless of activity. Default: 720h +PENPOT_AUTH_TOKEN_COOKIE_MAX_AGE_ABSOLUTE: 720h +``` + +Durations use the `` form with hour, minute or second units, +for example `168h`, `30m` or `90s` (day units like `7d` are not accepted). + +While a user is active the session is automatically renewed every 6 hours (not +configurable). Renewal extends the cookie, but never the absolute maximum. The +backend checks the idle timeout on every request, measured from the last +renewal: once it passes, the session stops working at once, even for a copied +token. A daily task (`session-gc`) then deletes the sessions that have exceeded +either window. Legacy v1 sessions and the old `http_session` table are no +longer used. + +Sessions created before 2.19.0 carry no `:exp` in their token; they are still +removed by the 30-day `created_at` cleanup and acquire `:exp` on their next +renewal. + +The `secure` and `same-site` attributes of the session cookie are controlled by +the `disable-secure-session-cookies`, `strict-session-cookies` and `enable-cors` +flags, and by whether `PENPOT_PUBLIC_URI` is served over HTTPS. See +[Penpot URI](#penpot-uri). + ### Database Penpot only supports PostgreSQL and we highly recommend >=13 version. If you are using official diff --git a/docs/technical-guide/developer/subsystems/authentication.md b/docs/technical-guide/developer/subsystems/authentication.md index da437194ee..b5946696ac 100644 --- a/docs/technical-guide/developer/subsystems/authentication.md +++ b/docs/technical-guide/developer/subsystems/authentication.md @@ -94,14 +94,45 @@ Similarly as the OIDC backend, it checks if the profile exists, and calls ## Sessions -User sessions are created when a user logs in via any one of the backends. A -session token is generated (a JWT token that does not currently contain any data) -and returned to frontend as a cookie. +User sessions are created when a user logs in via any one of the backends. The +backend generates a signed JWT token and returns it to the frontend as an +auth-token cookie. A matching row is stored in +the http_session_v2 table with the profile id +and the session timestamps. -Normally the session is stored in a DB table with the information of the user -profile and the session expiration. But if a frontend connects to the backend in -"read only" mode (for example, to debug something in production with the local -devenv), sessions are stored in memory (may be lost if the backend restarts). +A request is authenticated only when the token verifies, its session row still +exists and the session is within the idle timeout. The token claims carry the session row id (:sid), the last activity instant (:iat) and an absolute expiration (:exp). The server enforces two independent +limits: + +* **Idle timeout:** a session that is not renewed within + PENPOT_AUTH_TOKEN_COOKIE_MAX_AGE (default 7 + days) stops working at once. The backend checks the session's last renewal + time on every request, so the row does not need to be deleted first. +* **Absolute maximum:** a session cannot live longer than + PENPOT_AUTH_TOKEN_COOKIE_MAX_AGE_ABSOLUTE + (default 30 days) from its creation, no matter how much it is renewed. The + :exp claim enforces this even when the + cookie is still present. + +Sessions are automatically renewed every 6 hours of use (not configurable). +Renewal issues a new token but keeps the same session row, so the absolute +maximum is not extended. A daily garbage collector +(session-gc) deletes rows that exceed either +the idle window or the absolute maximum. + +Sessions created before 2.19.0 carry no :exp in their token; they are still removed by +the 30-day created_at cleanup and acquire :exp on their next renewal. + +The normal storage is the database. When the backend uses a read-only database +pool (for example, to debug something in production with the local devenv), +sessions are kept in memory and are lost when the backend restarts. The +organization SSO gate keeps an additional 4-hour entry inside the same session +row, separate from the session token lifetime. ## Team invitations diff --git a/frontend/src/app/main.cljs b/frontend/src/app/main.cljs index b6fbfb3ca5..c7d87ba8ba 100644 --- a/frontend/src/app/main.cljs +++ b/frontend/src/app/main.cljs @@ -54,8 +54,12 @@ (let [el (dom/get-element "app")] (mf/create-root el))) +;; Root component of the last render; a hot reload of `app.main.ui` replaces it +(defonce ^:private rendered-app (atom nil)) + (defn init-ui [] + (reset! rendered-app ui/app) (mf/render! app-root (mf/element ui/app))) (defn- initialize-rasterizer @@ -146,7 +150,10 @@ (init-ui))) (defn ^:dev/after-load after-load + "Rerender the UI after a hot reload. When the reload replaced the root + component, unmount the whole tree first so its cleanups run before + the new tree mounts." [] - (reinit)) + (reinit (not (identical? @rendered-app ui/app)))) (set! (.-stackTraceLimit js/Error) 50) diff --git a/frontend/src/app/plugins/library.cljs b/frontend/src/app/plugins/library.cljs index 998f8b0b5c..56782eb1e2 100644 --- a/frontend/src/app/plugins/library.cljs +++ b/frontend/src/app/plugins/library.cljs @@ -1248,15 +1248,26 @@ (not (r/check-permission plugin-id "library:write")) (u/reject-not-valid reject :connectLibrary "Plugin doesn't have 'library:write' permission") - (not (string? library-id)) + (not (uuid/parse* library-id)) (u/reject-not-valid reject :connectLibrary library-id) :else - (let [file-id (:current-file-id @st/state) + (let [file-id (:current-file-id @st/state) + team-id (:current-team-id @st/state) library-id (uuid/parse library-id)] - (->> st/stream - (rx/filter (ptk/type? ::dwl/link-file-to-library-finished)) - (rx/take 1) - (rx/subs! #(resolve (library-proxy plugin-id library-id)) reject)) - (st/emit! (-> (dwl/link-file-to-library file-id library-id) - (se/add-event plugin-id)))))))))) + (->> (rp/cmd! :get-team-shared-files {:team-id team-id}) + (rx/subs! + (fn [shared-files] + ;; Only published libraries of the team can be linked + (if (or (= library-id file-id) + (not (some #(= library-id (:id %)) shared-files))) + (u/reject-not-valid reject :connectLibrary + (dm/str library-id " is not a published library")) + (do + (->> st/stream + (rx/filter (ptk/type? ::dwl/link-file-to-library-finished)) + (rx/take 1) + (rx/subs! #(resolve (library-proxy plugin-id library-id)) reject)) + (st/emit! (-> (dwl/link-file-to-library file-id library-id) + (se/add-event plugin-id)))))) + reject))))))))) diff --git a/frontend/src/app/render_wasm/api.cljs b/frontend/src/app/render_wasm/api.cljs index dd5d75f4da..d9340bc5c5 100644 --- a/frontend/src/app/render_wasm/api.cljs +++ b/frontend/src/app/render_wasm/api.cljs @@ -2479,6 +2479,13 @@ :cause cause) nil))))) +;; Stable listener identities, so `removeEventListener` matches across hot reloads +(defonce ^:private webgl-context-lost-listener + (fn [event] (on-webgl-context-lost event))) + +(defonce ^:private webgl-context-restored-listener + (fn [event] (on-webgl-context-restored event))) + (defn init-canvas-context [canvas] (if-not (wasm/module-ready?) @@ -2523,8 +2530,8 @@ ;; Add event listeners for WebGL context lost (set! wasm/canvas canvas) (when can-listen? - (.addEventListener canvas "webglcontextlost" on-webgl-context-lost) - (.addEventListener canvas "webglcontextrestored" on-webgl-context-restored)) + (.addEventListener canvas "webglcontextlost" webgl-context-lost-listener) + (.addEventListener canvas "webglcontextrestored" webgl-context-restored-listener)) (start-canvas-snapshot-listener!) (reset! wasm/context-lost? false) (set! wasm/context-initialized? true) @@ -2558,8 +2565,8 @@ ;; Remove listener before losing/deleting context. (when wasm/canvas - (.removeEventListener wasm/canvas "webglcontextlost" on-webgl-context-lost) - (.removeEventListener wasm/canvas "webglcontextrestored" on-webgl-context-restored)) + (.removeEventListener wasm/canvas "webglcontextlost" webgl-context-lost-listener) + (.removeEventListener wasm/canvas "webglcontextrestored" webgl-context-restored-listener)) (stop-canvas-snapshot-listener!) (when (wasm/module-ready?) diff --git a/frontend/test/frontend_tests/plugins/library_test.cljs b/frontend/test/frontend_tests/plugins/library_test.cljs index d951cddb05..70d3e3fb7d 100644 --- a/frontend/test/frontend_tests/plugins/library_test.cljs +++ b/frontend/test/frontend_tests/plugins/library_test.cljs @@ -8,16 +8,23 @@ (:require [app.common.types.component :as ctk] [app.common.uuid :as uuid] + [app.main.data.event :as ev] [app.main.data.workspace.libraries :as dwl] [app.main.data.workspace.texts :as dwt] [app.main.data.workspace.variants :as dwv] + [app.main.repo :as rp] [app.main.store :as st] [app.plugins.library :as library] [app.plugins.register :as r] [app.plugins.text :as text] [app.plugins.utils :as u] + [app.util.object :as obj] + [beicon.v2.core :as rx] [cljs.test :as t :include-macros true] - [frontend-tests.helpers.mock :as mock])) + [cuerdas.core :as str] + [frontend-tests.helpers.async :as async] + [frontend-tests.helpers.mock :as mock] + [potok.v2.core :as ptk])) (def ^:private plugin-id "00000000-0000-0000-0000-000000000000") @@ -205,3 +212,103 @@ (t/is (= 1 (count @errors))) (t/is (= [plugin-id :setVariantProperty "Plugin doesn't have 'library:write' permission"] (first @errors))))))) + +;; --------------------------------------------------------------------------- +;; connectLibrary +;; --------------------------------------------------------------------------- + +(defn- shared-files-cmd + "Answers every RPC command asynchronously with `shared-files`, recording + the calls in `mock/rpc-calls`." + [shared-files] + (mock/stub + (fn [cmd params] + (swap! mock/rpc-calls conj {:cmd cmd :params params}) + (rx/observe-on :async (rx/of shared-files))))) + +(defn- settled + "Resolves with `{:resolved value}` or `{:rejected cause}` once `p` settles, + or with `{:timeout true}` when it stays pending for 2 seconds." + [p] + (js/Promise.race + #js [(.then p (fn [v] {:resolved v}) (fn [e] {:rejected e})) + (js/Promise. (fn [resolve] (js/setTimeout #(resolve {:timeout true}) 2000)))])) + +(t/deftest ^:async connect-library-links-a-published-library + (let [file-id (uuid/next) + team-id (uuid/next) + library-id (uuid/next) + stream (rx/subject) + events (atom [])] + (await + (mock/with-mocks* + {st/state (atom {:current-file-id file-id :current-team-id team-id}) + st/stream stream + st/emit! (mock/stub (fn [& emitted] (swap! events into emitted))) + rp/cmd! (shared-files-cmd [{:id library-id :name "Published"}]) + r/check-permission (mock/stub (constantly true))} + (let [context (library/library-subcontext plugin-id) + result (settled (.connectLibrary context (str library-id))) + link? #(= ::dwl/link-file-to-library (ptk/type %))] + (await (async/wait-for #(some link? @events) "link event emitted")) + (rx/push! stream (ptk/data-event ::dwl/link-file-to-library-finished)) + (let [{:keys [resolved]} (await result) + link-data (ev/-data (first (filter link? @events)))] + (t/is (= library-id (obj/get resolved "$id"))) + (t/is (= file-id (:file-id link-data))) + (t/is (= library-id (:library-id link-data))) + (t/is (= [{:cmd :get-team-shared-files :params {:team-id team-id}}] + @mock/rpc-calls)))))))) + +(t/deftest ^:async connect-library-rejects-an-unpublished-file + (let [file-id (uuid/next) + library-id (uuid/next) + events (atom [])] + (await + (mock/with-mocks* + {st/state (atom {:current-file-id file-id :current-team-id (uuid/next)}) + st/emit! (mock/stub (fn [& emitted] (swap! events into emitted))) + rp/cmd! (shared-files-cmd [{:id (uuid/next) :name "Other published"}]) + r/check-permission (mock/stub (constantly true))} + (let [context (library/library-subcontext plugin-id) + {:keys [rejected]} (await (settled (.connectLibrary context (str library-id))))] + (t/is (str/includes? (str rejected) "is not a published library")) + (t/is (empty? @events))))))) + +(t/deftest ^:async connect-library-rejects-the-current-file + (let [file-id (uuid/next) + events (atom [])] + (await + (mock/with-mocks* + {st/state (atom {:current-file-id file-id :current-team-id (uuid/next)}) + st/emit! (mock/stub (fn [& emitted] (swap! events into emitted))) + rp/cmd! (shared-files-cmd [{:id file-id :name "Current file"}]) + r/check-permission (mock/stub (constantly true))} + (let [context (library/library-subcontext plugin-id) + {:keys [rejected]} (await (settled (.connectLibrary context (str file-id))))] + (t/is (str/includes? (str rejected) "is not a published library")) + (t/is (empty? @events))))))) + +(t/deftest ^:async connect-library-rejects-an-invalid-id + (await + (mock/with-mocks* + {st/state (atom {:current-file-id (uuid/next) :current-team-id (uuid/next)}) + st/emit! mock/noop + rp/cmd! (shared-files-cmd []) + r/check-permission (mock/stub (constantly true))} + (let [context (library/library-subcontext plugin-id) + {:keys [rejected]} (await (settled (.connectLibrary context "not-a-uuid")))] + (t/is (str/includes? (str rejected) "not-a-uuid")) + (t/is (empty? @mock/rpc-calls)))))) + +(t/deftest ^:async connect-library-checks-permission + (await + (mock/with-mocks* + {st/state (atom {:current-file-id (uuid/next) :current-team-id (uuid/next)}) + st/emit! mock/noop + rp/cmd! (shared-files-cmd []) + r/check-permission (mock/stub (constantly false))} + (let [context (library/library-subcontext plugin-id) + {:keys [rejected]} (await (settled (.connectLibrary context (str (uuid/next)))))] + (t/is (str/includes? (str rejected) "Plugin doesn't have 'library:write' permission")) + (t/is (empty? @mock/rpc-calls))))))