Merge remote-tracking branch 'origin/staging' into develop

This commit is contained in:
Andrey Antukh 2026-10-01 10:59:50 +02:00
commit c561a0baff
58 changed files with 1794 additions and 375 deletions

16
.github/dependabot.yml vendored Normal file
View File

@ -0,0 +1,16 @@
# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
version: 2
updates:
# Actions are pinned to a commit SHA with a `# vX.Y.Z` comment; Dependabot
# bumps both. All updates are grouped into a single monthly PR.
- package-ecosystem: "github-actions"
directory: "/"
schedule:
interval: "monthly"
open-pull-requests-limit: 5
commit-message:
prefix: ":arrow_up:"
groups:
github-actions:
patterns:
- "*"

View File

@ -12,14 +12,14 @@ jobs:
steps:
- name: Generate GitHub App token
id: triage-app-token
uses: actions/create-github-app-token@v1
uses: actions/create-github-app-token@d72941d797fd3113feb6b93fd0dec494b13a2547 # v1.12.0
with:
app-id: ${{ secrets.TRIAGE_APP_ID }}
private-key: ${{ secrets.TRIAGE_APP_PRIVATE_KEY }}
owner: penpot
- name: Process Issue or PR
uses: actions/github-script@v7
uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0
with:
github-token: ${{ steps.triage-app-token.outputs.token }}
script: |

View File

@ -23,7 +23,12 @@ on:
jobs:
build-bundle:
uses: ./.github/workflows/build-bundle.yml
secrets: inherit
secrets:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_REGION: ${{ secrets.AWS_REGION }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }}
S3_BUCKET: ${{ secrets.S3_BUCKET }}
with:
gh_ref: ${{ inputs.gh_ref }}
force: ${{ inputs.force }}
@ -31,7 +36,17 @@ jobs:
build-docker:
needs: build-bundle
uses: ./.github/workflows/build-docker.yml
secrets: inherit
secrets:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_REGION: ${{ secrets.AWS_REGION }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
DOCKER_REGISTRY: ${{ secrets.DOCKER_REGISTRY }}
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }}
PUB_DOCKER_PASSWORD: ${{ secrets.PUB_DOCKER_PASSWORD }}
PUB_DOCKER_USERNAME: ${{ secrets.PUB_DOCKER_USERNAME }}
S3_BUCKET: ${{ secrets.S3_BUCKET }}
with:
gh_ref: ${{ inputs.gh_ref }}
# Pin build-docker to the commit build-bundle actually bundled.
@ -40,7 +55,8 @@ jobs:
build-docker-admin-console:
uses: ./.github/workflows/build-docker-admin-console.yml
secrets: inherit
secrets:
ORG_WORKFLOW_TOKEN: ${{ secrets.ORG_WORKFLOW_TOKEN }}
with:
gh_ref: ${{ inputs.nitrate_ref || inputs.gh_ref }}
force: ${{ inputs.force }}

View File

@ -29,6 +29,17 @@ on:
sha:
description: 'Bundled commit, full 40-char SHA.'
value: ${{ jobs.check.outputs.sha }}
secrets:
AWS_ACCESS_KEY_ID:
required: false
AWS_REGION:
required: false
AWS_SECRET_ACCESS_KEY:
required: false
MATTERMOST_WEBHOOK:
required: false
S3_BUCKET:
required: false
# Literal group name: under `workflow_call`, `github.workflow` resolves to the
# caller's workflow, which put this workflow and the other reusable one called
@ -54,8 +65,9 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@v6
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
with:
persist-credentials: false
fetch-depth: 0
ref: ${{ inputs.gh_ref }}
@ -121,8 +133,9 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@v6
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
with:
persist-credentials: false
fetch-depth: 0
ref: ${{ inputs.gh_ref }}

View File

@ -21,7 +21,12 @@ concurrency:
jobs:
build-bundle:
uses: ./.github/workflows/build-bundle.yml
secrets: inherit
secrets:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_REGION: ${{ secrets.AWS_REGION }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }}
S3_BUCKET: ${{ secrets.S3_BUCKET }}
with:
gh_ref: "develop"
force: ${{ inputs.force || false }}
@ -29,7 +34,17 @@ jobs:
build-docker:
needs: build-bundle
uses: ./.github/workflows/build-docker.yml
secrets: inherit
secrets:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_REGION: ${{ secrets.AWS_REGION }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
DOCKER_REGISTRY: ${{ secrets.DOCKER_REGISTRY }}
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }}
PUB_DOCKER_PASSWORD: ${{ secrets.PUB_DOCKER_PASSWORD }}
PUB_DOCKER_USERNAME: ${{ secrets.PUB_DOCKER_USERNAME }}
S3_BUCKET: ${{ secrets.S3_BUCKET }}
with:
gh_ref: "develop"
# Pin build-docker to the commit build-bundle actually bundled.
@ -38,7 +53,8 @@ jobs:
build-docker-admin-console:
uses: ./.github/workflows/build-docker-admin-console.yml
secrets: inherit
secrets:
ORG_WORKFLOW_TOKEN: ${{ secrets.ORG_WORKFLOW_TOKEN }}
with:
gh_ref: "develop"
force: ${{ inputs.force || false }}

View File

@ -15,26 +15,28 @@ jobs:
echo "DOCKER_CONFIG=${{ runner.temp }}/.docker-${{ github.run_id }}-${{ github.job }}" >> $GITHUB_ENV
- name: Checkout code
uses: actions/checkout@v6
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
with:
persist-credentials: false
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1
- name: Login to Docker Registry (push destination)
uses: docker/login-action@v4
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
with:
username: ${{ secrets.PUB_DOCKER_USERNAME }}
password: ${{ secrets.PUB_DOCKER_PASSWORD }}
- name: Login to Docker Hardened Images registry (base image pull)
uses: docker/login-action@v4
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
with:
registry: dhi.io
username: ${{ secrets.PUB_DOCKER_USERNAME }}
password: ${{ secrets.PUB_DOCKER_PASSWORD }}
- name: Build and push DevEnv Docker image
uses: docker/build-push-action@v7
uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0
env:
DOCKER_IMAGE: 'penpotapp/devenv'
with:

View File

@ -32,6 +32,27 @@ on:
type: boolean
required: false
default: false
secrets:
AWS_ACCESS_KEY_ID:
required: false
AWS_REGION:
required: false
AWS_SECRET_ACCESS_KEY:
required: false
DOCKER_PASSWORD:
required: false
DOCKER_REGISTRY:
required: false
DOCKER_USERNAME:
required: false
MATTERMOST_WEBHOOK:
required: false
PUB_DOCKER_PASSWORD:
required: false
PUB_DOCKER_USERNAME:
required: false
S3_BUCKET:
required: false
# Literal group name: under `workflow_call`, `github.workflow` resolves to the
# caller's workflow, which put this workflow and the other reusable one called
@ -65,8 +86,9 @@ jobs:
steps:
- name: Checkout code
uses: actions/checkout@v6
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
with:
persist-credentials: false
fetch-depth: 0
ref: ${{ inputs.sha != '' && inputs.sha || inputs.gh_ref }}
@ -159,12 +181,13 @@ jobs:
echo "DOCKER_CONFIG=${{ runner.temp }}/.docker-${{ github.run_id }}-${{ github.job }}-${{ matrix.image }}" >> $GITHUB_ENV
- name: Checkout code
uses: actions/checkout@v6
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
with:
persist-credentials: false
ref: ${{ inputs.sha != '' && inputs.sha || inputs.gh_ref }}
- name: Login to Docker Registry
uses: docker/login-action@v4
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
with:
registry: ${{ secrets.DOCKER_REGISTRY }}
username: ${{ secrets.DOCKER_USERNAME }}
@ -174,7 +197,7 @@ jobs:
# images from DockerHub for unregistered users.
# https://docs.docker.com/docker-hub/usage/
- name: Login to DockerHub Registry
uses: docker/login-action@v4
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
with:
username: ${{ secrets.PUB_DOCKER_USERNAME }}
password: ${{ secrets.PUB_DOCKER_PASSWORD }}
@ -186,7 +209,7 @@ jobs:
# reuses the same PUB_DOCKER_* credentials as the DockerHub
# login above.
- name: Login to Docker Hardened Images registry (base image pull)
uses: docker/login-action@v4
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
with:
registry: dhi.io
username: ${{ secrets.PUB_DOCKER_USERNAME }}
@ -218,23 +241,23 @@ jobs:
popd
- name: Set up QEMU (stable)
uses: docker/setup-qemu-action@v4
uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4.4.0
with:
platforms: linux/amd64,linux/arm64
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1
- name: Extract metadata (tags, labels)
id: meta
uses: docker/metadata-action@v6
uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0
with:
images: ${{ matrix.image }}
labels: |
bundle_version=${{ needs.prepare.outputs.bundle_version }}
- name: Build and push Docker image
uses: docker/build-push-action@v7
uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0
with:
context: ./docker/images/
file: ./docker/images/Dockerfile.${{ matrix.image }}
@ -262,10 +285,10 @@ jobs:
echo "DOCKER_CONFIG=${{ runner.temp }}/.docker-${{ github.run_id }}-${{ github.job }}" >> $GITHUB_ENV
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1
- name: Login to Docker Registry
uses: docker/login-action@v4
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
with:
registry: ${{ secrets.DOCKER_REGISTRY }}
username: ${{ secrets.DOCKER_USERNAME }}
@ -331,10 +354,10 @@ jobs:
echo "DOCKER_CONFIG=${{ runner.temp }}/.docker-${{ github.run_id }}-${{ github.job }}" >> $GITHUB_ENV
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1
- name: Login to Docker Registry
uses: docker/login-action@v4
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
with:
registry: ${{ secrets.DOCKER_REGISTRY }}
username: ${{ secrets.DOCKER_USERNAME }}

View File

@ -21,7 +21,12 @@ concurrency:
jobs:
build-bundle:
uses: ./.github/workflows/build-bundle.yml
secrets: inherit
secrets:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_REGION: ${{ secrets.AWS_REGION }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }}
S3_BUCKET: ${{ secrets.S3_BUCKET }}
with:
gh_ref: "staging"
force: ${{ inputs.force || false }}
@ -29,7 +34,17 @@ jobs:
build-docker:
needs: build-bundle
uses: ./.github/workflows/build-docker.yml
secrets: inherit
secrets:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_REGION: ${{ secrets.AWS_REGION }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
DOCKER_REGISTRY: ${{ secrets.DOCKER_REGISTRY }}
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }}
PUB_DOCKER_PASSWORD: ${{ secrets.PUB_DOCKER_PASSWORD }}
PUB_DOCKER_USERNAME: ${{ secrets.PUB_DOCKER_USERNAME }}
S3_BUCKET: ${{ secrets.S3_BUCKET }}
with:
gh_ref: "staging"
# Pin build-docker to the commit build-bundle actually bundled.
@ -38,7 +53,8 @@ jobs:
build-docker-admin-console:
uses: ./.github/workflows/build-docker-admin-console.yml
secrets: inherit
secrets:
ORG_WORKFLOW_TOKEN: ${{ secrets.ORG_WORKFLOW_TOKEN }}
with:
gh_ref: "staging"
force: ${{ inputs.force || false }}

View File

@ -24,7 +24,12 @@ concurrency:
jobs:
build-bundle:
uses: ./.github/workflows/build-bundle.yml
secrets: inherit
secrets:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_REGION: ${{ secrets.AWS_REGION }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }}
S3_BUCKET: ${{ secrets.S3_BUCKET }}
with:
gh_ref: ${{ github.ref_name }}
force: ${{ inputs.force || false }}
@ -32,14 +37,25 @@ jobs:
build-docker:
needs: build-bundle
uses: ./.github/workflows/build-docker.yml
secrets: inherit
secrets:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_REGION: ${{ secrets.AWS_REGION }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
DOCKER_REGISTRY: ${{ secrets.DOCKER_REGISTRY }}
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }}
PUB_DOCKER_PASSWORD: ${{ secrets.PUB_DOCKER_PASSWORD }}
PUB_DOCKER_USERNAME: ${{ secrets.PUB_DOCKER_USERNAME }}
S3_BUCKET: ${{ secrets.S3_BUCKET }}
with:
gh_ref: ${{ github.ref_name }}
force: ${{ inputs.force || false }}
build-docker-admin-console:
uses: ./.github/workflows/build-docker-admin-console.yml
secrets: inherit
secrets:
ORG_WORKFLOW_TOKEN: ${{ secrets.ORG_WORKFLOW_TOKEN }}
with:
gh_ref: ${{ github.ref_name }}
force: ${{ inputs.force || false }}
@ -67,6 +83,13 @@ jobs:
- build-docker
- build-docker-admin-console
uses: ./.github/workflows/release.yml
secrets: inherit
secrets:
AWS_REGION: ${{ secrets.AWS_REGION }}
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
DOCKER_REGISTRY: ${{ secrets.DOCKER_REGISTRY }}
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }}
PUB_DOCKER_PASSWORD: ${{ secrets.PUB_DOCKER_PASSWORD }}
PUB_DOCKER_USERNAME: ${{ secrets.PUB_DOCKER_USERNAME }}
with:
gh_ref: ${{ github.ref_name }}

View File

@ -12,13 +12,28 @@ concurrency:
jobs:
build-bundle:
uses: ./.github/workflows/build-bundle.yml
secrets: inherit
secrets:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_REGION: ${{ secrets.AWS_REGION }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }}
S3_BUCKET: ${{ secrets.S3_BUCKET }}
with:
gh_ref: "hiru-tokens-in-libs"
build-docker:
needs: build-bundle
uses: ./.github/workflows/build-docker.yml
secrets: inherit
secrets:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_REGION: ${{ secrets.AWS_REGION }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
DOCKER_REGISTRY: ${{ secrets.DOCKER_REGISTRY }}
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }}
PUB_DOCKER_PASSWORD: ${{ secrets.PUB_DOCKER_PASSWORD }}
PUB_DOCKER_USERNAME: ${{ secrets.PUB_DOCKER_USERNAME }}
S3_BUCKET: ${{ secrets.S3_BUCKET }}
with:
gh_ref: "hiru-tokens-in-libs"

View File

@ -27,7 +27,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Check Commit Type
uses: gsactions/commit-message-checker@v2
uses: gsactions/commit-message-checker@16fa2d5de096ae0d35626443bcd24f1e756cafee # v2.0.0
with:
pattern: '^(((:(lipstick|globe_with_meridians|wrench|books|arrow_up|arrow_down|zap|ambulance|construction|boom|fire|whale|bug|sparkles|paperclip|tada|recycle|rewind|construction_worker):)\s[A-Z].*[^.])|(Merge|Revert|Reapply).+[^.])$'
flags: 'gm'

View File

@ -39,14 +39,15 @@ jobs:
echo "gh_ref=$GH_REF" >> $GITHUB_OUTPUT
- name: Checkout
uses: actions/checkout@v6
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
with:
persist-credentials: false
fetch-depth: 0
ref: ${{ steps.vars.outputs.gh_ref }}
# START: Setup Node and PNPM enabling cache
- name: Setup pnpm + Node.js
uses: pnpm/setup@v2
uses: pnpm/setup@84cb39b217b10273981911c288cd62326dc7c6d2 # v2.0.2
with:
working-directory: plugins
runtime: node@24.21.0
@ -59,7 +60,7 @@ jobs:
run: echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_OUTPUT
- name: Cache pnpm store
uses: actions/cache@v5
uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0
with:
path: ${{ steps.pnpm-store.outputs.STORE_PATH }}
key: ${{ runner.os }}-pnpm-${{ hashFiles('plugins/pnpm-lock.yaml') }}

View File

@ -27,6 +27,13 @@ on:
description: 'Publig name (from plugins/apps/<plugin_name>-plugin)'
type: string
required: true
secrets:
CLOUDFLARE_ACCOUNT_ID:
required: false
CLOUDFLARE_API_TOKEN:
required: false
MATTERMOST_WEBHOOK:
required: false
permissions:
contents: read
@ -58,8 +65,9 @@ jobs:
fi
- name: Checkout
uses: actions/checkout@v6
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
with:
persist-credentials: false
fetch-depth: 0
ref: ${{ inputs.gh_ref }}

View File

@ -36,9 +36,11 @@ jobs:
# [For new plugins]
# Add more outputs here
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
with:
persist-credentials: false
- id: filter
uses: dorny/paths-filter@v4
uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4.0.3
with:
filters: |
colors_to_tokens:
@ -72,7 +74,10 @@ jobs:
needs: detect-changes
if: github.event_name == 'workflow_dispatch' || needs.detect-changes.outputs.colors_to_tokens == 'true'
uses: ./.github/workflows/plugins-deploy-package.yml
secrets: inherit
secrets:
CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }}
with:
gh_ref: "${{ inputs.gh_ref || github.ref_name }}"
plugin_name: colors-to-tokens
@ -81,7 +86,10 @@ jobs:
needs: detect-changes
if: github.event_name == 'workflow_dispatch' || needs.detect-changes.outputs.contrast == 'true'
uses: ./.github/workflows/plugins-deploy-package.yml
secrets: inherit
secrets:
CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }}
with:
gh_ref: "${{ inputs.gh_ref || github.ref_name }}"
plugin_name: contrast
@ -90,7 +98,10 @@ jobs:
needs: detect-changes
if: github.event_name == 'workflow_dispatch' || needs.detect-changes.outputs.create_palette == 'true'
uses: ./.github/workflows/plugins-deploy-package.yml
secrets: inherit
secrets:
CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }}
with:
gh_ref: "${{ inputs.gh_ref || github.ref_name }}"
plugin_name: create-palette
@ -99,7 +110,10 @@ jobs:
needs: detect-changes
if: github.event_name == 'workflow_dispatch' || needs.detect-changes.outputs.icons == 'true'
uses: ./.github/workflows/plugins-deploy-package.yml
secrets: inherit
secrets:
CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }}
with:
gh_ref: "${{ inputs.gh_ref || github.ref_name }}"
plugin_name: icons
@ -108,7 +122,10 @@ jobs:
needs: detect-changes
if: github.event_name == 'workflow_dispatch' || needs.detect-changes.outputs.lorem_ipsum == 'true'
uses: ./.github/workflows/plugins-deploy-package.yml
secrets: inherit
secrets:
CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }}
with:
gh_ref: "${{ inputs.gh_ref || github.ref_name }}"
plugin_name: lorem-ipsum
@ -117,7 +134,10 @@ jobs:
needs: detect-changes
if: github.event_name == 'workflow_dispatch' || needs.detect-changes.outputs.rename_layers == 'true'
uses: ./.github/workflows/plugins-deploy-package.yml
secrets: inherit
secrets:
CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }}
with:
gh_ref: "${{ inputs.gh_ref || github.ref_name }}"
plugin_name: rename-layers
@ -126,7 +146,10 @@ jobs:
needs: detect-changes
if: github.event_name == 'workflow_dispatch' || needs.detect-changes.outputs.table == 'true'
uses: ./.github/workflows/plugins-deploy-package.yml
secrets: inherit
secrets:
CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }}
with:
gh_ref: "${{ inputs.gh_ref || github.ref_name }}"
plugin_name: table

View File

@ -37,14 +37,15 @@ jobs:
echo "gh_ref=$GH_REF" >> $GITHUB_OUTPUT
- name: Checkout
uses: actions/checkout@v6
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
with:
persist-credentials: false
fetch-depth: 0
ref: ${{ steps.vars.outputs.gh_ref }}
# START: Setup Node and PNPM enabling cache
- name: Setup pnpm + Node.js
uses: pnpm/setup@v2
uses: pnpm/setup@84cb39b217b10273981911c288cd62326dc7c6d2 # v2.0.2
with:
working-directory: plugins
runtime: node@24.21.0
@ -57,7 +58,7 @@ jobs:
run: echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_OUTPUT
- name: Cache pnpm store
uses: actions/cache@v5
uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0
with:
path: ${{ steps.pnpm-store.outputs.STORE_PATH }}
key: ${{ runner.os }}-pnpm-${{ hashFiles('plugins/pnpm-lock.yaml') }}

View File

@ -13,6 +13,21 @@ on:
description: 'Tag to release'
type: string
required: true
secrets:
AWS_REGION:
required: false
DOCKER_PASSWORD:
required: false
DOCKER_REGISTRY:
required: false
DOCKER_USERNAME:
required: false
MATTERMOST_WEBHOOK:
required: false
PUB_DOCKER_PASSWORD:
required: false
PUB_DOCKER_USERNAME:
required: false
permissions:
contents: write
@ -32,14 +47,15 @@ jobs:
echo "gh_ref=$GH_REF" >> $GITHUB_OUTPUT
- name: Checkout code
uses: actions/checkout@v6
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
with:
persist-credentials: false
fetch-depth: 0
ref: ${{ steps.vars.outputs.gh_ref }}
# --- Publicly release the docker images ---
- name: Configure ECR credentials
uses: aws-actions/configure-aws-credentials@v4
uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1
with:
aws-access-key-id: ${{ secrets.DOCKER_USERNAME }}
aws-secret-access-key: ${{ secrets.DOCKER_PASSWORD }}
@ -94,14 +110,19 @@ jobs:
echo "EOF" >> $GITHUB_OUTPUT
# --- Create GitHub release ---
# Updates the release if it already exists, so re-running is safe.
- name: Create GitHub release
uses: softprops/action-gh-release@v2
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
tag_name: ${{ steps.vars.outputs.gh_ref }}
name: ${{ steps.vars.outputs.gh_ref }}
body: ${{ steps.extract_release_notes.outputs.release_notes }}
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GH_REPO: ${{ github.repository }}
TAG: ${{ steps.vars.outputs.gh_ref }}
NOTES: ${{ steps.extract_release_notes.outputs.release_notes }}
run: |
if gh release view "$TAG" > /dev/null 2>&1; then
gh release edit "$TAG" --title "$TAG" --notes "$NOTES"
else
gh release create "$TAG" --verify-tag --title "$TAG" --notes "$NOTES"
fi
- name: Notify Mattermost
if: failure()

View File

@ -60,7 +60,9 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@v6
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
with:
persist-credentials: false
- name: Fmt
working-directory: ./backend

View File

@ -39,7 +39,9 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@v6
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
with:
persist-credentials: false
- name: Lint
working-directory: ./common

View File

@ -107,8 +107,9 @@ jobs:
# An empty `ref` makes checkout fall back to its default (the PR merge
# ref on pull_request, the pushed ref on push).
- name: Checkout repository
uses: actions/checkout@v6
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
with:
persist-credentials: false
ref: ${{ inputs.gh_ref }}
# The cache key must come from the SHA actually checked out: on a manual
@ -120,7 +121,7 @@ jobs:
- name: Restore Bundle Cache
id: restore
uses: actions/cache/restore@v5
uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0
with:
key: ${{ steps.vars.outputs.bundle_key }}
path: frontend/resources/public
@ -133,7 +134,7 @@ jobs:
- name: Store Bundle Cache
if: steps.restore.outputs.cache-hit != 'true'
uses: actions/cache/save@v5
uses: actions/cache/save@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0
with:
key: ${{ steps.vars.outputs.bundle_key }}
path: frontend/resources/public
@ -163,12 +164,13 @@ jobs:
steps:
- name: Checkout Repository
uses: actions/checkout@v6
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
with:
persist-credentials: false
ref: ${{ inputs.gh_ref }}
- name: Restore Cache
uses: actions/cache/restore@v5
uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0
with:
key: ${{ needs.build-bundle.outputs.bundle_key }}
path: frontend/resources/public
@ -201,7 +203,7 @@ jobs:
--reporter=blob
- name: Upload blob report
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
if: always()
with:
name: integration-blob-report-${{ matrix.shard }}
@ -210,7 +212,7 @@ jobs:
retention-days: 3
- name: Upload test result
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
if: always()
with:
name: integration-tests-result-${{ matrix.shard }}
@ -234,11 +236,13 @@ jobs:
- /var/cache/github-runner/gitlib:/root/.gitlibs
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
with:
persist-credentials: false
# The driver serves the prebuilt bundle from frontend/resources/public.
- name: Restore Cache
uses: actions/cache/restore@v5
uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0
with:
key: ${{ needs.build-bundle.outputs.bundle_key }}
path: frontend/resources/public
@ -272,11 +276,13 @@ jobs:
- /var/cache/github-runner/gitlib:/root/.gitlibs
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
with:
persist-credentials: false
# Mocked mode serves the prebuilt bundle from frontend/resources/public.
- name: Restore Cache
uses: actions/cache/restore@v5
uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0
with:
key: ${{ needs.build-bundle.outputs.bundle_key }}
path: frontend/resources/public
@ -318,8 +324,9 @@ jobs:
steps:
- name: Checkout Repository
uses: actions/checkout@v6
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
with:
persist-credentials: false
ref: ${{ inputs.gh_ref }}
- name: Install deps
@ -328,7 +335,7 @@ jobs:
pnpm install --frozen-lockfile;
- name: Download blob reports
uses: actions/download-artifact@v7
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
with:
path: frontend/all-blob-reports
pattern: integration-blob-report-*
@ -357,7 +364,7 @@ jobs:
>> "$GITHUB_STEP_SUMMARY"
- name: Upload JSON report
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
if: always()
with:
name: integration-json-report
@ -367,7 +374,7 @@ jobs:
retention-days: 7
- name: Upload HTML report
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
if: always()
with:
name: integration-html-report

View File

@ -43,7 +43,9 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@v6
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
with:
persist-credentials: false
- name: Lint
working-directory: ./exporter

View File

@ -43,7 +43,9 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@v6
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
with:
persist-credentials: false
- name: Lint
working-directory: ./frontend

View File

@ -41,7 +41,9 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@v6
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
with:
persist-credentials: false
- name: Lint
working-directory: ./library

View File

@ -33,7 +33,9 @@ jobs:
steps:
- name: Checkout code
uses: actions/checkout@v6
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
with:
persist-credentials: false
- name: Setup
working-directory: ./mcp

View File

@ -38,7 +38,9 @@ jobs:
- /var/cache/github-runner/gitlib:/root/.gitlibs
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
with:
persist-credentials: false
- name: Install deps
working-directory: ./plugins

View File

@ -39,7 +39,9 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@v6
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
with:
persist-credentials: false
- name: Format
working-directory: ./render-wasm

55
.github/zizmor.yml vendored
View File

@ -12,26 +12,6 @@ rules:
- plugins-deploy-api-doc.yml
- plugins-deploy-package.yml
- plugins-deploy-styles-doc.yml
# Use `persist-credentials: false` in actions/checkout.
artipacked:
ignore:
- build-bundle.yml
- build-docker-devenv.yml
- build-docker.yml
- plugins-deploy-api-doc.yml
- plugins-deploy-package.yml
- plugins-deploy-packages.yml
- plugins-deploy-styles-doc.yml
- release.yml
- tests-backend.yml
- tests-common.yml
- tests-e2e.yml
- tests-exporter.yml
- tests-frontend.yml
- tests-library.yml
- tests-mcp.yml
- tests-plugins.yml
- tests-wasm.yml
# Avoid restoring caches in release/publish workflows.
cache-poisoning:
ignore:
@ -70,22 +50,9 @@ rules:
github-app:
ignore:
- auto-label.yml
# Pass only the secrets each reusable workflow needs.
secrets-inherit:
ignore:
- build-adhoc.yml
- build-develop.yml
- build-staging.yml
- build-tag.yml
- build-tmp-tokens.yml
- plugins-deploy-packages.yml
# Style nudge towards the `$/...` syntax; not worth enforcing.
self-repository:
disable: true
# Replace actions that duplicate built-in runner tools.
superfluous-actions:
ignore:
- release.yml
# Pin container images to a digest.
unpinned-images:
ignore:
@ -99,25 +66,3 @@ rules:
- tests-mcp.yml
- tests-plugins.yml
- tests-wasm.yml
# Pin actions to a full commit SHA.
unpinned-uses:
ignore:
- auto-label.yml
- build-bundle.yml
- build-docker-devenv.yml
- build-docker.yml
- commit-checker.yml
- plugins-deploy-api-doc.yml
- plugins-deploy-package.yml
- plugins-deploy-packages.yml
- plugins-deploy-styles-doc.yml
- release.yml
- tests-backend.yml
- tests-common.yml
- tests-e2e.yml
- tests-exporter.yml
- tests-frontend.yml
- tests-library.yml
- tests-mcp.yml
- tests-plugins.yml
- tests-wasm.yml

View File

@ -13,7 +13,7 @@
- Logout may return an OIDC provider redirect URI when the session claims include provider/session data and the provider has a logout URI.
- Invitation tokens are verified through token issuers and only accepted when the token member id/email matches the authenticated profile; otherwise login proceeds without consuming the invitation.
- When public registration is disabled, invitation-based password registration also requires a matching, non-expired `team_invitation` row; registration revalidates it under a row lock before creating the profile.
- HTTP/session parsing details such as cookie/header precedence, JWT session token versions, and SameSite behavior are in `mem:backend/subtleties`.
- HTTP/session parsing details such as cookie/header precedence and SameSite behavior are in `mem:backend/subtleties`.
## Permission model

View File

@ -7,6 +7,7 @@ Backend: JVM Clojure; Integrant; PostgreSQL; Redis/Valkey; RPC; HTTP; storage; m
- Cross-cutting backend subtleties (RPC, DB, workers, cron, HTTP/sessions, storage, file data): `mem:backend/subtleties`
- Storage abstraction, logical buckets, object lifecycle, deduplication, access, and garbage collection: `mem:backend/storage`.
- Embedded Ladybug graph experiment, projection, incremental sync, console, and risks: `mem:backend/graph-experiment`
- Session lifetime config, token `:exp`, and idle/absolute GC: `mem:backend/session-expiration`.
- Auth flows, permission model, teams, projects, invitations, comments, webhooks, audit: `mem:backend/auth-permissions-product-domains`
- Audit-log event collection (RPC wrapper, frontend ingestion), telemetry duality, webhook fan-out, error reporters, Nexus archival and retention: `mem:backend/audit-log`
- Services, task-queue/Pub-Sub topology constraints -> `mem:prod-infra/core`.

View File

@ -0,0 +1,35 @@
# Backend Session Expiration
## Config
- `:auth-token-cookie-name` / `PENPOT_AUTH_TOKEN_COOKIE_NAME` (default `auth-token`): cookie name.
- `:auth-token-cookie-max-age` / `PENPOT_AUTH_TOKEN_COOKIE_MAX_AGE` (default 7d): idle window; drives the sliding cookie `Expires` and the GC idle threshold.
- `:auth-token-cookie-max-age-absolute` / `PENPOT_AUTH_TOKEN_COOKIE_MAX_AGE_ABSOLUTE` (default 30d): hard cap from `created-at`; drives the token `:exp` and the GC absolute threshold.
- Durations decode as `<n><unit>` with hour/minute/second units (`168h`, `30m`); day units like `7d` are rejected by the `Duration/parse` string path, use hours (`168h`, `720h`).
- Session lifetime defaults live only as code constants in `session.clj` (`default-cookie-max-age`, `default-cookie-max-age-absolute`); do not duplicate them in the `config.clj` default map. Call sites always pass the constant as the `cf/get` fallback.
## Token and session model
- Sessions live only in `http_session_v2`. Legacy v1 support (`http_session`, string ids, `:ver 0` tokens) was removed and the table dropped (`0155-drop-http-session-table`).
- `assign-token` emits header `{:kid 1 :ver 1}` with claims `:sid`, `:iat` (= `modified-at`) and `:exp` = `created-at + absolute-max-age` (omitted only when `created-at` is nil, which neither manager produces today; the branch is defensive).
- `:exp` is anchored to `created-at`, never `modified-at`, so renewal cannot extend the absolute maximum.
- Tokens issued before `:exp` existed carry no `:exp`; they are still bounded by the GC's `created_at` condition and acquire `:exp` on their next renewal (self-healing, no operator action).
- `wrap-authz` resolves the session by `(:sid claims)` only; there is no fallback to reading a session by the raw token string.
- `middleware/wrap-auth` attaches `::http/auth-data` only for `kid=1`/`ver=1` tokens with a configured decoder; anything else stays unauthenticated.
- Renewal fires when `modified-at` is older than 6h (`default-renewal-max-age`, not configurable). It `UPDATE`s the same row (`modified-at` only) and issues a new token that keeps the original `:exp`.
- `read-session` does not check age. Idle expiration is enforced on every request by `wrap-authz` (sessions past the idle window authenticate like missing ones and are never renewed); the GC only deletes the rows afterwards.
## GC (`::tasks/gc`, cron `session-gc`, daily)
- Deletes from `http_session_v2` where `modified_at < now - :auth-token-cookie-max-age` **or** `created_at < now - :auth-token-cookie-max-age-absolute`.
- The two thresholds are separate task params (`::tasks/max-age`, `::tasks/max-age-absolute`) built in `ig/expand-key`; both must be durations.
- Do not collapse the two conditions: `modified_at` alone never collects active sessions; `created_at` alone logs out active users at the idle window.
## Related lifetimes
- Organization SSO entries in session `:props` (`:sso {org-id exp}`) last 4h, hardcoded in `app.auth.oidc`.
- Access tokens (`app.rpc.commands.access-token`) have a user-chosen `:expires-at` (Never/30/60/90/180d), independent from HTTP sessions.
- Cookie `SameSite`/`secure` depend on `:cors`, `:strict-session-cookies`, `:secure-session-cookies`; `parse-flags` auto-adds `:disable-secure-session-cookies` for non-localhost HTTP `public-uri`.
- Without `PENPOT_SECRET_KEY`, derived subsystem keys change on restart and all sessions/invitations are invalidated.
- Read-only DB pool: sessions fall back to `inmemory-manager`, lost on restart, no GC.
- HTTP middleware and cookie/header precedence details: `mem:backend/subtleties`.

View File

@ -54,8 +54,7 @@
### Key Warning (from function notes):
The improved note in `import-storage-objects` and `handle-persistence` warns:
**Do not reuse the main database connection for storage operations within a transaction.** The storage upload process can fail mid-operation, leaving orphaned objects on the backend. If the outer transaction aborts, pending storage objects become unreconciliable because the storage subsystem registers its pending state in separate transactions.
The improved note in `import-storage-objects` and `handle-persistence` warns: **Do not reuse the main database connection for storage operations within a transaction.** The storage upload process can fail mid-operation, leaving orphaned objects on the backend. If the outer transaction aborts, pending storage objects become unreconciliable because the storage subsystem registers its pending state in separate transactions.
### Rule of Thumb for `sto/put-object!`:
@ -66,10 +65,7 @@ Since `put-object!` uses backend-specific operations (`impl/resolve-backend` + `
- Deduplication requires `::sto/deduplicate?`, a content hash, and bucket metadata.
- The lookup matches hash, bucket, backend, and `deleted_at IS NULL`.
- The lookup only considers rows with `status='valid'`; pending rows are invisible.
- A hit whose blob is missing is repaired in place: the same row/id is kept,
and `put-object!` rewrites the blob under that id. This heals all existing
references to the object. If the rewrite fails, the row is left live and
valid for a later retry.
- A hit whose blob is missing is repaired in place: the same row/id is kept, and `put-object!` rewrites the blob under that id. This heals all existing references to the object. If the rewrite fails, the row is left live and valid for a later retry.
- The lookup does not include file ID, profile ID, team ID, or organization ID.
- Objects can therefore share content across users and files within one bucket.
- Deleted objects are not reused.
@ -130,6 +126,8 @@ Since `put-object!` uses backend-specific operations (`impl/resolve-backend` + `
- Logical storage operations (`app.storage`, `::mtx/metrics` required by the schema):
- `penpot_storage_operations_total{op,bucket,backend}` — `put`, `repair`, `get-data`, `get-bytes`, `del`, `touch`, `exists`. All ops are success-only: `put`/`repair` emit after the backend write, `get-*` after the backend fetch opens, `touch`/`del` only when a row actually changed. `touch-object!`/`del-object!` take the object id (UUID) only — no object overload. Labels come from the updated row itself via `UPDATE ... RETURNING id, backend, metadata` (no extra `SELECT`); with no row matched they emit nothing. `del-object!` only matches live rows (`deleted_at IS NULL`): a repeated del returns `false` and emits nothing. Post-open stream read errors stay counted as attempts. `del` only marks `deleted_at`; physical deletion is a GC concern. `exists` is emitted per deduplication-hit probe, always paired with a `hit`/`repair` outcome (never on probe failure), not per user-facing existence check.
- `penpot_storage_dedup_total{result,bucket}` — `hit`, `miss`, `repair`, `skip`.
- Storage GC poison (`app.storage.gc-touched`, `::mtx/metrics` required in the handler cfg):
- `penpot_storage_gc_poison_total` — touched rows whose metadata fails to decode. Each one is logged at error level and deferred exactly one day instead of aborting the chunk. No labels; incremented once per row per scan.
- Asset serving (`app.http.assets`, `::mtx/metrics` required in the handler cfg):
- `penpot_storage_asset_requests_total{route,backend,bucket,result}` — `route` is `by-id`, `by-file-media-id`, or `thumbnail`; `result` is `served` (<400), `not-found` (404), `unauthorized` (401/403), or `error` (everything else, including a nil/non-number status: every serve path must set `::yres/status`). Serve-path exceptions are counted by `serve-object-measured` and then rethrown. Permission-denied file-media requests and tempfile ownership mismatches both answer HTTP 404 (to avoid leaking existence) but are counted as `unauthorized`. Malformed UUIDs raise before any emission point and are never counted. Counts backend requests that trigger a browser GET to the object store (one per cache miss), so it is a proxy for object GETs, not an exact count.
- The physical and logical counters intentionally overlap in coverage but differ in meaning; do not sum them.

View File

@ -33,10 +33,10 @@
- The backend sets Clojure `*assert*` globally from the `:backend-asserts` feature flag. Assertion-dependent checks can therefore differ by runtime flags.
- Request body parsing is mostly POST-oriented and supports Transit JSON plus plain JSON. Plain JSON request keys are kebab-decoded before being merged into `:params`.
- Response formatting negotiates with `Accept` or `_fmt=json`. Transit is the default for collection/boolean bodies; JSON encoding has special pointer-map handling.
- Auth prefers the session cookie token before the `Authorization` header. Headers may be `Token` or `Bearer`; JWTs with `kid=1` and `ver=1` are decoded as v1 session tokens, otherwise they are treated as legacy tokens.
- Auth prefers the session cookie token before the `Authorization` header. Headers may be `Token` or `Bearer`. Only `kid=1`/`ver=1` tokens are decoded as session tokens; anything else is left unauthenticated (legacy v1 tokens were removed).
- Shared-key auth requires `x-shared-key` as `<key-id> <key>` and stores the lowercased key id on the request. If no shared keys are configured it always rejects.
- Session management uses DB storage unless the DB pool is read-only, then falls back to the in-memory manager. DB sessions support both legacy string ids and v2 UUID session ids.
- Session cookies are renewed when using a legacy string id or when `modified-at` is older than the renewal interval. SameSite is `none` for CORS, otherwise strict/lax based on config.
- Session management uses DB storage unless the DB pool is read-only, then falls back to the in-memory manager. Sessions use only the v2 UUID model (`http_session_v2`); legacy string ids were removed.
- Session cookies are renewed when `modified-at` is older than the 6h renewal interval. SameSite is `none` for CORS, otherwise strict/lax based on config. Session lifetime config and GC: `mem:backend/session-expiration`.
## HTTP server self-metrics

View File

@ -47,6 +47,15 @@
- Export multiple fills to SVG [#11466](https://github.com/penpot/penpot/issues/11466) (PR: [#11467](https://github.com/penpot/penpot/pull/11467))
- Add Penpot-specific board size presets (file thumbnail, template cover, plugin icon/cover) [#11561](https://github.com/penpot/penpot/issues/11561) (PR: [#11565](https://github.com/penpot/penpot/pull/11565))
## 2.18.1
### :bug: Bugs fixed
- Fix workspace freezing with a stack overflow when opening the context menu on a shape that isn't on the current page [#11882](https://github.com/penpot/penpot/issues/11882) (PR: [#11891](https://github.com/penpot/penpot/pull/11891))
- Fix workspace showing repeated error toasts when a third-party script injected into the page throws [#11813](https://github.com/penpot/penpot/issues/11813) (PR: [#11816](https://github.com/penpot/penpot/pull/11816))
- Fix register flow from workspace URL not working [#11753](https://github.com/penpot/penpot/issues/11753) (PR: [#11756](https://github.com/penpot/penpot/pull/11756))
- Fix empty response when exporting bin using API (by @filipsajdak) [#7649](https://github.com/penpot/penpot/issues/7649) (PR: [#10341](https://github.com/penpot/penpot/pull/10341))
## 2.18.0
### :rocket: Epics and highlights

View File

@ -61,7 +61,6 @@
:objects-storage-fs-directory "assets"
:auth-token-cookie-name "auth-token"
:auth-token-cookie-max-age-absolute (ct/duration {:days 30})
:assets-path "/internal/assets/"
:smtp-default-reply-to "Penpot <no-reply@example.com>"
@ -304,6 +303,10 @@
[:objects-storage-s3-region {:optional true} :keyword]
[:objects-storage-s3-endpoint {:optional true} ::sm/uri]
;; Write storage_object.metadata as plain JSON instead of
;; Transit-JSON. Unset by default (Phase 1: keep writing Transit).
[:storage-metadata-as-json {:optional true} ::sm/boolean]
;; SSRF protection
[:ssrf-allowed-hosts {:optional true} [::sm/set :string]]
[:ssrf-extra-blocked-cidrs {:optional true} [::sm/set :string]]]))

View File

@ -675,7 +675,7 @@
(contains? params :block)
(do
(db/update! conn :profile {:is-blocked true} {:id (:id profile)})
(db/delete! conn :http-session {:profile-id (:id profile)})
(session/invalidate-all cfg (:id profile))
{::yres/status 200
::yres/headers {"content-type" "text/plain"}

View File

@ -306,16 +306,15 @@
(let [decode-fn (get decoders type)]
(if (or (= type :cookie) (= type :bearer))
(let [metadata (tokens/decode-header token)]
;; NOTE: we only proceed to decode claims on new
;; cookie tokens. The old cookies dont need to be
;; decoded because they use the token string as ID
;; NOTE: only current (kid=1/ver=1) cookie tokens carry
;; decodable claims. Anything else is left unauthenticated.
(if (and (= (:kid metadata) 1)
(= (:ver metadata) 1)
(some? decode-fn))
(assoc request ::http/auth-data (assoc auth
:claims (decode-fn token)
:metadata metadata))
(assoc request ::http/auth-data (assoc auth :metadata {:ver 0}))))
request))
(if decode-fn
(assoc request ::http/auth-data (assoc auth :claims (decode-fn token)))

View File

@ -81,14 +81,8 @@
[pool]
(reify ISessionManager
(read-session [_ id]
(if (string? id)
;; Backward compatibility: http_session (v1) has no props column
(let [session (db/exec-one! pool (sql/select :http-session {:id id}))]
(-> session
(assoc :modified-at (:updated-at session))
(dissoc :updated-at)))
(some-> (db/exec-one! pool (sql/select :http-session-v2 {:id id}))
(decode-session))))
(some-> (db/exec-one! pool (sql/select :http-session-v2 {:id id}))
(decode-session)))
(create-session [_ params]
(assert (valid-params? params) "expect valid session params")
@ -103,23 +97,15 @@
(update-session [_ session]
(let [modified-at (ct/now)]
(if (string? (:id session))
(db/insert! pool :http-session-v2
(-> session
(assoc :id (uuid/next))
(assoc :created-at modified-at)
(assoc :modified-at modified-at)))
(db/update! pool :http-session-v2
(cond-> {:modified-at modified-at}
(some? (:props session))
(assoc :props (db/tjson (:props session))))
{:id (:id session)}
{::db/return-keys true}))))
(db/update! pool :http-session-v2
(cond-> {:modified-at modified-at}
(some? (:props session))
(assoc :props (db/tjson (:props session))))
{:id (:id session)}
{::db/return-keys true})))
(delete-session [_ id]
(if (string? id)
(db/delete! pool :http-session {:id id} {::db/return-keys false})
(db/delete! pool :http-session-v2 {:id id} {::db/return-keys false}))
(db/delete! pool :http-session-v2 {:id id} {::db/return-keys false})
nil)))
(defn inmemory-manager
@ -256,25 +242,28 @@
(db/exec! pool [sql:clear-organization-sso-sessions organization-key organization-key])))
(defn- renew-session?
[{:keys [id modified-at] :as session}]
(or (string? id)
(and (ct/inst? modified-at)
(let [elapsed (ct/diff modified-at (ct/now))]
(neg? (compare default-renewal-max-age elapsed))))))
[{:keys [modified-at]}]
(and (ct/inst? modified-at)
(let [elapsed (ct/diff modified-at (ct/now))]
(neg? (compare default-renewal-max-age elapsed)))))
(defn- session-expired?
[{:keys [modified-at]}]
(and (ct/inst? modified-at)
(let [max-age (cf/get :auth-token-cookie-max-age default-cookie-max-age)]
(pos? (compare (ct/now) (ct/plus modified-at max-age))))))
(defn- wrap-authz
[handler {:keys [::manager] :as cfg}]
(assert (manager? manager) "expected valid session manager")
(fn [request]
(let [{:keys [type token claims metadata]} (get request ::http/auth-data)]
(let [{:keys [type claims]} (get request ::http/auth-data)]
(cond
(= type :cookie)
(let [session
(case (:ver metadata)
;; BACKWARD COMPATIBILITY WITH OLD TOKENS
0 (read-session manager token)
1 (some->> (:sid claims) (read-session manager))
nil)
(let [session (some->> (:sid claims) (read-session manager))
;; Idle-expired sessions authenticate like missing ones:
;; no profile, no renewal. The GC deletes the row later.
session (when-not (session-expired? session) session)
request
(cond-> request
@ -294,11 +283,8 @@
response))
(= type :bearer)
(let [session (case (:ver metadata)
;; BACKWARD COMPATIBILITY WITH OLD TOKENS
0 (read-session manager token)
1 (some->> (:sid claims) (read-session manager))
nil)
(let [session (some->> (:sid claims) (read-session manager))
session (when-not (session-expired? session) session)
request (cond-> request
(some? session)
(-> (assoc ::profile-id (:profile-id session))
@ -317,9 +303,9 @@
(defn- assign-session-cookie
[response {token :token modified-at :modified-at}]
(let [max-age (cf/get :auth-token-cookie-max-age default-cookie-max-age)
created-at modified-at
renewal (ct/plus created-at default-renewal-max-age)
expires (ct/plus created-at max-age)
renewal-at modified-at
renewal (ct/plus renewal-at default-renewal-max-age)
expires (ct/plus renewal-at max-age)
secure? (contains? cf/flags :secure-session-cookies)
strict? (contains? cf/flags :strict-session-cookies)
cors? (contains? cf/flags :cors)
@ -346,40 +332,46 @@
(defmethod ig/assert-key ::tasks/gc
[_ params]
(assert (db/pool? (::db/pool params)) "expected valid database pool")
(assert (ct/duration? (::tasks/max-age params))))
(assert (ct/duration? (::tasks/max-age params)))
(assert (ct/duration? (::tasks/max-age-absolute params)))
;; NOTE: explicit throw (not `assert`) so the guard fires even when
;; the `:backend-asserts` flag leaves `*assert*` false.
(when (pos? (compare (::tasks/max-age params)
(::tasks/max-age-absolute params)))
(throw (IllegalArgumentException.
"absolute session max-age must be greater than or equal to idle max-age"))))
(defmethod ig/expand-key ::tasks/gc
[k v]
(let [max-age (cf/get :auth-token-cookie-max-age default-cookie-max-age)]
{k (merge {::tasks/max-age max-age} (d/without-nils v))}))
(def ^:private
sql:delete-expired
"DELETE FROM http_session
WHERE updated_at < ?::timestamptz
or (updated_at is null and
created_at < ?::timestamptz)")
(let [max-age (cf/get :auth-token-cookie-max-age default-cookie-max-age)
max-age-absolute (cf/get :auth-token-cookie-max-age-absolute
default-cookie-max-age-absolute)]
{k (merge {::tasks/max-age max-age
::tasks/max-age-absolute max-age-absolute}
(d/without-nils v))}))
(def ^:private
sql:delete-expired-v2
"DELETE FROM http_session_v2
WHERE created_at < ?::timestamptz")
WHERE modified_at < ?::timestamptz
OR created_at < ?::timestamptz")
(defn- collect-expired-tasks
[{:keys [::db/conn ::tasks/max-age]}]
(let [threshold (ct/minus (ct/now) max-age)
result-legacy (-> (db/exec-one! conn [sql:delete-expired threshold threshold])
(db/get-update-count))
result-v2 (-> (db/exec-one! conn [sql:delete-expired-v2 threshold])
(db/get-update-count))]
[{:keys [::db/conn ::tasks/max-age ::tasks/max-age-absolute]}]
(let [idle-threshold (ct/minus (ct/now) max-age)
abs-threshold (ct/minus (ct/now) max-age-absolute)
result (-> (db/exec-one! conn [sql:delete-expired-v2
idle-threshold abs-threshold])
(db/get-update-count))]
(l/dbg :task "gc"
:hint "clean http sessions"
:deleted-legacy result-legacy
:deleted-v2 result-v2)
(+ result-legacy result-v2)))
:deleted result)
result))
(defmethod ig/init-key ::tasks/gc
[_ {:keys [::tasks/max-age] :as cfg}]
(l/dbg :hint "initializing session gc task" :max-age max-age)
[_ {:keys [::tasks/max-age ::tasks/max-age-absolute] :as cfg}]
(l/dbg :hint "initializing session gc task"
:max-age max-age
:max-age-absolute max-age-absolute)
(fn [_]
(db/tx-run! cfg collect-expired-tasks)))

View File

@ -180,6 +180,12 @@
::mdef/labels ["route" "backend" "bucket" "result"]
::mdef/type :counter}
:storage-gc-poison
{::mdef/name "penpot_storage_gc_poison_total"
::mdef/help "Storage objects deferred by storage-gc-touched because their metadata is corrupt."
::mdef/labels []
::mdef/type :counter}
:http-server-dispatch-timing
{::mdef/name "penpot_http_server_dispatch_timing"
::mdef/help "Histogram of dispatch handler"
@ -271,7 +277,8 @@
::sto/storage (ig/ref ::sto/storage)}
::sto.gc-touched/handler
{::db/pool (ig/ref ::db/pool)}
{::db/pool (ig/ref ::db/pool)
::mtx/metrics (ig/ref ::mtx/metrics)}
::sto.pending-gc/handler
{::db/pool (ig/ref ::db/pool)

View File

@ -505,7 +505,16 @@
:fn (mg/resource "app/migrations/sql/0153-add-storage-object-status-and-deletion-attempts.sql")}
{:name "0154-add-upload-session-chunk-table"
:fn (mg/resource "app/migrations/sql/0154-add-upload-session-chunk-table.sql")}])
:fn (mg/resource "app/migrations/sql/0154-add-upload-session-chunk-table.sql")}
{:name "0155-normalize-storage-object-metadata"
:fn (mg/resource "app/migrations/sql/0155-normalize-storage-object-metadata.sql")}
{:name "0155-drop-http-session-table"
:fn (mg/resource "app/migrations/sql/0155-drop-http-session-table.sql")}
{:name "0156-add-storage-object-json-dedup-index"
:fn (mg/resource "app/migrations/sql/0156-add-storage-object-json-dedup-index.sql")}])
(defn apply-migrations!
[pool name migrations]

View File

@ -0,0 +1,4 @@
-- Legacy v1 HTTP sessions have been removed from the backend; the
-- http_session table is no longer read or written by any code path.
DROP TABLE http_session;

View File

@ -0,0 +1,52 @@
--- Normalize storage_object.metadata inside Transit (Phase 1).
---
--- Brings every row to a canonical shape WITHOUT changing the encoding
--- family (still Transit JSON, "~:" keys), so old and new code keep
--- reading while it runs. Each statement below documents its own rule.
---
--- NOTE: key-existence is tested with `-> ... IS [NOT] NULL` instead of
--- the `?` operator because the migration runner prepares statements
--- and a bare `?` is parsed as a bind placeholder.
---
--- Idempotent: re-running changes zero rows.
--- NOTE for large instances (e.g. our production, ~33M rows): DO NOT run
--- this inline. Mark the migration as executed (fake) and run the batched
--- equivalent by hand: .agents/plans/2026-09-11-storage-normalize-phase1.sql
-- Drop leftovers from the old chunked-upload metadata: no code reads
-- "~:upload-id" / "~:chunk-index" anymore (#11651 moved chunks to
-- upload_session_chunk and objects-gc purges the orphan rows), and the
-- Phase 1 readers already ignore them on decode.
UPDATE storage_object
SET metadata = metadata - '~:upload-id' - '~:chunk-index'
WHERE (metadata -> '~:upload-id') IS NOT NULL
OR (metadata -> '~:chunk-index') IS NOT NULL;
-- Rows that carry only the legacy "~:reference": promote it to "~:bucket"
-- with the same value. The value comes as a Transit keyword ("~:xxx"), so
-- the "~:" prefix is stripped; plain strings are kept as they are.
UPDATE storage_object
SET metadata = (metadata - '~:reference')
|| jsonb_build_object('~:bucket',
CASE WHEN metadata->>'~:reference' LIKE '~:%'
THEN substr(metadata->>'~:reference', 3)
ELSE metadata->>'~:reference'
END)
WHERE (metadata -> '~:reference') IS NOT NULL
AND (metadata -> '~:bucket') IS NULL;
-- Rows that carry both keys: "~:bucket" wins and the residual
-- "~:reference" is dropped.
UPDATE storage_object
SET metadata = metadata - '~:reference'
WHERE (metadata -> '~:reference') IS NOT NULL
AND (metadata -> '~:bucket') IS NOT NULL;
-- Rows with neither key get the historical default bucket
-- "file-media-object" (the value `lookup-bucket` used to fall back to).
-- NULL columns match this rule too (`->` on NULL is NULL) and are
-- backfilled by coalesce.
UPDATE storage_object
SET metadata = coalesce(metadata, '{}') || '{"~:bucket": "file-media-object"}'
WHERE (metadata -> '~:bucket') IS NULL
AND (metadata -> '~:reference') IS NULL;

View File

@ -0,0 +1,11 @@
-- Dedup lookup index for the plain-JSON encoding of
-- storage_object.metadata ("hash" / "bucket").
--
-- The dedup lookup matches both metadata encodings while Transit rows
-- still exist (a UNION ALL of two indexable branches). The legacy "~:"
-- index (storage_object__hash_backend_bucket__idx, migration 0068) keeps
-- covering the Transit branch until Transit support is removed.
CREATE INDEX storage_object__hash_backend_bucket_json__idx
ON storage_object ((metadata->>'hash'), (metadata->>'bucket'), backend)
WHERE deleted_at IS NULL AND status = 'valid';

View File

@ -125,7 +125,7 @@
{:columns [:id :email]})]
(when-not (:is-blocked profile)
(db/update! conn :profile {:is-blocked true} {:id (:id profile)})
(db/delete! conn :http-session {:profile-id (:id profile)})
(session/invalidate-all system (:id profile))
:blocked))))))
(defn reset-password!

View File

@ -20,6 +20,7 @@
[app.storage.fs :as sfs]
[app.storage.impl :as impl]
[app.storage.s3 :as ss3]
[app.storage.schema :as stsch]
[cuerdas.core :as str]
[datoteka.fs :as fs]
[integrant.core :as ig])
@ -37,28 +38,18 @@
nil)))
(def default-bucket
"file-media-object")
stsch/default-bucket)
(def tempfile-bucket
"Bucket name for temporary file uploads (10-minute expiry)."
"tempfile")
stsch/tempfile-bucket)
(def upload-session-bucket
"Bucket name for chunked-upload chunks."
"upload-session")
stsch/upload-session-bucket)
(def valid-buckets
#{"file-media-object"
"team-font-variant"
"file-object-thumbnail"
"file-thumbnail"
"profile"
"organization"
tempfile-bucket
upload-session-bucket
"file-data"
"file-data-fragment"
"file-change"})
stsch/metadata-buckets)
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Storage Module State
@ -118,18 +109,37 @@
params
params))
;; Matches both metadata encodings while Transit rows still exist. The
;; UNION ALL keeps each branch indexable: migration 0068 covers the legacy
;; "~:" keys and 0156 covers the plain JSON keys. Phase 3 drops the legacy
;; branch once no "~:" rows remain.
(def ^:private sql:get-database-object-by-hash
"(
select * from storage_object
where (metadata->>'~:hash') = ?
and (metadata->>'~:bucket') = ?
and backend = ?
and deleted_at is null
and status = 'valid'
limit 1
) union all (
select * from storage_object
where (metadata->>'hash') = ?
and (metadata->>'bucket') = ?
and backend = ?
and deleted_at is null
and status = 'valid'
limit 1
) limit 1")
;; NOTE: metadata is left encoded; row->storage-object is responsible for
;; decoding it.
(defn- get-database-object-by-hash
[connectable backend bucket hash]
(let [sql (str "select * from storage_object "
" where (metadata->>'~:hash') = ? "
" and (metadata->>'~:bucket') = ? "
" and backend = ?"
" and deleted_at is null"
" and status = 'valid'"
" limit 1")]
;; NOTE: metadata is left encoded; row->storage-object is
;; responsible for decoding it.
(db/exec-one! connectable [sql hash bucket (name backend)])))
(let [backend (name backend)]
(db/exec-one! connectable [sql:get-database-object-by-hash
hash bucket backend
hash bucket backend])))
(defn- promote-object!
[storage object]
@ -147,7 +157,7 @@
res))
(defn row->storage-object [res]
(let [mdata (or (some-> (:metadata res) (db/decode-transit-pgobject)) {})]
(let [mdata (or (some-> (:metadata res) (stsch/decode-metadata)) {})]
(impl/storage-object
(:id res)
(:size res)
@ -289,7 +299,7 @@
{:id id
:size (impl/get-size content)
:backend (name backend)
:metadata (db/tjson mdata)
:metadata (stsch/encode-metadata mdata)
:deleted-at expired-at
:touched-at touched-at
:status "pending"})

View File

@ -51,7 +51,7 @@
"DELETE FROM storage_object
WHERE id = ANY(?::uuid[])")
(defn- delete-sobjects!
(defn- delete-sobjects
[conn ids]
(let [ids (db/create-array conn "uuid" ids)]
(-> (db/exec-one! conn [sql:delete-sobjects ids])
@ -61,7 +61,7 @@
"DELETE FROM upload_session_chunk
WHERE object_id = ANY(?::uuid[])")
(defn- delete-upload-session-chunks!
(defn- delete-upload-session-chunks
"Remove the chunk mappings for the given storage object ids. This must run
before the storage_object rows are deleted: the upload_session_chunk
foreign keys are ON DELETE NO ACTION."
@ -75,7 +75,7 @@
deleted_at = NOW() + INTERVAL '1 day'
WHERE id = ANY(?::uuid[])")
(defn- increment-attempts-and-defer!
(defn- increment-attempts-and-defer
[conn ids]
(let [ids (db/create-array conn "uuid" ids)]
(db/exec-one! conn [sql:increment-attempts-and-defer ids])))
@ -85,7 +85,7 @@
WHERE id = ANY(?::uuid[])
AND deletion_attempts >= ?")
(defn- delete-give-up!
(defn- delete-give-up
[conn ids]
(let [ids (db/create-array conn "uuid" ids)]
(db/exec-one! conn [sql:delete-give-up ids max-attempts])))
@ -93,7 +93,7 @@
(defn- process-chunk
"Attempt to delete a chunk of storage objects from a specific backend.
This function runs inside the caller's transaction (clean-deleted!) —
This function runs inside the caller's transaction (clean-deleted) —
it does NOT open its own transaction. The caller is responsible for
ensuring the rows are locked via FOR UPDATE SKIP LOCKED before calling.
@ -121,18 +121,18 @@
;; storage_object rows (NO ACTION foreign keys). It only affects
;; objects of the upload-session bucket; for any other bucket the
;; delete matches no rows.
(delete-upload-session-chunks! conn ok-ids)
(delete-sobjects! conn ok-ids))
(delete-upload-session-chunks conn ok-ids)
(delete-sobjects conn ok-ids))
(when (seq fail-ids)
(increment-attempts-and-defer! conn fail-ids)
(increment-attempts-and-defer conn fail-ids)
;; NOTE: same NO ACTION ordering as above: the give-up DELETE below
;; removes storage_object rows, so chunk mappings must go first.
;; Deferred objects keep their rows; only the mapping of a
;; permanently given-up object disappears early, and that object is
;; already deleted-marked.
(delete-upload-session-chunks! conn fail-ids)
(let [given-up (delete-give-up! conn fail-ids)]
(delete-upload-session-chunks conn fail-ids)
(let [given-up (delete-give-up conn fail-ids)]
(when (pos? (db/get-update-count given-up))
(l/wrn :hint "giving up on orphan blob after max attempts"
:ids fail-ids
@ -160,7 +160,7 @@
[conn size]
(db/exec! conn [sql:get-deleted-chunk (ct/now) size]))
(defn- clean-deleted!
(defn- clean-deleted
[cfg]
(loop [total 0]
(let [deleted (db/tx-run! cfg
@ -184,6 +184,6 @@
(defmethod ig/init-key ::handler
[_ cfg]
(fn [_]
(let [total (clean-deleted! cfg)]
(let [total (clean-deleted cfg)]
(l/inf :hint "task finished" :total total)
{:deleted total})))

View File

@ -24,6 +24,7 @@
[app.common.logging :as l]
[app.common.time :as ct]
[app.db :as db]
[app.metrics :as mtx]
[app.storage :as sto]
[app.storage.impl :as impl]
[integrant.core :as ig]))
@ -95,7 +96,7 @@
SET touched_at = NULL
WHERE id = ANY(?::uuid[])")
(defn- mark-freeze-in-bulk!
(defn- mark-freeze-in-bulk
[conn ids]
(let [ids (db/create-array conn "uuid" ids)]
(db/exec-one! conn [sql:mark-freeze-in-bulk ids])))
@ -106,11 +107,21 @@
touched_at = NULL
WHERE id = ANY(?::uuid[])")
(defn- mark-delete-in-bulk!
(defn- mark-delete-in-bulk
[conn ids]
(let [ids (db/create-array conn "uuid" ids)]
(db/exec-one! conn [sql:mark-delete-in-bulk (ct/now) ids])))
(def ^:private sql:defer-in-bulk
"UPDATE storage_object
SET touched_at = ?
WHERE id = ANY(?::uuid[])")
(defn- defer-in-bulk
[conn ids timestamp]
(let [ids (db/create-array conn "uuid" ids)]
(db/exec-one! conn [sql:defer-in-bulk timestamp ids])))
;; NOTE: A getter that retrieves the key which will be used for group
;; ids; previously we have no value, then we introduced the
;; `:reference` prop, and then it is renamed to `:bucket` and now is
@ -125,12 +136,17 @@
;; have value, it means :file-media-object.
(defn- lookup-bucket
[{:keys [metadata]}]
[{:keys [id metadata]}]
(or (some-> metadata :bucket)
(some-> metadata :reference d/name)
sto/default-bucket))
(do
;; Only reachable when the metadata column is NULL: the decode
;; always sets :bucket on non-nil metadata (0155 also backfills
;; NULL columns). Keep working, but make it visible.
(l/wrn :hint "storage object without bucket metadata, using fallback"
:id (str id))
sto/default-bucket)))
(defn- process-objects!
(defn- process-objects
[conn has-refs? bucket objects]
(loop [to-freeze #{}
to-delete #{}
@ -148,31 +164,37 @@
:bucket bucket)
(recur to-freeze (conj to-delete id) (rest objects))))
(do
(some->> (seq to-freeze) (mark-freeze-in-bulk! conn))
(some->> (seq to-delete) (mark-delete-in-bulk! conn))
(some->> (seq to-freeze) (mark-freeze-in-bulk conn))
(some->> (seq to-delete) (mark-delete-in-bulk conn))
[(count to-freeze) (count to-delete)]))))
(defn- process-bucket!
(defn- process-bucket
[conn bucket objects]
(cond
(= bucket "file-media-object") (process-objects! conn has-file-media-object-refs? bucket objects)
(= bucket "team-font-variant") (process-objects! conn has-team-font-variant-refs? bucket objects)
(= bucket "file-object-thumbnail") (process-objects! conn has-file-object-thumbnails-refs? bucket objects)
(= bucket "file-thumbnail") (process-objects! conn has-file-thumbnails-refs? bucket objects)
(= bucket "profile") (process-objects! conn has-profile-refs? bucket objects)
(= bucket "file-data") (process-objects! conn has-file-data-refs? bucket objects)
(= bucket sto/tempfile-bucket) (process-objects! conn (constantly false) sto/tempfile-bucket objects)
(= bucket sto/upload-session-bucket) (process-objects! conn (constantly false) sto/upload-session-bucket objects)
(= bucket "organization") (process-objects! conn (constantly false) bucket objects)
(= bucket "file-media-object") (process-objects conn has-file-media-object-refs? bucket objects)
(= bucket "team-font-variant") (process-objects conn has-team-font-variant-refs? bucket objects)
(= bucket "file-object-thumbnail") (process-objects conn has-file-object-thumbnails-refs? bucket objects)
(= bucket "file-thumbnail") (process-objects conn has-file-thumbnails-refs? bucket objects)
(= bucket "profile") (process-objects conn has-profile-refs? bucket objects)
(= bucket "file-data") (process-objects conn has-file-data-refs? bucket objects)
(= bucket sto/tempfile-bucket) (process-objects conn (constantly false) sto/tempfile-bucket objects)
(= bucket sto/upload-session-bucket) (process-objects conn (constantly false) sto/upload-session-bucket objects)
(= bucket "organization") (process-objects conn (constantly false) bucket objects)
:else
(ex/raise :type :internal
:code :unexpected-unknown-reference
:hint (dm/fmt "unknown reference '%'" bucket))))
(defn process-chunk!
(defn- defer-poison
"Defer corrupt rows by one day in their own transaction, separate from
the healthy-chunk work."
[{:keys [::db/conn]} poison-ids]
(defer-in-bulk conn poison-ids (ct/plus (ct/now) {:days 1})))
(defn process-chunk
[{:keys [::db/conn]} chunk]
(reduce-kv (fn [[nfo ndo] bucket objects]
(let [[nfo' ndo'] (process-bucket! conn bucket objects)]
(let [[nfo' ndo'] (process-bucket conn bucket objects)]
[(+ nfo nfo')
(+ ndo ndo')]))
[0 0]
@ -189,21 +211,45 @@
SKIP LOCKED
LIMIT 10")
(defn- try-decode-row
"Decode a touched row, capturing corrupt metadata as poison instead
of aborting the whole chunk. Poison rows are deferred by the caller."
[row]
(try
[:ok (impl/decode-row row)]
;; Exception, not Throwable: JVM Errors (OOM, StackOverflow) must
;; not be swallowed as a deferrable data problem.
(catch Exception cause
(l/err :hint "storage object with corrupt metadata, deferring evaluation"
:id (str (:id row))
:cause cause)
[:poison (:id row)])))
(defn get-chunk
[conn timestamp]
(->> (db/exec! conn [sql:get-touched-storage-objects timestamp])
(map impl/decode-row)
(not-empty)))
(let [grouped (->> (db/exec! conn [sql:get-touched-storage-objects timestamp])
(map try-decode-row)
(group-by first))]
{:chunk (not-empty (mapv second (:ok grouped)))
:poison (not-empty (mapv second (:poison grouped)))}))
(defn- process-touched!
[{:keys [::db/pool ::timestamp] :as cfg}]
(defn- process-touched
[{:keys [::db/pool ::mtx/metrics ::timestamp] :as cfg}]
(loop [freezed 0
deleted 0]
(if-let [chunk (get-chunk pool timestamp)]
(let [[nfo ndo] (db/tx-run! cfg process-chunk! chunk)]
(recur (long (+ freezed nfo))
(long (+ deleted ndo))))
{:freeze freezed :delete deleted})))
(let [{:keys [chunk poison]} (get-chunk pool timestamp)]
(when (seq poison)
(mtx/run! metrics :id :storage-gc-poison :inc (count poison))
(db/tx-run! cfg defer-poison poison))
;; Keep draining after a poison-only batch: the deferred rows leave
;; the selection and the next batch may hold healthy objects.
(if (or (seq chunk) (seq poison))
(let [[nfo ndo] (if (seq chunk)
(db/tx-run! cfg process-chunk chunk)
[0 0])]
(recur (long (+ freezed nfo))
(long (+ deleted ndo))))
{:freeze freezed :delete deleted}))))
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; HANDLER
@ -211,7 +257,8 @@
(defmethod ig/assert-key ::handler
[_ params]
(assert (db/pool? (::db/pool params)) "expect valid storage"))
(assert (db/pool? (::db/pool params)) "expect valid storage")
(assert (mtx/metrics? (::mtx/metrics params)) "expect valid metrics"))
(defmethod ig/expand-key ::handler
[k v]
@ -223,5 +270,5 @@
(let [threshold (if (:skip-delay props)
(ct/now)
(ct/minus (ct/now) min-age))]
(process-touched! (assoc cfg ::timestamp threshold)))))
(process-touched (assoc cfg ::timestamp threshold)))))

View File

@ -9,8 +9,8 @@
(:require
[app.common.data.macros :as dm]
[app.common.exceptions :as ex]
[app.db :as db]
[app.storage :as-alias sto]
[app.storage.schema :as stsch]
[buddy.core.codecs :as bc]
[buddy.core.hash :as bh]
[clojure.java.io :as jio]
@ -26,7 +26,7 @@
[{:keys [metadata] :as row}]
(cond-> row
(some? metadata)
(assoc :metadata (db/decode-transit-pgobject metadata))))
(assoc :metadata (stsch/decode-metadata metadata))))
;; --- API Definition

View File

@ -0,0 +1,149 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; Copyright (c) KALEIDOS SUBSIDIARY SL
(ns app.storage.schema
"Schema, validation and encode/decode helpers for
`storage_object.metadata`.
Phase 1: reads accept both the legacy Transit encoding and plain
JSON (sniffed by the `\"~:` marker); writes validate + normalize
but still serialize as Transit unless the
`:storage-metadata-as-json` config flag is set."
(:require
[app.common.data :as d]
[app.common.exceptions :as ex]
[app.common.schema :as sm]
[app.config :as cf]
[app.db :as db]
[cuerdas.core :as str])
(:import
org.postgresql.util.PGobject))
(def default-bucket
"file-media-object")
(def tempfile-bucket
"Bucket name for temporary file uploads (10-minute expiry)."
"tempfile")
(def upload-session-bucket
"Bucket name for chunked-upload chunks."
"upload-session")
(def bucket-requirements
"Canonical buckets and the extra keys each one must carry, so a new
bucket and its contract live in one place. Listed keys are declared as
optional in `schema:metadata` and only checked for presence; their type
is enforced by the map. Only buckets whose keys are read somewhere need
an entry: `file-data` ids are resolved by the GC, `organization-id` is
the logo's owner."
{"file-media-object" #{}
"team-font-variant" #{}
"file-object-thumbnail" #{}
"file-thumbnail" #{}
"profile" #{}
"organization" #{:organization-id}
tempfile-bucket #{}
upload-session-bucket #{}
"file-data" #{:file-id :id}
"file-data-fragment" #{}
"file-change" #{}})
(def metadata-buckets
"Canonical bucket set. `app.storage/valid-buckets` aliases it so the
list lives in exactly one place."
(set (keys bucket-requirements)))
(defn- bucket-requirements-present?
[{:keys [bucket] :as mdata}]
(every? #(some? (get mdata %)) (get bucket-requirements bucket #{})))
(def schema:metadata
"Closed schema for `storage_object.metadata`. A single shape shared by
every bucket: `:bucket` is checked against `metadata-buckets` and the
rest are typed optional keys. `bucket-requirements` adds the per-bucket
presence checks."
[:and
[:map {:closed true}
[:bucket [::sm/one-of {:format :string} metadata-buckets]]
[:content-type :string]
[:hash {:optional true} :string]
[:profile-id {:optional true} ::sm/uuid]
[:organization-id {:optional true} ::sm/uuid]
[:file-id {:optional true} ::sm/uuid]
[:id {:optional true} ::sm/uuid]]
[:fn {:error/message "storage metadata is missing a required key for its bucket"}
bucket-requirements-present?]])
(defn- ->bucket
[v]
(cond
(string? v) v
(keyword? v) (d/name v)
:else (str v)))
(defn- normalize-metadata
"Bring decoded (or incoming) metadata to its canonical shape:
`:bucket` is always present (`:reference` is mapped to it, keyword
or string, and defaults to `default-bucket`), and the legacy
`:reference`, `:upload-id` and `:chunk-index` keys are dropped."
[mdata]
(let [bucket (or (not-empty (->bucket (:bucket mdata)))
(not-empty (->bucket (:reference mdata)))
default-bucket)]
(-> mdata
(dissoc :reference :upload-id :chunk-index)
(assoc :bucket bucket))))
;; Decoder and encoder are built once per process: creating them
;; compiles the closed schema, and `decode-metadata` runs on every read
;; path (get-object, dedup probes, GC batches).
(def ^:private decode-metadata-fn
(sm/decode-fn schema:metadata (sm/json-transformer)))
(def ^:private encode-metadata-fn
(sm/encoder schema:metadata (sm/json-transformer)))
(defn- transit-encoded?
"Legacy metadata is Transit json-verbose, where every top-level key is
a keyword, so the document starts with `{\"~:`. Anchoring the check to
the start avoids reading a plain-JSON document as Transit just because
one of its values contains `\"~:`."
[value]
(and (string? value)
(str/starts-with? (str/trim value) "{\"~:")))
(defn decode-metadata
"Decode storage metadata from a PGobject, accepting both the legacy
Transit encoding (sniffed by the `\"~:` marker) and plain JSON.
Always returns the normalized shape with id fields as UUIDs."
[o]
(when (some? o)
(let [raw (if (transit-encoded? (.getValue ^PGobject o))
(db/decode-transit-pgobject o)
(db/decode-json-pgobject o))]
(when-not (map? raw)
(ex/raise :type :internal
:code :invalid-storage-metadata
:hint "expected a map on storage object metadata"))
(decode-metadata-fn (normalize-metadata raw)))))
(def ^:private check-metadata
(sm/check-fn schema:metadata
:hint "invalid storage object metadata"
:type :validation
:code :invalid-storage-metadata))
(defn encode-metadata
"Validate, normalize and encode storage metadata into a PGobject
ready for the `metadata` column. Serializes as Transit while the
`:storage-metadata-as-json` config flag is unset, as plain JSON
when it is set."
[mdata]
(let [mdata (check-metadata (normalize-metadata mdata))]
(if (cf/get :storage-metadata-as-json)
(db/json (encode-metadata-fn mdata))
(db/tjson mdata))))

View File

@ -56,7 +56,17 @@
:bucket bucket
:content-type "text/plain"}
(some? profile-id)
(assoc :profile-id profile-id)))))
(assoc :profile-id profile-id)
;; file-data objects require file/id
;; references for GC (has-file-data-refs?).
(= bucket "file-data")
(assoc :file-id (uuid/random)
:id (uuid/random))
;; organization objects require their owner id.
(= bucket "organization")
(assoc :organization-id (uuid/random))))))
(defn- make-metrics
[]

View File

@ -8,6 +8,8 @@
(:require
[app.common.exceptions :as ex]
[app.common.time :as ct]
[app.common.uuid :as uuid]
[app.config :as cf]
[app.db :as db]
[app.http :as-alias http]
[app.http.access-token]
@ -21,6 +23,7 @@
[backend-tests.helpers :as th]
[clojure.string :as str]
[clojure.test :as t]
[integrant.core :as ig]
[mockery.core :refer [with-mocks]]
[yetti.request :as yreq]
[yetti.response :as yres])
@ -56,12 +59,11 @@
(handler (th/make-dummy-request {}))
(t/is (nil? (::http/auth-data @request)))
;; A bearer token is only attached when it is a current session
;; token (kid=1/ver=1) and a decoder is configured. Otherwise the
;; request stays unauthenticated.
(handler (th/make-dummy-request {:headers {"authorization" "Bearer aaaa"}}))
(let [{:keys [token claims] token-type :type} (get @request ::http/auth-data)]
(t/is (= :bearer token-type))
(t/is (= "aaaa" token))
(t/is (nil? claims)))))
(t/is (nil? (::http/auth-data @request)))))
(t/deftest auth-middleware-3
(let [request (volatile! nil)
@ -73,11 +75,7 @@
(t/is (nil? (::http/auth-data @request)))
(handler (th/make-dummy-request {:cookies {"auth-token" "foobar"}}))
(let [{:keys [token claims] token-type :type} (get @request ::http/auth-data)]
(t/is (= :cookie token-type))
(t/is (= "foobar" token))
(t/is (nil? claims)))))
(t/is (nil? (::http/auth-data @request)))))
(t/deftest shared-key-auth
(let [handler (#'app.http.middleware/wrap-shared-key-auth
@ -230,56 +228,282 @@
(t/is (ct/inst? exp) "exp should be an instant")))
(t/deftest session-token-exp-based-on-created-at
(let [cfg th/*system*
manager (session/inmemory-manager)
profile (th/create-profile* 1)
session (->> (session/create-session manager {:profile-id (:id profile)
:user-agent "user agent"})
(#'session/assign-token cfg))
claims (tokens/decode cfg (:token session))
expected-exp (ct/plus (:created-at session) (ct/duration {:days 30}))]
(let [cfg th/*system*
manager (session/inmemory-manager)
profile (th/create-profile* 1)
session (->> (session/create-session manager {:profile-id (:id profile)
:user-agent "user agent"})
(#'session/assign-token cfg))
claims (tokens/decode cfg (:token session))
expected-exp (ct/plus (:created-at session) (ct/duration {:days 30}))]
(t/is (some? (:exp claims)) "session token should contain :exp claim")
(t/is (= (inst-ms (:exp claims))
(inst-ms expected-exp))
"exp should equal created-at + 30 days")))
(t/deftest session-token-past-exp-is-rejected
(let [cfg th/*system*
manager (session/inmemory-manager)
profile (th/create-profile* 1)
session (->> (session/create-session manager {:profile-id (:id profile)
:user-agent "user agent"})
(#'session/assign-token cfg))
claims (tokens/decode cfg (:token session))
;; Manually create a token with exp in the past
(let [cfg th/*system*
manager (session/inmemory-manager)
profile (th/create-profile* 1)
session (->> (session/create-session manager {:profile-id (:id profile)
:user-agent "user agent"})
(#'session/assign-token cfg))
claims (tokens/decode cfg (:token session))
past-claims (assoc claims :exp (ct/minus (ct/now) (ct/duration {:days 1})))
header {:kid 1 :ver 1}
past-token (tokens/generate cfg past-claims header)]
past-token (tokens/generate cfg past-claims {:kid 1 :ver 1})]
(t/is (nil? (session/decode-token cfg past-token))
"token with exp in the past should be rejected")))
(t/deftest session-renewal-preserves-original-exp
(let [cfg th/*system*
manager (session/inmemory-manager)
profile (th/create-profile* 1)
created (ct/minus (ct/now) (ct/duration {:days 1}))
session {:id (uuid/random)
:profile-id (:id profile)
:user-agent "user agent"
:created-at created
:modified-at (ct/minus (ct/now) (ct/duration {:hours 7}))}
manager (reify session/ISessionManager
(read-session [_ _] session)
(create-session [_ _] session)
(update-session [_ s] (assoc s :modified-at (ct/now)))
(delete-session [_ _] nil))
old-token (:token (#'session/assign-token cfg session))
original-exp (:exp (tokens/decode cfg old-token))
handler (-> (fn [req] req)
(#'session/wrap-authz (assoc th/*system* ::session/manager manager))
(#'mw/wrap-auth {:bearer (partial session/decode-token cfg)
:cookie (partial session/decode-token cfg)}))
response (handler (th/make-dummy-request {:cookies {"auth-token" old-token}}))
renewed-token (get-in response [::yres/cookies "auth-token" :value])
renewed-claims (tokens/decode cfg renewed-token)]
(t/is (some? original-exp) "original token should have :exp")
(t/is (not= old-token renewed-token) "renewal should issue a new token string")
(t/is (= (inst-ms original-exp) (inst-ms (:exp renewed-claims)))
"renewed token should preserve the original :exp, not extend it")))
(t/deftest session-renewal-preserves-exp-with-db-manager
(let [cfg th/*system*
manager (::session/manager th/*system*)
profile (th/create-profile* 1)
created (session/create-session manager {:profile-id (:id profile)
:user-agent "user agent"})
_ (th/db-exec-one! ["UPDATE http_session_v2
SET modified_at = now() - interval '7 hours'
WHERE id = ?" (:id created)])
stale (session/read-session manager (:id created))
old-token (:token (#'session/assign-token cfg stale))
original-exp (:exp (tokens/decode cfg old-token))
handler (-> (fn [req] req)
(#'session/wrap-authz cfg)
(#'mw/wrap-auth {:bearer (partial session/decode-token cfg)
:cookie (partial session/decode-token cfg)}))
response (handler (th/make-dummy-request {:cookies {"auth-token" old-token}}))
renewed (get-in response [::yres/cookies "auth-token" :value])
renewed-exp (:exp (tokens/decode cfg renewed))
expected-exp (ct/plus (:created-at created) (ct/duration {:days 30}))
current (session/read-session manager (:id created))]
(t/is (some? original-exp) "original token should have :exp")
(t/is (some? renewed) "renewal should issue a new cookie token")
(t/is (not= old-token renewed) "renewal should issue a new token string")
(t/is (= (inst-ms original-exp) (inst-ms renewed-exp))
"renewed token should preserve the original :exp, not extend it")
(t/is (= (inst-ms expected-exp) (inst-ms renewed-exp))
"renewed :exp should equal created-at + 30 days")
(t/is (some? current) "session row must still exist after renewal")
(t/is (pos? (compare (:modified-at current) (:modified-at stale)))
"persisted modified_at must move forward on renewal")))
(t/deftest session-renewal-cookie-expires-diverges-from-token-exp
(let [cfg th/*system*
manager (::session/manager th/*system*)
profile (th/create-profile* 1)
created (session/create-session manager {:profile-id (:id profile)
:user-agent "user agent"})
_ (th/db-exec-one! ["UPDATE http_session_v2
SET created_at = now() - interval '29 days',
modified_at = now() - interval '7 hours'
WHERE id = ?" (:id created)])
stale (session/read-session manager (:id created))
old-token (:token (#'session/assign-token cfg stale))
handler (-> (fn [req] req)
(#'session/wrap-authz cfg)
(#'mw/wrap-auth {:bearer (partial session/decode-token cfg)
:cookie (partial session/decode-token cfg)}))
response (handler (th/make-dummy-request {:cookies {"auth-token" old-token}}))
cookie (get-in response [::yres/cookies "auth-token"])
renewed (:value cookie)
renewed-exp (:exp (tokens/decode cfg renewed))
expected-exp (ct/plus (:created-at stale) (ct/duration {:days 30}))
close-to? (fn [a b tolerance-ms]
(<= (Math/abs (- (inst-ms a) (inst-ms b))) tolerance-ms))]
(t/is (some? renewed) "renewal should issue a new cookie token")
(t/is (not= old-token renewed) "renewal should issue a new token string")
(t/is (= (inst-ms expected-exp) (inst-ms renewed-exp))
"renewed :exp should equal created-at + 30 days")
(t/is (close-to? renewed-exp (ct/plus (ct/now) (ct/duration {:days 1}))
(* 10 60 1000))
"renewed :exp should be ~1 day out (absolute cap is near)")
(t/is (close-to? (:expires cookie) (ct/plus (ct/now) (ct/duration {:days 7}))
(* 10 60 1000))
"cookie Expires should slide ~7 days out from now")
(t/is (pos? (compare (:expires cookie) renewed-exp))
"cookie Expires should stay ahead of the token :exp")))
(t/deftest legacy-session-token-is-rejected
(let [cfg th/*system*
manager (session/inmemory-manager)
handler (-> (fn [req] req)
(#'session/wrap-authz {::session/manager manager})
(#'session/wrap-authz {::session/manager manager})
(#'mw/wrap-auth {:bearer (partial session/decode-token cfg)
:cookie (partial session/decode-token cfg)}))
session (->> (session/create-session manager {:profile-id (:id profile)
:user-agent "user agent"})
(#'session/assign-token cfg))
original-exp (:exp (tokens/decode cfg (:token session)))
;; Force renewal by setting modified-at to 7 hours ago
old-session (assoc session :modified-at (ct/minus (ct/now) (ct/duration {:hours 7})))
response (handler (th/make-dummy-request {:cookies {"auth-token" (:token old-session)}}))
{:keys [token claims]} (get response ::http/auth-data)
new-exp (:exp claims)]
(t/is (some? original-exp) "original token should have :exp")
(t/is (some? new-exp) "renewed token should have :exp")
(t/is (= (inst-ms original-exp)
(inst-ms new-exp))
"renewed token should preserve original :exp, not extend it")))
token (tokens/generate cfg {:sid "legacy-session-id"} {:kid 0 :ver 0})
response (handler (th/make-dummy-request {:cookies {"auth-token" token}}))]
(t/is (nil? (get response ::http/auth-data))
"legacy tokens must not be attached as auth data")
(t/is (nil? (::session/profile-id response))
"legacy tokens must not authenticate")))
(t/deftest session-gc-deletes-idle-and-absolute-expired-rows
(let [profile (th/create-profile* 1)
fresh (uuid/random)
idle (uuid/random)
absolute (uuid/random)
valid (uuid/random)]
(th/db-exec-one! ["INSERT INTO http_session_v2 (id, profile_id, created_at, modified_at)
VALUES (?, ?, now(), now())"
fresh (:id profile)])
(th/db-exec-one! ["INSERT INTO http_session_v2 (id, profile_id, created_at, modified_at)
VALUES (?, ?, now() - interval '1 day', now() - interval '8 days')"
idle (:id profile)])
(th/db-exec-one! ["INSERT INTO http_session_v2 (id, profile_id, created_at, modified_at)
VALUES (?, ?, now() - interval '31 days', now())"
absolute (:id profile)])
(th/db-exec-one! ["INSERT INTO http_session_v2 (id, profile_id, created_at, modified_at)
VALUES (?, ?, now() - interval '1 day', now() - interval '6 days')"
valid (:id profile)])
(db/tx-run! th/*system*
(fn [cfg]
(#'session/collect-expired-tasks
(assoc cfg
:app.http.session.tasks/max-age (ct/duration {:days 7})
:app.http.session.tasks/max-age-absolute (ct/duration {:days 30})))))
(let [ids (->> (th/db-exec! ["SELECT id FROM http_session_v2 WHERE profile_id = ?" (:id profile)])
(map :id)
(set))]
(t/is (contains? ids fresh) "fresh session must be kept")
(t/is (contains? ids valid) "session within both windows must be kept")
(t/is (not (contains? ids idle)) "idle session must be deleted")
(t/is (not (contains? ids absolute)) "session past the absolute cap must be deleted"))))
(t/deftest session-gc-config-wiring
(let [idle (ct/duration {:days 3})
absolute (ct/duration {:days 10})]
(with-redefs [cf/get (fn
([k] (case k
:auth-token-cookie-max-age idle
:auth-token-cookie-max-age-absolute absolute
nil))
([k default] (case k
:auth-token-cookie-max-age idle
:auth-token-cookie-max-age-absolute absolute
default)))]
(let [expanded (ig/expand-key :app.http.session.tasks/gc {})]
(t/is (= idle
(get-in expanded [:app.http.session.tasks/gc
:app.http.session.tasks/max-age]))
"task max-age should carry the configured idle window")
(t/is (= absolute
(get-in expanded [:app.http.session.tasks/gc
:app.http.session.tasks/max-age-absolute]))
"task max-age-absolute should carry the configured absolute cap")))
(with-redefs [cf/get (fn
([_k] nil)
([_k default] default))]
(let [expanded (ig/expand-key :app.http.session.tasks/gc {})]
(t/is (= session/default-cookie-max-age
(get-in expanded [:app.http.session.tasks/gc
:app.http.session.tasks/max-age]))
"task max-age should fall back to the default idle window")
(t/is (= session/default-cookie-max-age-absolute
(get-in expanded [:app.http.session.tasks/gc
:app.http.session.tasks/max-age-absolute]))
"task max-age-absolute should fall back to the default absolute cap")))))
(t/deftest session-gc-rejects-absolute-below-idle
(let [pool (:app.db/pool th/*system*)
params {:app.db/pool pool
:app.http.session.tasks/max-age (ct/duration {:days 7})
:app.http.session.tasks/max-age-absolute (ct/duration {:days 30})}]
(t/is (nil? (ig/assert-key :app.http.session.tasks/gc params))
"absolute cap above the idle window should pass")
(t/is (nil? (ig/assert-key :app.http.session.tasks/gc
(assoc params
:app.http.session.tasks/max-age-absolute
(ct/duration {:days 7}))))
"absolute cap equal to the idle window should pass")
(t/is (thrown? IllegalArgumentException
(ig/assert-key :app.http.session.tasks/gc
(assoc params
:app.http.session.tasks/max-age-absolute
(ct/duration {:days 3}))))
"absolute cap below the idle window should fail fast")))
(t/deftest idle-expired-session-is-rejected
(let [cfg th/*system*
profile (th/create-profile* 1)
updated? (atom false)
session {:id (uuid/random)
:profile-id (:id profile)
:user-agent "user agent"
:created-at (ct/now)
:modified-at (ct/minus (ct/now) (ct/duration {:days 8}))}
manager (reify session/ISessionManager
(read-session [_ _] session)
(create-session [_ _] session)
(update-session [_ s] (reset! updated? true) s)
(delete-session [_ _] nil))
token (:token (#'session/assign-token cfg session))
handler (-> (fn [req] req)
(#'session/wrap-authz (assoc cfg ::session/manager manager))
(#'mw/wrap-auth {:bearer (partial session/decode-token cfg)
:cookie (partial session/decode-token cfg)}))
cookie-r (handler (th/make-dummy-request {:cookies {"auth-token" token}}))
bearer-r (handler (th/make-dummy-request {:headers {"authorization" (str "Bearer " token)}}))]
(t/is (nil? (::session/profile-id cookie-r))
"idle-expired session must not authenticate via cookie")
(t/is (nil? (::session/session cookie-r))
"idle-expired session must not be attached via cookie")
(t/is (nil? (::session/profile-id bearer-r))
"idle-expired session must not authenticate via bearer")
(t/is (false? @updated?)
"idle-expired session must not be renewed")))
(t/deftest idle-session-within-window-is-accepted
(let [cfg th/*system*
profile (th/create-profile* 1)
session {:id (uuid/random)
:profile-id (:id profile)
:user-agent "user agent"
:created-at (ct/now)
:modified-at (ct/minus (ct/now) (ct/duration {:days 6}))}
manager (reify session/ISessionManager
(read-session [_ _] session)
(create-session [_ _] session)
(update-session [_ s] (assoc s :modified-at (ct/now)))
(delete-session [_ _] nil))
token (:token (#'session/assign-token cfg session))
handler (-> (fn [req] req)
(#'session/wrap-authz (assoc cfg ::session/manager manager))
(#'mw/wrap-auth {:bearer (partial session/decode-token cfg)
:cookie (partial session/decode-token cfg)}))
response (handler (th/make-dummy-request {:cookies {"auth-token" token}}))]
(t/is (= (:id profile) (::session/profile-id response))
"session within the idle window must authenticate")))
(t/deftest parse-request-illegal-argument-exception
;; clojure.data.json raises IllegalArgumentException (case

View File

@ -85,8 +85,7 @@
(configure-storage-backend))
sobject (sto/put-object! storage {::sto/content (sto/content "content")
:content-type "text/plain"
:other "data"})
:content-type "text/plain"})
profile (th/create-profile* 1 {:is-active true})
project (th/create-project* 1 {:team-id (:default-team-id profile)
:profile-id (:id profile)})
@ -157,8 +156,7 @@
(let [storage (-> (:app.storage/storage th/*system*)
(configure-storage-backend))
sobject (sto/put-object! storage {::sto/content (sto/content "content")
:content-type "text/plain"
:other "data"})
:content-type "text/plain"})
profile (th/create-profile* 1 {:is-active true})
project (th/create-project* 1 {:team-id (:default-team-id profile)
@ -214,8 +212,7 @@
(configure-storage-backend))
sobject (sto/put-object! storage {::sto/content (sto/content "content")
:content-type "text/plain"
:other "data"})
:content-type "text/plain"})
profile (th/create-profile* 1 {:is-active true})
project (th/create-project* 1 {:team-id (:default-team-id profile)
@ -278,8 +275,7 @@
(let [storage (-> (:app.storage/storage th/*system*)
(configure-storage-backend))
sobject (sto/put-object! storage {::sto/content (sto/content "content")
:content-type "text/plain"
:other "data"})
:content-type "text/plain"})
profile (th/create-profile* 1 {:is-active true})
project (th/create-project* 1 {:team-id (:default-team-id profile)
:profile-id (:id profile)})

View File

@ -0,0 +1,276 @@
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; Copyright (c) KALEIDOS SUBSIDIARY SL
(ns backend-tests.storage-metadata-test
(:require
[app.config :as cf]
[app.db :as db]
[app.storage.schema :as stsch]
[clojure.string :as str]
[clojure.test :as t])
(:import
org.postgresql.util.PGobject))
(defn- pgobject
[^String value]
(doto (PGobject.)
(.setType "jsonb")
(.setValue value)))
;; Raw Transit payloads shaped like the production rows sampled on
;; 2026-09-11 (verbose transit: "~:key" keys, "~u<uuid>" uuid values,
;; "~:xxx" keyword values).
(def ^:private transit-media
"{\"~:hash\":\"blake2b:9f1c2e\",\"~:bucket\":\"file-media-object\",\"~:content-type\":\"image/png\"}")
(def ^:private transit-tempfile
"{\"~:hash\":\"blake2b:9f1c2e\",\"~:bucket\":\"tempfile\",\"~:content-type\":\"application/zip\",\"~:profile-id\":\"~u86907e95-1cb8-8122-8008-4eb7ba07d89d\"}")
(def ^:private transit-legacy-reference
"{\"~:reference\":\"~:file-media-object\",\"~:content-type\":\"image/png\",\"~:hash\":\"blake2b:9f1c2e\"}")
(def ^:private transit-no-bucket
"{\"~:content-type\":\"image/svg+xml\"}")
(def ^:private transit-chunk-leftovers
"{\"~:bucket\":\"tempfile\",\"~:content-type\":\"application/zip\",\"~:upload-id\":\"~u86907e95-1cb8-8122-8008-4eb7ba07d89d\",\"~:chunk-index\":3}")
(def ^:private json-file-data
"{\"bucket\":\"file-data\",\"content-type\":\"application/octet-stream\",\"file-id\":\"86907e95-1cb8-8122-8008-4eb7ba07d89d\",\"id\":\"83df2f92-6bd4-4e6d-9c9a-3f6d2b1a4c55\"}")
(t/deftest decode-transit-returns-native-types
(let [mdata (stsch/decode-metadata (pgobject transit-tempfile))]
(t/is (= "tempfile" (:bucket mdata)))
(t/is (= "application/zip" (:content-type mdata)))
(t/is (= "blake2b:9f1c2e" (:hash mdata)))
(t/is (uuid? (:profile-id mdata)))
(t/is (= (parse-uuid "86907e95-1cb8-8122-8008-4eb7ba07d89d")
(:profile-id mdata)))))
(t/deftest decode-transit-maps-reference-to-bucket
(let [mdata (stsch/decode-metadata (pgobject transit-legacy-reference))]
(t/is (= "file-media-object" (:bucket mdata)))
(t/is (= "image/png" (:content-type mdata)))
(t/is (not (contains? mdata :reference)))))
(t/deftest decode-transit-defaults-missing-bucket
(let [mdata (stsch/decode-metadata (pgobject transit-no-bucket))]
(t/is (= "file-media-object" (:bucket mdata)))
(t/is (= "image/svg+xml" (:content-type mdata)))))
(t/deftest decode-transit-drops-chunk-leftovers
(let [mdata (stsch/decode-metadata (pgobject transit-chunk-leftovers))]
(t/is (= "tempfile" (:bucket mdata)))
(t/is (not (contains? mdata :upload-id)))
(t/is (not (contains? mdata :chunk-index)))))
(t/deftest decode-json-coerces-uuids
(let [mdata (stsch/decode-metadata (pgobject json-file-data))]
(t/is (= "file-data" (:bucket mdata)))
(t/is (uuid? (:file-id mdata)))
(t/is (uuid? (:id mdata)))
(t/is (= (parse-uuid "86907e95-1cb8-8122-8008-4eb7ba07d89d")
(:file-id mdata)))))
(t/deftest sniff-has-no-false-positives-on-values
;; "~:" inside a value (not anchored to a quote) must not trigger
;; the transit branch.
(let [mdata (stsch/decode-metadata
(pgobject "{\"bucket\":\"tempfile\",\"content-type\":\"text/plain;~:x\"}"))]
(t/is (= "tempfile" (:bucket mdata)))
(t/is (= "text/plain;~:x" (:content-type mdata)))))
(t/deftest encode-writes-transit-by-default
;; Pinned off: without the binding this test inherits the ambient
;; config and proves nothing where the flag is set.
(binding [cf/config (assoc cf/config :storage-metadata-as-json nil)]
(let [encoded (stsch/encode-metadata {:bucket "file-media-object"
:content-type "image/png"
:hash "blake2b:9f1c2e"})
value (.getValue ^PGobject encoded)]
(t/is (string? value))
(t/is (str/includes? value "\"~:bucket\""))
(t/is (not (str/includes? value "\"~:reference\""))))))
(t/deftest encode-normalizes-legacy-input
(binding [cf/config (assoc cf/config :storage-metadata-as-json nil)]
(let [encoded (stsch/encode-metadata {:reference :file-media-object
:content-type "image/png"})
value (.getValue ^PGobject encoded)
decoded (stsch/decode-metadata encoded)]
(t/is (= "file-media-object" (:bucket decoded)))
(t/is (not (str/includes? value "\"~:reference\""))))))
(t/deftest encode-writes-plain-json-with-flag
(binding [cf/config (assoc cf/config :storage-metadata-as-json true)]
(let [file-id (parse-uuid "86907e95-1cb8-8122-8008-4eb7ba07d89d")
encoded (stsch/encode-metadata {:bucket "file-data"
:content-type "application/octet-stream"
:file-id file-id
:id (parse-uuid "83df2f92-6bd4-4e6d-9c9a-3f6d2b1a4c55")})
value (.getValue ^PGobject encoded)]
(t/is (str/includes? value "\"bucket\""))
(t/is (not (str/includes? value "\"~:")))
;; uuids travel as plain strings on the JSON encoding
(t/is (str/includes? value (str file-id)))
(let [decoded (stsch/decode-metadata encoded)]
(t/is (uuid? (:file-id decoded)))
(t/is (= file-id (:file-id decoded)))))))
(t/deftest encode-rejects-string-uuids
;; Encoding does not coerce input types: callers must pass native UUIDs
;; (every producer does; reads still coerce on the way back).
(t/is (thrown-with-msg? clojure.lang.ExceptionInfo
#"invalid storage object metadata"
(stsch/encode-metadata {:bucket "tempfile"
:content-type "application/zip"
:profile-id "86907e95-1cb8-8122-8008-4eb7ba07d89d"}))))
(t/deftest encode-rejects-unknown-bucket
(t/is (thrown-with-msg? clojure.lang.ExceptionInfo
#"invalid storage object metadata"
(stsch/encode-metadata {:bucket "no-such-bucket"
:content-type "image/png"}))))
(t/deftest encode-rejects-extra-key
(t/is (thrown-with-msg? clojure.lang.ExceptionInfo
#"invalid storage object metadata"
(stsch/encode-metadata {:bucket "file-media-object"
:content-type "image/png"
:other "data"}))))
(t/deftest encode-rejects-malformed-uuid
(t/is (thrown-with-msg? clojure.lang.ExceptionInfo
#"invalid storage object metadata"
(stsch/encode-metadata {:bucket "file-data"
:content-type "application/octet-stream"
:file-id "not-a-uuid"
:id "83df2f92-6bd4-4e6d-9c9a-3f6d2b1a4c55"}))))
(t/deftest encode-rejects-missing-content-type
(t/is (thrown-with-msg? clojure.lang.ExceptionInfo
#"invalid storage object metadata"
(stsch/encode-metadata {:bucket "file-media-object"}))))
(t/deftest encode-rejects-missing-file-data-ids
;; Both ids are required for file-data; omitting one fails the bucket
;; requirement check (the file-id here is a native UUID, so the map
;; itself is valid).
(t/is (thrown-with-msg? clojure.lang.ExceptionInfo
#"invalid storage object metadata"
(stsch/encode-metadata {:bucket "file-data"
:content-type "application/octet-stream"
:file-id (parse-uuid "86907e95-1cb8-8122-8008-4eb7ba07d89d")}))))
(t/deftest encode-rejects-missing-organization-id
(t/is (thrown-with-msg? clojure.lang.ExceptionInfo
#"invalid storage object metadata"
(stsch/encode-metadata {:bucket "organization"
:content-type "image/svg+xml"}))))
(t/deftest decode-real-transit-payload
;; Same legacy shape as above but produced by the real transit
;; encoder, so the test breaks if the encoding ever drifts.
(let [mdata (stsch/decode-metadata
(db/tjson {:reference :team-font-variant
:content-type "font/woff2"}))]
(t/is (= "team-font-variant" (:bucket mdata)))
(t/is (not (contains? mdata :reference)))))
(t/deftest transit-and-json-encodings-decode-to-same-shape
(let [mdata {:bucket "tempfile"
:content-type "application/zip"
:profile-id (parse-uuid "86907e95-1cb8-8122-8008-4eb7ba07d89d")}
transit (binding [cf/config (assoc cf/config :storage-metadata-as-json nil)]
(stsch/decode-metadata (stsch/encode-metadata mdata)))
json (binding [cf/config (assoc cf/config :storage-metadata-as-json true)]
(stsch/decode-metadata (stsch/encode-metadata mdata)))]
(t/is (= transit json))
(t/is (= mdata json))))
(t/deftest flag-on-and-off-differ-only-in-encoding-family
;; The Phase 2 rollback contract: flipping the flag changes how the
;; same logical metadata hits the disk, never what it means.
(let [mdata {:bucket "file-media-object"
:content-type "image/png"
:hash "blake2b:9f1c2e"}
transit (binding [cf/config (assoc cf/config :storage-metadata-as-json nil)]
(.getValue ^PGobject (stsch/encode-metadata mdata)))
json (binding [cf/config (assoc cf/config :storage-metadata-as-json true)]
(.getValue ^PGobject (stsch/encode-metadata mdata)))]
(t/is (str/includes? transit "\"~:bucket\""))
(t/is (str/includes? json "\"bucket\""))
(t/is (not (str/includes? json "\"~:")))
(t/is (= (stsch/decode-metadata (pgobject transit))
(stsch/decode-metadata (pgobject json))))))
(t/deftest decode-json-keeps-hash-byte-exact
;; Dedup matches on the hash string; it must survive the JSON
;; roundtrip untouched.
(let [mdata (stsch/decode-metadata
(pgobject "{\"bucket\":\"file-media-object\",\"content-type\":\"image/png\",\"hash\":\"blake2b:9f1c2e\"}"))]
(t/is (= "blake2b:9f1c2e" (:hash mdata)))))
(def ^:private bucket-samples
;; One representative payload per bucket, with every required key and
;; one optional key where the bucket declares any.
[["file-media-object"
{:bucket "file-media-object" :content-type "image/png" :hash "blake2b:9f1c2e"}]
["team-font-variant"
{:bucket "team-font-variant" :content-type "font/woff2"}]
["file-object-thumbnail"
{:bucket "file-object-thumbnail" :content-type "image/png"}]
["file-thumbnail"
{:bucket "file-thumbnail" :content-type "image/png"}]
["profile"
{:bucket "profile" :content-type "image/png"}]
["organization"
{:bucket "organization" :content-type "image/svg+xml"
:organization-id #uuid "11111111-2222-3333-4444-555555555555"}]
["tempfile"
{:bucket "tempfile" :content-type "application/zip"
:profile-id #uuid "86907e95-1cb8-8122-8008-4eb7ba07d89d"}]
["upload-session"
{:bucket "upload-session" :content-type "application/octet-stream"}]
["file-data"
{:bucket "file-data" :content-type "application/octet-stream"
:file-id #uuid "86907e95-1cb8-8122-8008-4eb7ba07d89d"
:id #uuid "83df2f92-6bd4-4e6d-9c9a-3f6d2b1a4c55"}]
["file-data-fragment"
{:bucket "file-data-fragment" :content-type "application/octet-stream"}]
["file-change"
{:bucket "file-change" :content-type "application/octet-stream"}]])
(t/deftest encode-decode-roundtrips-every-bucket
(binding [cf/config (assoc cf/config :storage-metadata-as-json nil)]
(doseq [[bucket mdata] bucket-samples]
(t/testing bucket
(t/is (= mdata (stsch/decode-metadata (stsch/encode-metadata mdata))))))))
(t/deftest json-encoding-roundtrips-every-bucket
(binding [cf/config (assoc cf/config :storage-metadata-as-json true)]
(doseq [[bucket mdata] bucket-samples]
(t/testing bucket
(t/is (= mdata (stsch/decode-metadata (stsch/encode-metadata mdata))))))))
(t/deftest metadata-buckets-is-the-canonical-set
(t/is (= #{"file-media-object" "team-font-variant" "file-object-thumbnail"
"file-thumbnail" "profile" "organization" "tempfile"
"upload-session" "file-data" "file-data-fragment" "file-change"}
stsch/metadata-buckets)))
(t/deftest decode-metadata-returns-nil-for-nil
(t/is (nil? (stsch/decode-metadata nil))))
(t/deftest sniff-treats-leading-tilde-value-as-json
;; A plain-JSON value starting with `~:` must not switch the reader to
;; the Transit branch: the sniff is anchored to the first key.
(let [encoded (binding [cf/config (assoc cf/config :storage-metadata-as-json true)]
(stsch/encode-metadata {:bucket "tempfile"
:content-type "~:not-transit"}))
decoded (stsch/decode-metadata encoded)]
(t/is (= "~:not-transit" (:content-type decoded)))
(t/is (= "tempfile" (:bucket decoded)))))

View File

@ -9,12 +9,16 @@
[app.common.exceptions :as ex]
[app.common.time :as ct]
[app.common.uuid :as uuid]
[app.config :as cf]
[app.db :as db]
[app.metrics :as mtx]
[app.metrics.definition :as-alias mdef]
[app.rpc :as-alias rpc]
[app.storage :as sto]
[app.storage.fs :as-alias sto.fs]
[app.storage.impl :as impl]
[app.storage.s3 :as-alias sto.s3]
[app.storage.schema :as stsch]
[backend-tests.helpers :as th]
[clojure.test :as t]
[cuerdas.core :as str]
@ -23,6 +27,11 @@
[mockery.core :refer [with-mocks]]
[promesa.core :as p])
(:import
(io.prometheus.client
Counter
Counter$Child)
(org.postgresql.util
PGobject)
(software.amazon.awssdk.services.s3
S3AsyncClient)
(software.amazon.awssdk.services.s3.model
@ -41,24 +50,51 @@
[storage]
(assoc storage ::sto/backend :fs))
(defn- poison-counter-value
"Read the process-wide gc-poison counter from the test system metrics."
[]
(let [metrics (:app.metrics/metrics th/*system*)
collector (mtx/get-collector metrics :storage-gc-poison)
instance (::mdef/instance collector)]
(.get ^Counter$Child (.labels ^Counter instance (into-array String [])))))
(t/deftest put-and-retrieve-object
(let [storage (-> (:app.storage/storage th/*system*)
(configure-storage-backend))
content (sto/content "content")
object (sto/put-object! storage {::sto/content content
:content-type "text/plain"
:other "data"})]
:content-type "text/plain"})]
(t/is (sto/object? object))
(t/is (fs/path? (sto/get-object-path storage object)))
(t/is (nil? (:expired-at object)))
(t/is (= :fs (:backend object)))
(t/is (= "data" (:other (meta object))))
(t/is (= "file-media-object" (:bucket (meta object))))
(t/is (= "text/plain" (:content-type (meta object))))
(t/is (= "content" (slurp (sto/get-object-data storage object))))
(t/is (= "content" (slurp (sto/get-object-path storage object))))))
(t/deftest put-and-retrieve-with-json-metadata-flag
;; End-to-end through the jsonb column: with the flag on, the row is
;; stored as plain JSON and still decodes to native types.
(binding [cf/config (assoc cf/config :storage-metadata-as-json true)]
(let [storage (-> (:app.storage/storage th/*system*)
(configure-storage-backend))
profile (uuid/random)
object (sto/put-object! storage {::sto/content (sto/content "content")
:bucket "tempfile"
:content-type "application/zip"
:profile-id profile})
row (th/db-exec-one! ["select metadata::text as metadata from storage_object where id = ?"
(:id object)])]
(t/is (not (str/includes? (:metadata row) "\"~:")))
(t/is (str/includes? (:metadata row) "\"bucket\""))
(let [loaded (sto/get-object storage (:id object))]
(t/is (= "tempfile" (:bucket (meta loaded))))
(t/is (= "application/zip" (:content-type (meta loaded))))
(t/is (= profile (:profile-id (meta loaded))))))))
(t/deftest tempfile-objects-are-not-deduplicated
(let [storage (-> (:app.storage/storage th/*system*)
(configure-storage-backend))
@ -94,8 +130,7 @@
(configure-storage-backend))
content (sto/content "content")
object (sto/put-object! storage {::sto/content content
:content-type "text/plain"
:expired-at (ct/in-future {:seconds 1})})]
:content-type "text/plain"})]
(t/is (sto/object? object))
(t/is (true? (sto/del-object! storage (:id object))))
@ -109,6 +144,18 @@
;; marked as deleted/expired.
(t/is (nil? (sto/get-object storage (:id object))))))
(t/deftest delete-expired-object-returns-false
;; An object stored with `::sto/expired-at` is born with `deleted_at`
;; set, so deleting it is a no-op and must report `false`.
(let [storage (-> (:app.storage/storage th/*system*)
(configure-storage-backend))
content (sto/content "content")
object (sto/put-object! storage {::sto/content content
:content-type "text/plain"
::sto/expired-at (ct/in-future {:hours 1})})]
(t/is (some? (:expired-at object)))
(t/is (false? (sto/del-object! storage (:id object))))))
(t/deftest deleted-gc-task
(let [storage (-> (:app.storage/storage th/*system*)
(configure-storage-backend))
@ -438,6 +485,146 @@
(let [row (th/db-exec-one! ["select deleted_at from storage_object where id = ?" (:id object1)])]
(t/is (ct/is-before-or-equal? (:deleted-at row) (ct/plus now {:seconds 1}))))))
(t/deftest storage-gc-touched-null-metadata
;; A NULL metadata column (predates any normalization) flows through
;; the lookup fallback with a warning instead of breaking the GC loop.
(let [storage (-> (:app.storage/storage th/*system*)
(configure-storage-backend))
content (sto/content "content")
object (sto/put-object! storage {::sto/content content
::sto/touch true
:content-type "text/plain"})]
(th/db-exec! ["update storage_object set metadata = null where id = ?" (:id object)])
(let [res (th/run-task! :storage-gc-touched {:skip-delay true})]
(t/is (= 0 (:freeze res)))
(t/is (= 1 (:delete res))))))
(t/deftest storage-gc-touched-defers-corrupt-metadata
;; A non-map metadata row neither blocks the chunk nor gets
;; collected: it is logged and deferred exactly one day, keeping its
;; metadata intact for a later repair.
(let [now (ct/now)
before (poison-counter-value)
storage (-> (:app.storage/storage th/*system*)
(configure-storage-backend))
healthy (sto/put-object! storage {::sto/content (sto/content "healthy")
::sto/touched-at now
:content-type "text/plain"})
poison (uuid/random)]
(th/db-exec! ["insert into storage_object (id, backend, metadata, touched_at) values (?, 'fs', '[]'::jsonb, ?)"
poison now])
(binding [ct/*clock* (ct/fixed-clock now)]
(let [res (th/run-task! :storage-gc-touched {:skip-delay true})]
(t/is (= 0 (:freeze res)))
(t/is (= 1 (:delete res)))))
(let [row (th/db-exec-one! ["select metadata::text as metadata, touched_at from storage_object where id = ?" poison])]
(t/is (= "[]" (:metadata row)))
;; inst-ms: timestamptz keeps micros, the frozen clock has nanos.
(t/is (= (inst-ms (ct/plus now {:days 1}))
(inst-ms (:touched-at row)))))
;; One poison row observed, one increment.
(t/is (= (inc before) (poison-counter-value)))))
(t/deftest storage-gc-touched-poison-only
;; A chunk made only of poison rows still terminates: the rows are
;; deferred, the next scan returns nothing, and no delete is counted.
(let [now (ct/now)
poison (uuid/random)]
(th/db-exec! ["insert into storage_object (id, backend, metadata, touched_at) values (?, 'fs', '[]'::jsonb, ?)"
poison now])
(binding [ct/*clock* (ct/fixed-clock now)]
(let [res (th/run-task! :storage-gc-touched {:skip-delay true})]
(t/is (= 0 (:freeze res)))
(t/is (= 0 (:delete res)))))
(let [row (th/db-exec-one! ["select touched_at from storage_object where id = ?" poison])]
(t/is (= (inst-ms (ct/plus now {:days 1}))
(inst-ms (:touched-at row)))))))
(t/deftest storage-gc-touched-drains-past-a-poison-only-batch
;; A full batch of poison rows must not stop the run: they are deferred
;; and the loop keeps draining, so a healthy row queued behind the
;; LIMIT (later touched_at) still gets collected.
(let [now (ct/now)
storage (-> (:app.storage/storage th/*system*)
(configure-storage-backend))
earlier (ct/minus now {:seconds 1})
healthy (sto/put-object! storage {::sto/content (sto/content "healthy")
::sto/touched-at now
:content-type "text/plain"})
poisons (repeatedly 10 uuid/random)]
(doseq [id poisons]
(th/db-exec! ["insert into storage_object (id, backend, metadata, touched_at) values (?, 'fs', '[]'::jsonb, ?)"
id earlier]))
(binding [ct/*clock* (ct/fixed-clock now)]
(let [res (th/run-task! :storage-gc-touched {:skip-delay true})]
(t/is (= 0 (:freeze res)))
(t/is (= 1 (:delete res)))))
(doseq [id poisons]
(let [row (th/db-exec-one! ["select touched_at from storage_object where id = ?" id])]
(t/is (= (inst-ms (ct/plus now {:days 1}))
(inst-ms (:touched-at row))))))))
(def ^:private migration-0155-fixtures
;; [id transit-metadata]: production-shaped legacy rows (nil payload
;; means a NULL column).
[["11111111-1111-1111-1111-111111111111"
"{\"~:reference\":\"~:file-media-object\",\"~:content-type\":\"image/png\",\"~:hash\":\"blake2b:aaa\"}"]
["22222222-2222-2222-2222-222222222222"
"{\"~:content-type\":\"image/svg+xml\"}"]
["33333333-3333-3333-3333-333333333333"
"{\"~:bucket\":\"tempfile\",\"~:reference\":\"~:tempfile\",\"~:content-type\":\"application/zip\"}"]
["44444444-4444-4444-4444-444444444444"
"{\"~:bucket\":\"tempfile\",\"~:content-type\":\"application/zip\",\"~:upload-id\":\"~u86907e95-1cb8-8122-8008-4eb7ba07d89d\",\"~:chunk-index\":3}"]
["55555555-5555-5555-5555-555555555555"
nil]])
(defn- run-migration-0155!
;; Re-runs the 0155 statements (not migratus: it already applied at
;; bootstrap) over the fixture rows above.
[]
(let [sql (-> (io/resource "app/migrations/sql/0155-normalize-storage-object-metadata.sql")
(slurp))
no-comments (->> (.split ^String sql "\n")
(remove #(.startsWith ^String (str/trim %) "--"))
(str/join "\n"))]
(doseq [stmt (->> (.split ^String no-comments ";")
(map str/trim)
(remove str/blank?))]
(th/db-exec! [stmt]))))
(defn- get-metadata-by-id
[id]
(:metadata (th/db-exec-one! ["select metadata from storage_object where id = ?"
(parse-uuid id)])))
(t/deftest storage-migration-0155-normalizes-legacy-rows
(doseq [[id mdata] migration-0155-fixtures]
(th/db-exec! ["insert into storage_object (id, backend, metadata) values (?, 'fs', ?::jsonb)"
(parse-uuid id) mdata]))
(run-migration-0155!)
(let [mdata (fn [id] (stsch/decode-metadata (get-metadata-by-id id)))]
(t/is (= {:bucket "file-media-object"
:content-type "image/png"
:hash "blake2b:aaa"}
(mdata "11111111-1111-1111-1111-111111111111")))
(t/is (= {:bucket "file-media-object"
:content-type "image/svg+xml"}
(mdata "22222222-2222-2222-2222-222222222222")))
(t/is (= {:bucket "tempfile"
:content-type "application/zip"}
(mdata "33333333-3333-3333-3333-333333333333")))
(t/is (= {:bucket "tempfile"
:content-type "application/zip"}
(mdata "44444444-4444-4444-4444-444444444444")))
(t/is (= {:bucket "file-media-object"}
(mdata "55555555-5555-5555-5555-555555555555"))))
;; second run changes nothing (idempotent)
(let [raw (fn [] (mapv #(.getValue ^PGobject (get-metadata-by-id %))
(map first migration-0155-fixtures)))
before (raw)]
(run-migration-0155!)
(t/is (= before (raw)))))
(t/deftest storage-gc-deleted-immediate
(let [storage (-> (:app.storage/storage th/*system*)
(configure-storage-backend))
@ -626,6 +813,43 @@
(let [row (th/db-exec-one! ["select count(*) from storage_object"])]
(t/is (= 1 (:count row))))))
(t/deftest dedup-reuses-json-encoded-blob
;; With the JSON flag on, the dedup lookup must find the row it just
;; wrote; a "~:"-only lookup used to miss it and duplicate the blob.
(binding [cf/config (assoc cf/config :storage-metadata-as-json true)]
(let [storage (-> (:app.storage/storage th/*system*)
(configure-storage-backend))
content (-> (sto/content "json-content")
(sto/wrap-with-hash "json-hash"))
params {::sto/content content
::sto/deduplicate? true
:bucket "file-media-object"
:content-type "text/plain"}
object1 (sto/put-object! storage params)
object2 (sto/put-object! storage params)]
(t/is (= (:id object1) (:id object2)))
(let [row (th/db-exec-one! ["select count(*) from storage_object"])]
(t/is (= 1 (:count row)))))))
(t/deftest dedup-json-put-finds-transit-row
;; Both encodings coexist during the transition: a JSON write must
;; reuse a blob already stored as Transit.
(let [storage (-> (:app.storage/storage th/*system*)
(configure-storage-backend))
content (-> (sto/content "mixed-content")
(sto/wrap-with-hash "mixed-hash"))
params {::sto/content content
::sto/deduplicate? true
:bucket "file-media-object"
:content-type "text/plain"}
object1 (binding [cf/config (assoc cf/config :storage-metadata-as-json nil)]
(sto/put-object! storage params))
object2 (binding [cf/config (assoc cf/config :storage-metadata-as-json true)]
(sto/put-object! storage params))]
(t/is (= (:id object1) (:id object2)))
(let [row (th/db-exec-one! ["select count(*) from storage_object"])]
(t/is (= 1 (:count row))))))
(t/deftest dedup-repairs-stale-object
(let [storage (-> (:app.storage/storage th/*system*)
(configure-storage-backend))

View File

@ -643,6 +643,44 @@ And configure it:
PENPOT_SECRET_KEY: my-super-secure-key
```
### Session expiration
__Since version 2.19.0__
User sessions are stored server-side and expire on two independent conditions: an
**idle timeout** and an **absolute maximum lifetime**. Both are backend only.
```bash
# Backend
# Idle timeout: the session stops working after this much inactivity.
# Default: 168h
PENPOT_AUTH_TOKEN_COOKIE_MAX_AGE: 168h
# Absolute maximum lifetime from the moment the session was created,
# regardless of activity. Default: 720h
PENPOT_AUTH_TOKEN_COOKIE_MAX_AGE_ABSOLUTE: 720h
```
Durations use the `<number><unit>` form with hour, minute or second units,
for example `168h`, `30m` or `90s` (day units like `7d` are not accepted).
While a user is active the session is automatically renewed every 6 hours (not
configurable). Renewal extends the cookie, but never the absolute maximum. The
backend checks the idle timeout on every request, measured from the last
renewal: once it passes, the session stops working at once, even for a copied
token. A daily task (`session-gc`) then deletes the sessions that have exceeded
either window. Legacy v1 sessions and the old `http_session` table are no
longer used.
Sessions created before 2.19.0 carry no `:exp` in their token; they are still
removed by the 30-day `created_at` cleanup and acquire `:exp` on their next
renewal.
The `secure` and `same-site` attributes of the session cookie are controlled by
the `disable-secure-session-cookies`, `strict-session-cookies` and `enable-cors`
flags, and by whether `PENPOT_PUBLIC_URI` is served over HTTPS. See
[Penpot URI](#penpot-uri).
### Database
Penpot only supports PostgreSQL and we highly recommend >=13 version. If you are using official

View File

@ -94,14 +94,45 @@ Similarly as the OIDC backend, it checks if the profile exists, and calls
## Sessions
User sessions are created when a user logs in via any one of the backends. A
session token is generated (a JWT token that does not currently contain any data)
and returned to frontend as a cookie.
User sessions are created when a user logs in via any one of the backends. The
backend generates a signed JWT token and returns it to the frontend as an
<code class="language-text">auth-token</code> cookie. A matching row is stored in
the <code class="language-text">http_session_v2</code> table with the profile id
and the session timestamps.
Normally the session is stored in a DB table with the information of the user
profile and the session expiration. But if a frontend connects to the backend in
"read only" mode (for example, to debug something in production with the local
devenv), sessions are stored in memory (may be lost if the backend restarts).
A request is authenticated only when the token verifies, its session row still
exists and the session is within the idle timeout. The token claims carry the session row id (<code
class="language-clojure">:sid</code>), the last activity instant (<code
class="language-clojure">:iat</code>) and an absolute expiration (<code
class="language-clojure">:exp</code>). The server enforces two independent
limits:
* **Idle timeout:** a session that is not renewed within
<code class="language-bash">PENPOT_AUTH_TOKEN_COOKIE_MAX_AGE</code> (default 7
days) stops working at once. The backend checks the session's last renewal
time on every request, so the row does not need to be deleted first.
* **Absolute maximum:** a session cannot live longer than
<code class="language-bash">PENPOT_AUTH_TOKEN_COOKIE_MAX_AGE_ABSOLUTE</code>
(default 30 days) from its creation, no matter how much it is renewed. The
<code class="language-clojure">:exp</code> claim enforces this even when the
cookie is still present.
Sessions are automatically renewed every 6 hours of use (not configurable).
Renewal issues a new token but keeps the same session row, so the absolute
maximum is not extended. A daily garbage collector
(<code class="language-text">session-gc</code>) deletes rows that exceed either
the idle window or the absolute maximum.
Sessions created before 2.19.0 carry no <code
class="language-clojure">:exp</code> in their token; they are still removed by
the 30-day <code class="language-text">created_at</code> cleanup and acquire <code
class="language-clojure">:exp</code> on their next renewal.
The normal storage is the database. When the backend uses a read-only database
pool (for example, to debug something in production with the local devenv),
sessions are kept in memory and are lost when the backend restarts. The
organization SSO gate keeps an additional 4-hour entry inside the same session
row, separate from the session token lifetime.
## Team invitations

View File

@ -54,8 +54,12 @@
(let [el (dom/get-element "app")]
(mf/create-root el)))
;; Root component of the last render; a hot reload of `app.main.ui` replaces it
(defonce ^:private rendered-app (atom nil))
(defn init-ui
[]
(reset! rendered-app ui/app)
(mf/render! app-root (mf/element ui/app)))
(defn- initialize-rasterizer
@ -146,7 +150,10 @@
(init-ui)))
(defn ^:dev/after-load after-load
"Rerender the UI after a hot reload. When the reload replaced the root
component, unmount the whole tree first so its cleanups run before
the new tree mounts."
[]
(reinit))
(reinit (not (identical? @rendered-app ui/app))))
(set! (.-stackTraceLimit js/Error) 50)

View File

@ -1248,15 +1248,26 @@
(not (r/check-permission plugin-id "library:write"))
(u/reject-not-valid reject :connectLibrary "Plugin doesn't have 'library:write' permission")
(not (string? library-id))
(not (uuid/parse* library-id))
(u/reject-not-valid reject :connectLibrary library-id)
:else
(let [file-id (:current-file-id @st/state)
(let [file-id (:current-file-id @st/state)
team-id (:current-team-id @st/state)
library-id (uuid/parse library-id)]
(->> st/stream
(rx/filter (ptk/type? ::dwl/link-file-to-library-finished))
(rx/take 1)
(rx/subs! #(resolve (library-proxy plugin-id library-id)) reject))
(st/emit! (-> (dwl/link-file-to-library file-id library-id)
(se/add-event plugin-id))))))))))
(->> (rp/cmd! :get-team-shared-files {:team-id team-id})
(rx/subs!
(fn [shared-files]
;; Only published libraries of the team can be linked
(if (or (= library-id file-id)
(not (some #(= library-id (:id %)) shared-files)))
(u/reject-not-valid reject :connectLibrary
(dm/str library-id " is not a published library"))
(do
(->> st/stream
(rx/filter (ptk/type? ::dwl/link-file-to-library-finished))
(rx/take 1)
(rx/subs! #(resolve (library-proxy plugin-id library-id)) reject))
(st/emit! (-> (dwl/link-file-to-library file-id library-id)
(se/add-event plugin-id))))))
reject)))))))))

View File

@ -2479,6 +2479,13 @@
:cause cause)
nil)))))
;; Stable listener identities, so `removeEventListener` matches across hot reloads
(defonce ^:private webgl-context-lost-listener
(fn [event] (on-webgl-context-lost event)))
(defonce ^:private webgl-context-restored-listener
(fn [event] (on-webgl-context-restored event)))
(defn init-canvas-context
[canvas]
(if-not (wasm/module-ready?)
@ -2523,8 +2530,8 @@
;; Add event listeners for WebGL context lost
(set! wasm/canvas canvas)
(when can-listen?
(.addEventListener canvas "webglcontextlost" on-webgl-context-lost)
(.addEventListener canvas "webglcontextrestored" on-webgl-context-restored))
(.addEventListener canvas "webglcontextlost" webgl-context-lost-listener)
(.addEventListener canvas "webglcontextrestored" webgl-context-restored-listener))
(start-canvas-snapshot-listener!)
(reset! wasm/context-lost? false)
(set! wasm/context-initialized? true)
@ -2558,8 +2565,8 @@
;; Remove listener before losing/deleting context.
(when wasm/canvas
(.removeEventListener wasm/canvas "webglcontextlost" on-webgl-context-lost)
(.removeEventListener wasm/canvas "webglcontextrestored" on-webgl-context-restored))
(.removeEventListener wasm/canvas "webglcontextlost" webgl-context-lost-listener)
(.removeEventListener wasm/canvas "webglcontextrestored" webgl-context-restored-listener))
(stop-canvas-snapshot-listener!)
(when (wasm/module-ready?)

View File

@ -8,16 +8,23 @@
(:require
[app.common.types.component :as ctk]
[app.common.uuid :as uuid]
[app.main.data.event :as ev]
[app.main.data.workspace.libraries :as dwl]
[app.main.data.workspace.texts :as dwt]
[app.main.data.workspace.variants :as dwv]
[app.main.repo :as rp]
[app.main.store :as st]
[app.plugins.library :as library]
[app.plugins.register :as r]
[app.plugins.text :as text]
[app.plugins.utils :as u]
[app.util.object :as obj]
[beicon.v2.core :as rx]
[cljs.test :as t :include-macros true]
[frontend-tests.helpers.mock :as mock]))
[cuerdas.core :as str]
[frontend-tests.helpers.async :as async]
[frontend-tests.helpers.mock :as mock]
[potok.v2.core :as ptk]))
(def ^:private plugin-id "00000000-0000-0000-0000-000000000000")
@ -205,3 +212,103 @@
(t/is (= 1 (count @errors)))
(t/is (= [plugin-id :setVariantProperty "Plugin doesn't have 'library:write' permission"]
(first @errors)))))))
;; ---------------------------------------------------------------------------
;; connectLibrary
;; ---------------------------------------------------------------------------
(defn- shared-files-cmd
"Answers every RPC command asynchronously with `shared-files`, recording
the calls in `mock/rpc-calls`."
[shared-files]
(mock/stub
(fn [cmd params]
(swap! mock/rpc-calls conj {:cmd cmd :params params})
(rx/observe-on :async (rx/of shared-files)))))
(defn- settled
"Resolves with `{:resolved value}` or `{:rejected cause}` once `p` settles,
or with `{:timeout true}` when it stays pending for 2 seconds."
[p]
(js/Promise.race
#js [(.then p (fn [v] {:resolved v}) (fn [e] {:rejected e}))
(js/Promise. (fn [resolve] (js/setTimeout #(resolve {:timeout true}) 2000)))]))
(t/deftest ^:async connect-library-links-a-published-library
(let [file-id (uuid/next)
team-id (uuid/next)
library-id (uuid/next)
stream (rx/subject)
events (atom [])]
(await
(mock/with-mocks*
{st/state (atom {:current-file-id file-id :current-team-id team-id})
st/stream stream
st/emit! (mock/stub (fn [& emitted] (swap! events into emitted)))
rp/cmd! (shared-files-cmd [{:id library-id :name "Published"}])
r/check-permission (mock/stub (constantly true))}
(let [context (library/library-subcontext plugin-id)
result (settled (.connectLibrary context (str library-id)))
link? #(= ::dwl/link-file-to-library (ptk/type %))]
(await (async/wait-for #(some link? @events) "link event emitted"))
(rx/push! stream (ptk/data-event ::dwl/link-file-to-library-finished))
(let [{:keys [resolved]} (await result)
link-data (ev/-data (first (filter link? @events)))]
(t/is (= library-id (obj/get resolved "$id")))
(t/is (= file-id (:file-id link-data)))
(t/is (= library-id (:library-id link-data)))
(t/is (= [{:cmd :get-team-shared-files :params {:team-id team-id}}]
@mock/rpc-calls))))))))
(t/deftest ^:async connect-library-rejects-an-unpublished-file
(let [file-id (uuid/next)
library-id (uuid/next)
events (atom [])]
(await
(mock/with-mocks*
{st/state (atom {:current-file-id file-id :current-team-id (uuid/next)})
st/emit! (mock/stub (fn [& emitted] (swap! events into emitted)))
rp/cmd! (shared-files-cmd [{:id (uuid/next) :name "Other published"}])
r/check-permission (mock/stub (constantly true))}
(let [context (library/library-subcontext plugin-id)
{:keys [rejected]} (await (settled (.connectLibrary context (str library-id))))]
(t/is (str/includes? (str rejected) "is not a published library"))
(t/is (empty? @events)))))))
(t/deftest ^:async connect-library-rejects-the-current-file
(let [file-id (uuid/next)
events (atom [])]
(await
(mock/with-mocks*
{st/state (atom {:current-file-id file-id :current-team-id (uuid/next)})
st/emit! (mock/stub (fn [& emitted] (swap! events into emitted)))
rp/cmd! (shared-files-cmd [{:id file-id :name "Current file"}])
r/check-permission (mock/stub (constantly true))}
(let [context (library/library-subcontext plugin-id)
{:keys [rejected]} (await (settled (.connectLibrary context (str file-id))))]
(t/is (str/includes? (str rejected) "is not a published library"))
(t/is (empty? @events)))))))
(t/deftest ^:async connect-library-rejects-an-invalid-id
(await
(mock/with-mocks*
{st/state (atom {:current-file-id (uuid/next) :current-team-id (uuid/next)})
st/emit! mock/noop
rp/cmd! (shared-files-cmd [])
r/check-permission (mock/stub (constantly true))}
(let [context (library/library-subcontext plugin-id)
{:keys [rejected]} (await (settled (.connectLibrary context "not-a-uuid")))]
(t/is (str/includes? (str rejected) "not-a-uuid"))
(t/is (empty? @mock/rpc-calls))))))
(t/deftest ^:async connect-library-checks-permission
(await
(mock/with-mocks*
{st/state (atom {:current-file-id (uuid/next) :current-team-id (uuid/next)})
st/emit! mock/noop
rp/cmd! (shared-files-cmd [])
r/check-permission (mock/stub (constantly false))}
(let [context (library/library-subcontext plugin-id)
{:keys [rejected]} (await (settled (.connectLibrary context (str (uuid/next)))))]
(t/is (str/includes? (str rejected) "Plugin doesn't have 'library:write' permission"))
(t/is (empty? @mock/rpc-calls))))))