mirror of
https://github.com/penpot/penpot.git
synced 2026-11-16 23:20:37 +00:00
Merge remote-tracking branch 'origin/staging' into develop
This commit is contained in:
commit
c561a0baff
16
.github/dependabot.yml
vendored
Normal file
16
.github/dependabot.yml
vendored
Normal file
@ -0,0 +1,16 @@
|
||||
# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
|
||||
version: 2
|
||||
updates:
|
||||
# Actions are pinned to a commit SHA with a `# vX.Y.Z` comment; Dependabot
|
||||
# bumps both. All updates are grouped into a single monthly PR.
|
||||
- package-ecosystem: "github-actions"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "monthly"
|
||||
open-pull-requests-limit: 5
|
||||
commit-message:
|
||||
prefix: ":arrow_up:"
|
||||
groups:
|
||||
github-actions:
|
||||
patterns:
|
||||
- "*"
|
||||
4
.github/workflows/auto-label.yml
vendored
4
.github/workflows/auto-label.yml
vendored
@ -12,14 +12,14 @@ jobs:
|
||||
steps:
|
||||
- name: Generate GitHub App token
|
||||
id: triage-app-token
|
||||
uses: actions/create-github-app-token@v1
|
||||
uses: actions/create-github-app-token@d72941d797fd3113feb6b93fd0dec494b13a2547 # v1.12.0
|
||||
with:
|
||||
app-id: ${{ secrets.TRIAGE_APP_ID }}
|
||||
private-key: ${{ secrets.TRIAGE_APP_PRIVATE_KEY }}
|
||||
owner: penpot
|
||||
|
||||
- name: Process Issue or PR
|
||||
uses: actions/github-script@v7
|
||||
uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0
|
||||
with:
|
||||
github-token: ${{ steps.triage-app-token.outputs.token }}
|
||||
script: |
|
||||
|
||||
22
.github/workflows/build-adhoc.yml
vendored
22
.github/workflows/build-adhoc.yml
vendored
@ -23,7 +23,12 @@ on:
|
||||
jobs:
|
||||
build-bundle:
|
||||
uses: ./.github/workflows/build-bundle.yml
|
||||
secrets: inherit
|
||||
secrets:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_REGION: ${{ secrets.AWS_REGION }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }}
|
||||
S3_BUCKET: ${{ secrets.S3_BUCKET }}
|
||||
with:
|
||||
gh_ref: ${{ inputs.gh_ref }}
|
||||
force: ${{ inputs.force }}
|
||||
@ -31,7 +36,17 @@ jobs:
|
||||
build-docker:
|
||||
needs: build-bundle
|
||||
uses: ./.github/workflows/build-docker.yml
|
||||
secrets: inherit
|
||||
secrets:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_REGION: ${{ secrets.AWS_REGION }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
|
||||
DOCKER_REGISTRY: ${{ secrets.DOCKER_REGISTRY }}
|
||||
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
|
||||
MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }}
|
||||
PUB_DOCKER_PASSWORD: ${{ secrets.PUB_DOCKER_PASSWORD }}
|
||||
PUB_DOCKER_USERNAME: ${{ secrets.PUB_DOCKER_USERNAME }}
|
||||
S3_BUCKET: ${{ secrets.S3_BUCKET }}
|
||||
with:
|
||||
gh_ref: ${{ inputs.gh_ref }}
|
||||
# Pin build-docker to the commit build-bundle actually bundled.
|
||||
@ -40,7 +55,8 @@ jobs:
|
||||
|
||||
build-docker-admin-console:
|
||||
uses: ./.github/workflows/build-docker-admin-console.yml
|
||||
secrets: inherit
|
||||
secrets:
|
||||
ORG_WORKFLOW_TOKEN: ${{ secrets.ORG_WORKFLOW_TOKEN }}
|
||||
with:
|
||||
gh_ref: ${{ inputs.nitrate_ref || inputs.gh_ref }}
|
||||
force: ${{ inputs.force }}
|
||||
|
||||
17
.github/workflows/build-bundle.yml
vendored
17
.github/workflows/build-bundle.yml
vendored
@ -29,6 +29,17 @@ on:
|
||||
sha:
|
||||
description: 'Bundled commit, full 40-char SHA.'
|
||||
value: ${{ jobs.check.outputs.sha }}
|
||||
secrets:
|
||||
AWS_ACCESS_KEY_ID:
|
||||
required: false
|
||||
AWS_REGION:
|
||||
required: false
|
||||
AWS_SECRET_ACCESS_KEY:
|
||||
required: false
|
||||
MATTERMOST_WEBHOOK:
|
||||
required: false
|
||||
S3_BUCKET:
|
||||
required: false
|
||||
|
||||
# Literal group name: under `workflow_call`, `github.workflow` resolves to the
|
||||
# caller's workflow, which put this workflow and the other reusable one called
|
||||
@ -54,8 +65,9 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
fetch-depth: 0
|
||||
ref: ${{ inputs.gh_ref }}
|
||||
|
||||
@ -121,8 +133,9 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
fetch-depth: 0
|
||||
ref: ${{ inputs.gh_ref }}
|
||||
|
||||
|
||||
22
.github/workflows/build-develop.yml
vendored
22
.github/workflows/build-develop.yml
vendored
@ -21,7 +21,12 @@ concurrency:
|
||||
jobs:
|
||||
build-bundle:
|
||||
uses: ./.github/workflows/build-bundle.yml
|
||||
secrets: inherit
|
||||
secrets:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_REGION: ${{ secrets.AWS_REGION }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }}
|
||||
S3_BUCKET: ${{ secrets.S3_BUCKET }}
|
||||
with:
|
||||
gh_ref: "develop"
|
||||
force: ${{ inputs.force || false }}
|
||||
@ -29,7 +34,17 @@ jobs:
|
||||
build-docker:
|
||||
needs: build-bundle
|
||||
uses: ./.github/workflows/build-docker.yml
|
||||
secrets: inherit
|
||||
secrets:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_REGION: ${{ secrets.AWS_REGION }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
|
||||
DOCKER_REGISTRY: ${{ secrets.DOCKER_REGISTRY }}
|
||||
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
|
||||
MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }}
|
||||
PUB_DOCKER_PASSWORD: ${{ secrets.PUB_DOCKER_PASSWORD }}
|
||||
PUB_DOCKER_USERNAME: ${{ secrets.PUB_DOCKER_USERNAME }}
|
||||
S3_BUCKET: ${{ secrets.S3_BUCKET }}
|
||||
with:
|
||||
gh_ref: "develop"
|
||||
# Pin build-docker to the commit build-bundle actually bundled.
|
||||
@ -38,7 +53,8 @@ jobs:
|
||||
|
||||
build-docker-admin-console:
|
||||
uses: ./.github/workflows/build-docker-admin-console.yml
|
||||
secrets: inherit
|
||||
secrets:
|
||||
ORG_WORKFLOW_TOKEN: ${{ secrets.ORG_WORKFLOW_TOKEN }}
|
||||
with:
|
||||
gh_ref: "develop"
|
||||
force: ${{ inputs.force || false }}
|
||||
|
||||
12
.github/workflows/build-docker-devenv.yml
vendored
12
.github/workflows/build-docker-devenv.yml
vendored
@ -15,26 +15,28 @@ jobs:
|
||||
echo "DOCKER_CONFIG=${{ runner.temp }}/.docker-${{ github.run_id }}-${{ github.job }}" >> $GITHUB_ENV
|
||||
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1
|
||||
|
||||
- name: Login to Docker Registry (push destination)
|
||||
uses: docker/login-action@v4
|
||||
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
|
||||
with:
|
||||
username: ${{ secrets.PUB_DOCKER_USERNAME }}
|
||||
password: ${{ secrets.PUB_DOCKER_PASSWORD }}
|
||||
|
||||
- name: Login to Docker Hardened Images registry (base image pull)
|
||||
uses: docker/login-action@v4
|
||||
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
|
||||
with:
|
||||
registry: dhi.io
|
||||
username: ${{ secrets.PUB_DOCKER_USERNAME }}
|
||||
password: ${{ secrets.PUB_DOCKER_PASSWORD }}
|
||||
|
||||
- name: Build and push DevEnv Docker image
|
||||
uses: docker/build-push-action@v7
|
||||
uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0
|
||||
env:
|
||||
DOCKER_IMAGE: 'penpotapp/devenv'
|
||||
with:
|
||||
|
||||
49
.github/workflows/build-docker.yml
vendored
49
.github/workflows/build-docker.yml
vendored
@ -32,6 +32,27 @@ on:
|
||||
type: boolean
|
||||
required: false
|
||||
default: false
|
||||
secrets:
|
||||
AWS_ACCESS_KEY_ID:
|
||||
required: false
|
||||
AWS_REGION:
|
||||
required: false
|
||||
AWS_SECRET_ACCESS_KEY:
|
||||
required: false
|
||||
DOCKER_PASSWORD:
|
||||
required: false
|
||||
DOCKER_REGISTRY:
|
||||
required: false
|
||||
DOCKER_USERNAME:
|
||||
required: false
|
||||
MATTERMOST_WEBHOOK:
|
||||
required: false
|
||||
PUB_DOCKER_PASSWORD:
|
||||
required: false
|
||||
PUB_DOCKER_USERNAME:
|
||||
required: false
|
||||
S3_BUCKET:
|
||||
required: false
|
||||
|
||||
# Literal group name: under `workflow_call`, `github.workflow` resolves to the
|
||||
# caller's workflow, which put this workflow and the other reusable one called
|
||||
@ -65,8 +86,9 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
fetch-depth: 0
|
||||
ref: ${{ inputs.sha != '' && inputs.sha || inputs.gh_ref }}
|
||||
|
||||
@ -159,12 +181,13 @@ jobs:
|
||||
echo "DOCKER_CONFIG=${{ runner.temp }}/.docker-${{ github.run_id }}-${{ github.job }}-${{ matrix.image }}" >> $GITHUB_ENV
|
||||
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: ${{ inputs.sha != '' && inputs.sha || inputs.gh_ref }}
|
||||
|
||||
- name: Login to Docker Registry
|
||||
uses: docker/login-action@v4
|
||||
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
|
||||
with:
|
||||
registry: ${{ secrets.DOCKER_REGISTRY }}
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
@ -174,7 +197,7 @@ jobs:
|
||||
# images from DockerHub for unregistered users.
|
||||
# https://docs.docker.com/docker-hub/usage/
|
||||
- name: Login to DockerHub Registry
|
||||
uses: docker/login-action@v4
|
||||
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
|
||||
with:
|
||||
username: ${{ secrets.PUB_DOCKER_USERNAME }}
|
||||
password: ${{ secrets.PUB_DOCKER_PASSWORD }}
|
||||
@ -186,7 +209,7 @@ jobs:
|
||||
# reuses the same PUB_DOCKER_* credentials as the DockerHub
|
||||
# login above.
|
||||
- name: Login to Docker Hardened Images registry (base image pull)
|
||||
uses: docker/login-action@v4
|
||||
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
|
||||
with:
|
||||
registry: dhi.io
|
||||
username: ${{ secrets.PUB_DOCKER_USERNAME }}
|
||||
@ -218,23 +241,23 @@ jobs:
|
||||
popd
|
||||
|
||||
- name: Set up QEMU (stable)
|
||||
uses: docker/setup-qemu-action@v4
|
||||
uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4.4.0
|
||||
with:
|
||||
platforms: linux/amd64,linux/arm64
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1
|
||||
|
||||
- name: Extract metadata (tags, labels)
|
||||
id: meta
|
||||
uses: docker/metadata-action@v6
|
||||
uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0
|
||||
with:
|
||||
images: ${{ matrix.image }}
|
||||
labels: |
|
||||
bundle_version=${{ needs.prepare.outputs.bundle_version }}
|
||||
|
||||
- name: Build and push Docker image
|
||||
uses: docker/build-push-action@v7
|
||||
uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0
|
||||
with:
|
||||
context: ./docker/images/
|
||||
file: ./docker/images/Dockerfile.${{ matrix.image }}
|
||||
@ -262,10 +285,10 @@ jobs:
|
||||
echo "DOCKER_CONFIG=${{ runner.temp }}/.docker-${{ github.run_id }}-${{ github.job }}" >> $GITHUB_ENV
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1
|
||||
|
||||
- name: Login to Docker Registry
|
||||
uses: docker/login-action@v4
|
||||
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
|
||||
with:
|
||||
registry: ${{ secrets.DOCKER_REGISTRY }}
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
@ -331,10 +354,10 @@ jobs:
|
||||
echo "DOCKER_CONFIG=${{ runner.temp }}/.docker-${{ github.run_id }}-${{ github.job }}" >> $GITHUB_ENV
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1
|
||||
|
||||
- name: Login to Docker Registry
|
||||
uses: docker/login-action@v4
|
||||
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
|
||||
with:
|
||||
registry: ${{ secrets.DOCKER_REGISTRY }}
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
|
||||
22
.github/workflows/build-staging.yml
vendored
22
.github/workflows/build-staging.yml
vendored
@ -21,7 +21,12 @@ concurrency:
|
||||
jobs:
|
||||
build-bundle:
|
||||
uses: ./.github/workflows/build-bundle.yml
|
||||
secrets: inherit
|
||||
secrets:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_REGION: ${{ secrets.AWS_REGION }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }}
|
||||
S3_BUCKET: ${{ secrets.S3_BUCKET }}
|
||||
with:
|
||||
gh_ref: "staging"
|
||||
force: ${{ inputs.force || false }}
|
||||
@ -29,7 +34,17 @@ jobs:
|
||||
build-docker:
|
||||
needs: build-bundle
|
||||
uses: ./.github/workflows/build-docker.yml
|
||||
secrets: inherit
|
||||
secrets:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_REGION: ${{ secrets.AWS_REGION }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
|
||||
DOCKER_REGISTRY: ${{ secrets.DOCKER_REGISTRY }}
|
||||
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
|
||||
MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }}
|
||||
PUB_DOCKER_PASSWORD: ${{ secrets.PUB_DOCKER_PASSWORD }}
|
||||
PUB_DOCKER_USERNAME: ${{ secrets.PUB_DOCKER_USERNAME }}
|
||||
S3_BUCKET: ${{ secrets.S3_BUCKET }}
|
||||
with:
|
||||
gh_ref: "staging"
|
||||
# Pin build-docker to the commit build-bundle actually bundled.
|
||||
@ -38,7 +53,8 @@ jobs:
|
||||
|
||||
build-docker-admin-console:
|
||||
uses: ./.github/workflows/build-docker-admin-console.yml
|
||||
secrets: inherit
|
||||
secrets:
|
||||
ORG_WORKFLOW_TOKEN: ${{ secrets.ORG_WORKFLOW_TOKEN }}
|
||||
with:
|
||||
gh_ref: "staging"
|
||||
force: ${{ inputs.force || false }}
|
||||
|
||||
31
.github/workflows/build-tag.yml
vendored
31
.github/workflows/build-tag.yml
vendored
@ -24,7 +24,12 @@ concurrency:
|
||||
jobs:
|
||||
build-bundle:
|
||||
uses: ./.github/workflows/build-bundle.yml
|
||||
secrets: inherit
|
||||
secrets:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_REGION: ${{ secrets.AWS_REGION }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }}
|
||||
S3_BUCKET: ${{ secrets.S3_BUCKET }}
|
||||
with:
|
||||
gh_ref: ${{ github.ref_name }}
|
||||
force: ${{ inputs.force || false }}
|
||||
@ -32,14 +37,25 @@ jobs:
|
||||
build-docker:
|
||||
needs: build-bundle
|
||||
uses: ./.github/workflows/build-docker.yml
|
||||
secrets: inherit
|
||||
secrets:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_REGION: ${{ secrets.AWS_REGION }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
|
||||
DOCKER_REGISTRY: ${{ secrets.DOCKER_REGISTRY }}
|
||||
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
|
||||
MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }}
|
||||
PUB_DOCKER_PASSWORD: ${{ secrets.PUB_DOCKER_PASSWORD }}
|
||||
PUB_DOCKER_USERNAME: ${{ secrets.PUB_DOCKER_USERNAME }}
|
||||
S3_BUCKET: ${{ secrets.S3_BUCKET }}
|
||||
with:
|
||||
gh_ref: ${{ github.ref_name }}
|
||||
force: ${{ inputs.force || false }}
|
||||
|
||||
build-docker-admin-console:
|
||||
uses: ./.github/workflows/build-docker-admin-console.yml
|
||||
secrets: inherit
|
||||
secrets:
|
||||
ORG_WORKFLOW_TOKEN: ${{ secrets.ORG_WORKFLOW_TOKEN }}
|
||||
with:
|
||||
gh_ref: ${{ github.ref_name }}
|
||||
force: ${{ inputs.force || false }}
|
||||
@ -67,6 +83,13 @@ jobs:
|
||||
- build-docker
|
||||
- build-docker-admin-console
|
||||
uses: ./.github/workflows/release.yml
|
||||
secrets: inherit
|
||||
secrets:
|
||||
AWS_REGION: ${{ secrets.AWS_REGION }}
|
||||
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
|
||||
DOCKER_REGISTRY: ${{ secrets.DOCKER_REGISTRY }}
|
||||
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
|
||||
MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }}
|
||||
PUB_DOCKER_PASSWORD: ${{ secrets.PUB_DOCKER_PASSWORD }}
|
||||
PUB_DOCKER_USERNAME: ${{ secrets.PUB_DOCKER_USERNAME }}
|
||||
with:
|
||||
gh_ref: ${{ github.ref_name }}
|
||||
|
||||
19
.github/workflows/build-tmp-tokens.yml
vendored
19
.github/workflows/build-tmp-tokens.yml
vendored
@ -12,13 +12,28 @@ concurrency:
|
||||
jobs:
|
||||
build-bundle:
|
||||
uses: ./.github/workflows/build-bundle.yml
|
||||
secrets: inherit
|
||||
secrets:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_REGION: ${{ secrets.AWS_REGION }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }}
|
||||
S3_BUCKET: ${{ secrets.S3_BUCKET }}
|
||||
with:
|
||||
gh_ref: "hiru-tokens-in-libs"
|
||||
|
||||
build-docker:
|
||||
needs: build-bundle
|
||||
uses: ./.github/workflows/build-docker.yml
|
||||
secrets: inherit
|
||||
secrets:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_REGION: ${{ secrets.AWS_REGION }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
|
||||
DOCKER_REGISTRY: ${{ secrets.DOCKER_REGISTRY }}
|
||||
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
|
||||
MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }}
|
||||
PUB_DOCKER_PASSWORD: ${{ secrets.PUB_DOCKER_PASSWORD }}
|
||||
PUB_DOCKER_USERNAME: ${{ secrets.PUB_DOCKER_USERNAME }}
|
||||
S3_BUCKET: ${{ secrets.S3_BUCKET }}
|
||||
with:
|
||||
gh_ref: "hiru-tokens-in-libs"
|
||||
|
||||
2
.github/workflows/commit-checker.yml
vendored
2
.github/workflows/commit-checker.yml
vendored
@ -27,7 +27,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Check Commit Type
|
||||
uses: gsactions/commit-message-checker@v2
|
||||
uses: gsactions/commit-message-checker@16fa2d5de096ae0d35626443bcd24f1e756cafee # v2.0.0
|
||||
with:
|
||||
pattern: '^(((:(lipstick|globe_with_meridians|wrench|books|arrow_up|arrow_down|zap|ambulance|construction|boom|fire|whale|bug|sparkles|paperclip|tada|recycle|rewind|construction_worker):)\s[A-Z].*[^.])|(Merge|Revert|Reapply).+[^.])$'
|
||||
flags: 'gm'
|
||||
|
||||
7
.github/workflows/plugins-deploy-api-doc.yml
vendored
7
.github/workflows/plugins-deploy-api-doc.yml
vendored
@ -39,14 +39,15 @@ jobs:
|
||||
echo "gh_ref=$GH_REF" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
fetch-depth: 0
|
||||
ref: ${{ steps.vars.outputs.gh_ref }}
|
||||
|
||||
# START: Setup Node and PNPM enabling cache
|
||||
- name: Setup pnpm + Node.js
|
||||
uses: pnpm/setup@v2
|
||||
uses: pnpm/setup@84cb39b217b10273981911c288cd62326dc7c6d2 # v2.0.2
|
||||
with:
|
||||
working-directory: plugins
|
||||
runtime: node@24.21.0
|
||||
@ -59,7 +60,7 @@ jobs:
|
||||
run: echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Cache pnpm store
|
||||
uses: actions/cache@v5
|
||||
uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0
|
||||
with:
|
||||
path: ${{ steps.pnpm-store.outputs.STORE_PATH }}
|
||||
key: ${{ runner.os }}-pnpm-${{ hashFiles('plugins/pnpm-lock.yaml') }}
|
||||
|
||||
10
.github/workflows/plugins-deploy-package.yml
vendored
10
.github/workflows/plugins-deploy-package.yml
vendored
@ -27,6 +27,13 @@ on:
|
||||
description: 'Publig name (from plugins/apps/<plugin_name>-plugin)'
|
||||
type: string
|
||||
required: true
|
||||
secrets:
|
||||
CLOUDFLARE_ACCOUNT_ID:
|
||||
required: false
|
||||
CLOUDFLARE_API_TOKEN:
|
||||
required: false
|
||||
MATTERMOST_WEBHOOK:
|
||||
required: false
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@ -58,8 +65,9 @@ jobs:
|
||||
fi
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
fetch-depth: 0
|
||||
ref: ${{ inputs.gh_ref }}
|
||||
|
||||
|
||||
41
.github/workflows/plugins-deploy-packages.yml
vendored
41
.github/workflows/plugins-deploy-packages.yml
vendored
@ -36,9 +36,11 @@ jobs:
|
||||
# [For new plugins]
|
||||
# Add more outputs here
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
- id: filter
|
||||
uses: dorny/paths-filter@v4
|
||||
uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4.0.3
|
||||
with:
|
||||
filters: |
|
||||
colors_to_tokens:
|
||||
@ -72,7 +74,10 @@ jobs:
|
||||
needs: detect-changes
|
||||
if: github.event_name == 'workflow_dispatch' || needs.detect-changes.outputs.colors_to_tokens == 'true'
|
||||
uses: ./.github/workflows/plugins-deploy-package.yml
|
||||
secrets: inherit
|
||||
secrets:
|
||||
CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
|
||||
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
|
||||
MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }}
|
||||
with:
|
||||
gh_ref: "${{ inputs.gh_ref || github.ref_name }}"
|
||||
plugin_name: colors-to-tokens
|
||||
@ -81,7 +86,10 @@ jobs:
|
||||
needs: detect-changes
|
||||
if: github.event_name == 'workflow_dispatch' || needs.detect-changes.outputs.contrast == 'true'
|
||||
uses: ./.github/workflows/plugins-deploy-package.yml
|
||||
secrets: inherit
|
||||
secrets:
|
||||
CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
|
||||
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
|
||||
MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }}
|
||||
with:
|
||||
gh_ref: "${{ inputs.gh_ref || github.ref_name }}"
|
||||
plugin_name: contrast
|
||||
@ -90,7 +98,10 @@ jobs:
|
||||
needs: detect-changes
|
||||
if: github.event_name == 'workflow_dispatch' || needs.detect-changes.outputs.create_palette == 'true'
|
||||
uses: ./.github/workflows/plugins-deploy-package.yml
|
||||
secrets: inherit
|
||||
secrets:
|
||||
CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
|
||||
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
|
||||
MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }}
|
||||
with:
|
||||
gh_ref: "${{ inputs.gh_ref || github.ref_name }}"
|
||||
plugin_name: create-palette
|
||||
@ -99,7 +110,10 @@ jobs:
|
||||
needs: detect-changes
|
||||
if: github.event_name == 'workflow_dispatch' || needs.detect-changes.outputs.icons == 'true'
|
||||
uses: ./.github/workflows/plugins-deploy-package.yml
|
||||
secrets: inherit
|
||||
secrets:
|
||||
CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
|
||||
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
|
||||
MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }}
|
||||
with:
|
||||
gh_ref: "${{ inputs.gh_ref || github.ref_name }}"
|
||||
plugin_name: icons
|
||||
@ -108,7 +122,10 @@ jobs:
|
||||
needs: detect-changes
|
||||
if: github.event_name == 'workflow_dispatch' || needs.detect-changes.outputs.lorem_ipsum == 'true'
|
||||
uses: ./.github/workflows/plugins-deploy-package.yml
|
||||
secrets: inherit
|
||||
secrets:
|
||||
CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
|
||||
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
|
||||
MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }}
|
||||
with:
|
||||
gh_ref: "${{ inputs.gh_ref || github.ref_name }}"
|
||||
plugin_name: lorem-ipsum
|
||||
@ -117,7 +134,10 @@ jobs:
|
||||
needs: detect-changes
|
||||
if: github.event_name == 'workflow_dispatch' || needs.detect-changes.outputs.rename_layers == 'true'
|
||||
uses: ./.github/workflows/plugins-deploy-package.yml
|
||||
secrets: inherit
|
||||
secrets:
|
||||
CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
|
||||
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
|
||||
MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }}
|
||||
with:
|
||||
gh_ref: "${{ inputs.gh_ref || github.ref_name }}"
|
||||
plugin_name: rename-layers
|
||||
@ -126,7 +146,10 @@ jobs:
|
||||
needs: detect-changes
|
||||
if: github.event_name == 'workflow_dispatch' || needs.detect-changes.outputs.table == 'true'
|
||||
uses: ./.github/workflows/plugins-deploy-package.yml
|
||||
secrets: inherit
|
||||
secrets:
|
||||
CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
|
||||
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
|
||||
MATTERMOST_WEBHOOK: ${{ secrets.MATTERMOST_WEBHOOK }}
|
||||
with:
|
||||
gh_ref: "${{ inputs.gh_ref || github.ref_name }}"
|
||||
plugin_name: table
|
||||
|
||||
@ -37,14 +37,15 @@ jobs:
|
||||
echo "gh_ref=$GH_REF" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
fetch-depth: 0
|
||||
ref: ${{ steps.vars.outputs.gh_ref }}
|
||||
|
||||
# START: Setup Node and PNPM enabling cache
|
||||
- name: Setup pnpm + Node.js
|
||||
uses: pnpm/setup@v2
|
||||
uses: pnpm/setup@84cb39b217b10273981911c288cd62326dc7c6d2 # v2.0.2
|
||||
with:
|
||||
working-directory: plugins
|
||||
runtime: node@24.21.0
|
||||
@ -57,7 +58,7 @@ jobs:
|
||||
run: echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Cache pnpm store
|
||||
uses: actions/cache@v5
|
||||
uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0
|
||||
with:
|
||||
path: ${{ steps.pnpm-store.outputs.STORE_PATH }}
|
||||
key: ${{ runner.os }}-pnpm-${{ hashFiles('plugins/pnpm-lock.yaml') }}
|
||||
|
||||
37
.github/workflows/release.yml
vendored
37
.github/workflows/release.yml
vendored
@ -13,6 +13,21 @@ on:
|
||||
description: 'Tag to release'
|
||||
type: string
|
||||
required: true
|
||||
secrets:
|
||||
AWS_REGION:
|
||||
required: false
|
||||
DOCKER_PASSWORD:
|
||||
required: false
|
||||
DOCKER_REGISTRY:
|
||||
required: false
|
||||
DOCKER_USERNAME:
|
||||
required: false
|
||||
MATTERMOST_WEBHOOK:
|
||||
required: false
|
||||
PUB_DOCKER_PASSWORD:
|
||||
required: false
|
||||
PUB_DOCKER_USERNAME:
|
||||
required: false
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
@ -32,14 +47,15 @@ jobs:
|
||||
echo "gh_ref=$GH_REF" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
fetch-depth: 0
|
||||
ref: ${{ steps.vars.outputs.gh_ref }}
|
||||
|
||||
# --- Publicly release the docker images ---
|
||||
- name: Configure ECR credentials
|
||||
uses: aws-actions/configure-aws-credentials@v4
|
||||
uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1
|
||||
with:
|
||||
aws-access-key-id: ${{ secrets.DOCKER_USERNAME }}
|
||||
aws-secret-access-key: ${{ secrets.DOCKER_PASSWORD }}
|
||||
@ -94,14 +110,19 @@ jobs:
|
||||
echo "EOF" >> $GITHUB_OUTPUT
|
||||
|
||||
# --- Create GitHub release ---
|
||||
# Updates the release if it already exists, so re-running is safe.
|
||||
- name: Create GitHub release
|
||||
uses: softprops/action-gh-release@v2
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
with:
|
||||
tag_name: ${{ steps.vars.outputs.gh_ref }}
|
||||
name: ${{ steps.vars.outputs.gh_ref }}
|
||||
body: ${{ steps.extract_release_notes.outputs.release_notes }}
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
GH_REPO: ${{ github.repository }}
|
||||
TAG: ${{ steps.vars.outputs.gh_ref }}
|
||||
NOTES: ${{ steps.extract_release_notes.outputs.release_notes }}
|
||||
run: |
|
||||
if gh release view "$TAG" > /dev/null 2>&1; then
|
||||
gh release edit "$TAG" --title "$TAG" --notes "$NOTES"
|
||||
else
|
||||
gh release create "$TAG" --verify-tag --title "$TAG" --notes "$NOTES"
|
||||
fi
|
||||
|
||||
- name: Notify Mattermost
|
||||
if: failure()
|
||||
|
||||
4
.github/workflows/tests-backend.yml
vendored
4
.github/workflows/tests-backend.yml
vendored
@ -60,7 +60,9 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Fmt
|
||||
working-directory: ./backend
|
||||
|
||||
4
.github/workflows/tests-common.yml
vendored
4
.github/workflows/tests-common.yml
vendored
@ -39,7 +39,9 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Lint
|
||||
working-directory: ./common
|
||||
|
||||
37
.github/workflows/tests-e2e.yml
vendored
37
.github/workflows/tests-e2e.yml
vendored
@ -107,8 +107,9 @@ jobs:
|
||||
# An empty `ref` makes checkout fall back to its default (the PR merge
|
||||
# ref on pull_request, the pushed ref on push).
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: ${{ inputs.gh_ref }}
|
||||
|
||||
# The cache key must come from the SHA actually checked out: on a manual
|
||||
@ -120,7 +121,7 @@ jobs:
|
||||
|
||||
- name: Restore Bundle Cache
|
||||
id: restore
|
||||
uses: actions/cache/restore@v5
|
||||
uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0
|
||||
with:
|
||||
key: ${{ steps.vars.outputs.bundle_key }}
|
||||
path: frontend/resources/public
|
||||
@ -133,7 +134,7 @@ jobs:
|
||||
|
||||
- name: Store Bundle Cache
|
||||
if: steps.restore.outputs.cache-hit != 'true'
|
||||
uses: actions/cache/save@v5
|
||||
uses: actions/cache/save@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0
|
||||
with:
|
||||
key: ${{ steps.vars.outputs.bundle_key }}
|
||||
path: frontend/resources/public
|
||||
@ -163,12 +164,13 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout Repository
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: ${{ inputs.gh_ref }}
|
||||
|
||||
- name: Restore Cache
|
||||
uses: actions/cache/restore@v5
|
||||
uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0
|
||||
with:
|
||||
key: ${{ needs.build-bundle.outputs.bundle_key }}
|
||||
path: frontend/resources/public
|
||||
@ -201,7 +203,7 @@ jobs:
|
||||
--reporter=blob
|
||||
|
||||
- name: Upload blob report
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
if: always()
|
||||
with:
|
||||
name: integration-blob-report-${{ matrix.shard }}
|
||||
@ -210,7 +212,7 @@ jobs:
|
||||
retention-days: 3
|
||||
|
||||
- name: Upload test result
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
if: always()
|
||||
with:
|
||||
name: integration-tests-result-${{ matrix.shard }}
|
||||
@ -234,11 +236,13 @@ jobs:
|
||||
- /var/cache/github-runner/gitlib:/root/.gitlibs
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
# The driver serves the prebuilt bundle from frontend/resources/public.
|
||||
- name: Restore Cache
|
||||
uses: actions/cache/restore@v5
|
||||
uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0
|
||||
with:
|
||||
key: ${{ needs.build-bundle.outputs.bundle_key }}
|
||||
path: frontend/resources/public
|
||||
@ -272,11 +276,13 @@ jobs:
|
||||
- /var/cache/github-runner/gitlib:/root/.gitlibs
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
# Mocked mode serves the prebuilt bundle from frontend/resources/public.
|
||||
- name: Restore Cache
|
||||
uses: actions/cache/restore@v5
|
||||
uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0
|
||||
with:
|
||||
key: ${{ needs.build-bundle.outputs.bundle_key }}
|
||||
path: frontend/resources/public
|
||||
@ -318,8 +324,9 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout Repository
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: ${{ inputs.gh_ref }}
|
||||
|
||||
- name: Install deps
|
||||
@ -328,7 +335,7 @@ jobs:
|
||||
pnpm install --frozen-lockfile;
|
||||
|
||||
- name: Download blob reports
|
||||
uses: actions/download-artifact@v7
|
||||
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
|
||||
with:
|
||||
path: frontend/all-blob-reports
|
||||
pattern: integration-blob-report-*
|
||||
@ -357,7 +364,7 @@ jobs:
|
||||
>> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
- name: Upload JSON report
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
if: always()
|
||||
with:
|
||||
name: integration-json-report
|
||||
@ -367,7 +374,7 @@ jobs:
|
||||
retention-days: 7
|
||||
|
||||
- name: Upload HTML report
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
if: always()
|
||||
with:
|
||||
name: integration-html-report
|
||||
|
||||
4
.github/workflows/tests-exporter.yml
vendored
4
.github/workflows/tests-exporter.yml
vendored
@ -43,7 +43,9 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Lint
|
||||
working-directory: ./exporter
|
||||
|
||||
4
.github/workflows/tests-frontend.yml
vendored
4
.github/workflows/tests-frontend.yml
vendored
@ -43,7 +43,9 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Lint
|
||||
working-directory: ./frontend
|
||||
|
||||
4
.github/workflows/tests-library.yml
vendored
4
.github/workflows/tests-library.yml
vendored
@ -41,7 +41,9 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Lint
|
||||
working-directory: ./library
|
||||
|
||||
4
.github/workflows/tests-mcp.yml
vendored
4
.github/workflows/tests-mcp.yml
vendored
@ -33,7 +33,9 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup
|
||||
working-directory: ./mcp
|
||||
|
||||
4
.github/workflows/tests-plugins.yml
vendored
4
.github/workflows/tests-plugins.yml
vendored
@ -38,7 +38,9 @@ jobs:
|
||||
- /var/cache/github-runner/gitlib:/root/.gitlibs
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Install deps
|
||||
working-directory: ./plugins
|
||||
|
||||
4
.github/workflows/tests-wasm.yml
vendored
4
.github/workflows/tests-wasm.yml
vendored
@ -39,7 +39,9 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Format
|
||||
working-directory: ./render-wasm
|
||||
|
||||
55
.github/zizmor.yml
vendored
55
.github/zizmor.yml
vendored
@ -12,26 +12,6 @@ rules:
|
||||
- plugins-deploy-api-doc.yml
|
||||
- plugins-deploy-package.yml
|
||||
- plugins-deploy-styles-doc.yml
|
||||
# Use `persist-credentials: false` in actions/checkout.
|
||||
artipacked:
|
||||
ignore:
|
||||
- build-bundle.yml
|
||||
- build-docker-devenv.yml
|
||||
- build-docker.yml
|
||||
- plugins-deploy-api-doc.yml
|
||||
- plugins-deploy-package.yml
|
||||
- plugins-deploy-packages.yml
|
||||
- plugins-deploy-styles-doc.yml
|
||||
- release.yml
|
||||
- tests-backend.yml
|
||||
- tests-common.yml
|
||||
- tests-e2e.yml
|
||||
- tests-exporter.yml
|
||||
- tests-frontend.yml
|
||||
- tests-library.yml
|
||||
- tests-mcp.yml
|
||||
- tests-plugins.yml
|
||||
- tests-wasm.yml
|
||||
# Avoid restoring caches in release/publish workflows.
|
||||
cache-poisoning:
|
||||
ignore:
|
||||
@ -70,22 +50,9 @@ rules:
|
||||
github-app:
|
||||
ignore:
|
||||
- auto-label.yml
|
||||
# Pass only the secrets each reusable workflow needs.
|
||||
secrets-inherit:
|
||||
ignore:
|
||||
- build-adhoc.yml
|
||||
- build-develop.yml
|
||||
- build-staging.yml
|
||||
- build-tag.yml
|
||||
- build-tmp-tokens.yml
|
||||
- plugins-deploy-packages.yml
|
||||
# Style nudge towards the `$/...` syntax; not worth enforcing.
|
||||
self-repository:
|
||||
disable: true
|
||||
# Replace actions that duplicate built-in runner tools.
|
||||
superfluous-actions:
|
||||
ignore:
|
||||
- release.yml
|
||||
# Pin container images to a digest.
|
||||
unpinned-images:
|
||||
ignore:
|
||||
@ -99,25 +66,3 @@ rules:
|
||||
- tests-mcp.yml
|
||||
- tests-plugins.yml
|
||||
- tests-wasm.yml
|
||||
# Pin actions to a full commit SHA.
|
||||
unpinned-uses:
|
||||
ignore:
|
||||
- auto-label.yml
|
||||
- build-bundle.yml
|
||||
- build-docker-devenv.yml
|
||||
- build-docker.yml
|
||||
- commit-checker.yml
|
||||
- plugins-deploy-api-doc.yml
|
||||
- plugins-deploy-package.yml
|
||||
- plugins-deploy-packages.yml
|
||||
- plugins-deploy-styles-doc.yml
|
||||
- release.yml
|
||||
- tests-backend.yml
|
||||
- tests-common.yml
|
||||
- tests-e2e.yml
|
||||
- tests-exporter.yml
|
||||
- tests-frontend.yml
|
||||
- tests-library.yml
|
||||
- tests-mcp.yml
|
||||
- tests-plugins.yml
|
||||
- tests-wasm.yml
|
||||
|
||||
@ -13,7 +13,7 @@
|
||||
- Logout may return an OIDC provider redirect URI when the session claims include provider/session data and the provider has a logout URI.
|
||||
- Invitation tokens are verified through token issuers and only accepted when the token member id/email matches the authenticated profile; otherwise login proceeds without consuming the invitation.
|
||||
- When public registration is disabled, invitation-based password registration also requires a matching, non-expired `team_invitation` row; registration revalidates it under a row lock before creating the profile.
|
||||
- HTTP/session parsing details such as cookie/header precedence, JWT session token versions, and SameSite behavior are in `mem:backend/subtleties`.
|
||||
- HTTP/session parsing details such as cookie/header precedence and SameSite behavior are in `mem:backend/subtleties`.
|
||||
|
||||
## Permission model
|
||||
|
||||
|
||||
@ -7,6 +7,7 @@ Backend: JVM Clojure; Integrant; PostgreSQL; Redis/Valkey; RPC; HTTP; storage; m
|
||||
- Cross-cutting backend subtleties (RPC, DB, workers, cron, HTTP/sessions, storage, file data): `mem:backend/subtleties`
|
||||
- Storage abstraction, logical buckets, object lifecycle, deduplication, access, and garbage collection: `mem:backend/storage`.
|
||||
- Embedded Ladybug graph experiment, projection, incremental sync, console, and risks: `mem:backend/graph-experiment`
|
||||
- Session lifetime config, token `:exp`, and idle/absolute GC: `mem:backend/session-expiration`.
|
||||
- Auth flows, permission model, teams, projects, invitations, comments, webhooks, audit: `mem:backend/auth-permissions-product-domains`
|
||||
- Audit-log event collection (RPC wrapper, frontend ingestion), telemetry duality, webhook fan-out, error reporters, Nexus archival and retention: `mem:backend/audit-log`
|
||||
- Services, task-queue/Pub-Sub topology constraints -> `mem:prod-infra/core`.
|
||||
|
||||
35
.serena/memories/backend/session-expiration.md
Normal file
35
.serena/memories/backend/session-expiration.md
Normal file
@ -0,0 +1,35 @@
|
||||
# Backend Session Expiration
|
||||
|
||||
## Config
|
||||
|
||||
- `:auth-token-cookie-name` / `PENPOT_AUTH_TOKEN_COOKIE_NAME` (default `auth-token`): cookie name.
|
||||
- `:auth-token-cookie-max-age` / `PENPOT_AUTH_TOKEN_COOKIE_MAX_AGE` (default 7d): idle window; drives the sliding cookie `Expires` and the GC idle threshold.
|
||||
- `:auth-token-cookie-max-age-absolute` / `PENPOT_AUTH_TOKEN_COOKIE_MAX_AGE_ABSOLUTE` (default 30d): hard cap from `created-at`; drives the token `:exp` and the GC absolute threshold.
|
||||
- Durations decode as `<n><unit>` with hour/minute/second units (`168h`, `30m`); day units like `7d` are rejected by the `Duration/parse` string path, use hours (`168h`, `720h`).
|
||||
- Session lifetime defaults live only as code constants in `session.clj` (`default-cookie-max-age`, `default-cookie-max-age-absolute`); do not duplicate them in the `config.clj` default map. Call sites always pass the constant as the `cf/get` fallback.
|
||||
|
||||
## Token and session model
|
||||
|
||||
- Sessions live only in `http_session_v2`. Legacy v1 support (`http_session`, string ids, `:ver 0` tokens) was removed and the table dropped (`0155-drop-http-session-table`).
|
||||
- `assign-token` emits header `{:kid 1 :ver 1}` with claims `:sid`, `:iat` (= `modified-at`) and `:exp` = `created-at + absolute-max-age` (omitted only when `created-at` is nil, which neither manager produces today; the branch is defensive).
|
||||
- `:exp` is anchored to `created-at`, never `modified-at`, so renewal cannot extend the absolute maximum.
|
||||
- Tokens issued before `:exp` existed carry no `:exp`; they are still bounded by the GC's `created_at` condition and acquire `:exp` on their next renewal (self-healing, no operator action).
|
||||
- `wrap-authz` resolves the session by `(:sid claims)` only; there is no fallback to reading a session by the raw token string.
|
||||
- `middleware/wrap-auth` attaches `::http/auth-data` only for `kid=1`/`ver=1` tokens with a configured decoder; anything else stays unauthenticated.
|
||||
- Renewal fires when `modified-at` is older than 6h (`default-renewal-max-age`, not configurable). It `UPDATE`s the same row (`modified-at` only) and issues a new token that keeps the original `:exp`.
|
||||
- `read-session` does not check age. Idle expiration is enforced on every request by `wrap-authz` (sessions past the idle window authenticate like missing ones and are never renewed); the GC only deletes the rows afterwards.
|
||||
|
||||
## GC (`::tasks/gc`, cron `session-gc`, daily)
|
||||
|
||||
- Deletes from `http_session_v2` where `modified_at < now - :auth-token-cookie-max-age` **or** `created_at < now - :auth-token-cookie-max-age-absolute`.
|
||||
- The two thresholds are separate task params (`::tasks/max-age`, `::tasks/max-age-absolute`) built in `ig/expand-key`; both must be durations.
|
||||
- Do not collapse the two conditions: `modified_at` alone never collects active sessions; `created_at` alone logs out active users at the idle window.
|
||||
|
||||
## Related lifetimes
|
||||
|
||||
- Organization SSO entries in session `:props` (`:sso {org-id exp}`) last 4h, hardcoded in `app.auth.oidc`.
|
||||
- Access tokens (`app.rpc.commands.access-token`) have a user-chosen `:expires-at` (Never/30/60/90/180d), independent from HTTP sessions.
|
||||
- Cookie `SameSite`/`secure` depend on `:cors`, `:strict-session-cookies`, `:secure-session-cookies`; `parse-flags` auto-adds `:disable-secure-session-cookies` for non-localhost HTTP `public-uri`.
|
||||
- Without `PENPOT_SECRET_KEY`, derived subsystem keys change on restart and all sessions/invitations are invalidated.
|
||||
- Read-only DB pool: sessions fall back to `inmemory-manager`, lost on restart, no GC.
|
||||
- HTTP middleware and cookie/header precedence details: `mem:backend/subtleties`.
|
||||
@ -54,8 +54,7 @@
|
||||
|
||||
### Key Warning (from function notes):
|
||||
|
||||
The improved note in `import-storage-objects` and `handle-persistence` warns:
|
||||
**Do not reuse the main database connection for storage operations within a transaction.** The storage upload process can fail mid-operation, leaving orphaned objects on the backend. If the outer transaction aborts, pending storage objects become unreconciliable because the storage subsystem registers its pending state in separate transactions.
|
||||
The improved note in `import-storage-objects` and `handle-persistence` warns: **Do not reuse the main database connection for storage operations within a transaction.** The storage upload process can fail mid-operation, leaving orphaned objects on the backend. If the outer transaction aborts, pending storage objects become unreconciliable because the storage subsystem registers its pending state in separate transactions.
|
||||
|
||||
### Rule of Thumb for `sto/put-object!`:
|
||||
|
||||
@ -66,10 +65,7 @@ Since `put-object!` uses backend-specific operations (`impl/resolve-backend` + `
|
||||
- Deduplication requires `::sto/deduplicate?`, a content hash, and bucket metadata.
|
||||
- The lookup matches hash, bucket, backend, and `deleted_at IS NULL`.
|
||||
- The lookup only considers rows with `status='valid'`; pending rows are invisible.
|
||||
- A hit whose blob is missing is repaired in place: the same row/id is kept,
|
||||
and `put-object!` rewrites the blob under that id. This heals all existing
|
||||
references to the object. If the rewrite fails, the row is left live and
|
||||
valid for a later retry.
|
||||
- A hit whose blob is missing is repaired in place: the same row/id is kept, and `put-object!` rewrites the blob under that id. This heals all existing references to the object. If the rewrite fails, the row is left live and valid for a later retry.
|
||||
- The lookup does not include file ID, profile ID, team ID, or organization ID.
|
||||
- Objects can therefore share content across users and files within one bucket.
|
||||
- Deleted objects are not reused.
|
||||
@ -130,6 +126,8 @@ Since `put-object!` uses backend-specific operations (`impl/resolve-backend` + `
|
||||
- Logical storage operations (`app.storage`, `::mtx/metrics` required by the schema):
|
||||
- `penpot_storage_operations_total{op,bucket,backend}` — `put`, `repair`, `get-data`, `get-bytes`, `del`, `touch`, `exists`. All ops are success-only: `put`/`repair` emit after the backend write, `get-*` after the backend fetch opens, `touch`/`del` only when a row actually changed. `touch-object!`/`del-object!` take the object id (UUID) only — no object overload. Labels come from the updated row itself via `UPDATE ... RETURNING id, backend, metadata` (no extra `SELECT`); with no row matched they emit nothing. `del-object!` only matches live rows (`deleted_at IS NULL`): a repeated del returns `false` and emits nothing. Post-open stream read errors stay counted as attempts. `del` only marks `deleted_at`; physical deletion is a GC concern. `exists` is emitted per deduplication-hit probe, always paired with a `hit`/`repair` outcome (never on probe failure), not per user-facing existence check.
|
||||
- `penpot_storage_dedup_total{result,bucket}` — `hit`, `miss`, `repair`, `skip`.
|
||||
- Storage GC poison (`app.storage.gc-touched`, `::mtx/metrics` required in the handler cfg):
|
||||
- `penpot_storage_gc_poison_total` — touched rows whose metadata fails to decode. Each one is logged at error level and deferred exactly one day instead of aborting the chunk. No labels; incremented once per row per scan.
|
||||
- Asset serving (`app.http.assets`, `::mtx/metrics` required in the handler cfg):
|
||||
- `penpot_storage_asset_requests_total{route,backend,bucket,result}` — `route` is `by-id`, `by-file-media-id`, or `thumbnail`; `result` is `served` (<400), `not-found` (404), `unauthorized` (401/403), or `error` (everything else, including a nil/non-number status: every serve path must set `::yres/status`). Serve-path exceptions are counted by `serve-object-measured` and then rethrown. Permission-denied file-media requests and tempfile ownership mismatches both answer HTTP 404 (to avoid leaking existence) but are counted as `unauthorized`. Malformed UUIDs raise before any emission point and are never counted. Counts backend requests that trigger a browser GET to the object store (one per cache miss), so it is a proxy for object GETs, not an exact count.
|
||||
- The physical and logical counters intentionally overlap in coverage but differ in meaning; do not sum them.
|
||||
|
||||
@ -33,10 +33,10 @@
|
||||
- The backend sets Clojure `*assert*` globally from the `:backend-asserts` feature flag. Assertion-dependent checks can therefore differ by runtime flags.
|
||||
- Request body parsing is mostly POST-oriented and supports Transit JSON plus plain JSON. Plain JSON request keys are kebab-decoded before being merged into `:params`.
|
||||
- Response formatting negotiates with `Accept` or `_fmt=json`. Transit is the default for collection/boolean bodies; JSON encoding has special pointer-map handling.
|
||||
- Auth prefers the session cookie token before the `Authorization` header. Headers may be `Token` or `Bearer`; JWTs with `kid=1` and `ver=1` are decoded as v1 session tokens, otherwise they are treated as legacy tokens.
|
||||
- Auth prefers the session cookie token before the `Authorization` header. Headers may be `Token` or `Bearer`. Only `kid=1`/`ver=1` tokens are decoded as session tokens; anything else is left unauthenticated (legacy v1 tokens were removed).
|
||||
- Shared-key auth requires `x-shared-key` as `<key-id> <key>` and stores the lowercased key id on the request. If no shared keys are configured it always rejects.
|
||||
- Session management uses DB storage unless the DB pool is read-only, then falls back to the in-memory manager. DB sessions support both legacy string ids and v2 UUID session ids.
|
||||
- Session cookies are renewed when using a legacy string id or when `modified-at` is older than the renewal interval. SameSite is `none` for CORS, otherwise strict/lax based on config.
|
||||
- Session management uses DB storage unless the DB pool is read-only, then falls back to the in-memory manager. Sessions use only the v2 UUID model (`http_session_v2`); legacy string ids were removed.
|
||||
- Session cookies are renewed when `modified-at` is older than the 6h renewal interval. SameSite is `none` for CORS, otherwise strict/lax based on config. Session lifetime config and GC: `mem:backend/session-expiration`.
|
||||
|
||||
## HTTP server self-metrics
|
||||
|
||||
|
||||
@ -47,6 +47,15 @@
|
||||
- Export multiple fills to SVG [#11466](https://github.com/penpot/penpot/issues/11466) (PR: [#11467](https://github.com/penpot/penpot/pull/11467))
|
||||
- Add Penpot-specific board size presets (file thumbnail, template cover, plugin icon/cover) [#11561](https://github.com/penpot/penpot/issues/11561) (PR: [#11565](https://github.com/penpot/penpot/pull/11565))
|
||||
|
||||
## 2.18.1
|
||||
|
||||
### :bug: Bugs fixed
|
||||
|
||||
- Fix workspace freezing with a stack overflow when opening the context menu on a shape that isn't on the current page [#11882](https://github.com/penpot/penpot/issues/11882) (PR: [#11891](https://github.com/penpot/penpot/pull/11891))
|
||||
- Fix workspace showing repeated error toasts when a third-party script injected into the page throws [#11813](https://github.com/penpot/penpot/issues/11813) (PR: [#11816](https://github.com/penpot/penpot/pull/11816))
|
||||
- Fix register flow from workspace URL not working [#11753](https://github.com/penpot/penpot/issues/11753) (PR: [#11756](https://github.com/penpot/penpot/pull/11756))
|
||||
- Fix empty response when exporting bin using API (by @filipsajdak) [#7649](https://github.com/penpot/penpot/issues/7649) (PR: [#10341](https://github.com/penpot/penpot/pull/10341))
|
||||
|
||||
## 2.18.0
|
||||
|
||||
### :rocket: Epics and highlights
|
||||
|
||||
@ -61,7 +61,6 @@
|
||||
:objects-storage-fs-directory "assets"
|
||||
|
||||
:auth-token-cookie-name "auth-token"
|
||||
:auth-token-cookie-max-age-absolute (ct/duration {:days 30})
|
||||
|
||||
:assets-path "/internal/assets/"
|
||||
:smtp-default-reply-to "Penpot <no-reply@example.com>"
|
||||
@ -304,6 +303,10 @@
|
||||
[:objects-storage-s3-region {:optional true} :keyword]
|
||||
[:objects-storage-s3-endpoint {:optional true} ::sm/uri]
|
||||
|
||||
;; Write storage_object.metadata as plain JSON instead of
|
||||
;; Transit-JSON. Unset by default (Phase 1: keep writing Transit).
|
||||
[:storage-metadata-as-json {:optional true} ::sm/boolean]
|
||||
|
||||
;; SSRF protection
|
||||
[:ssrf-allowed-hosts {:optional true} [::sm/set :string]]
|
||||
[:ssrf-extra-blocked-cidrs {:optional true} [::sm/set :string]]]))
|
||||
|
||||
@ -675,7 +675,7 @@
|
||||
(contains? params :block)
|
||||
(do
|
||||
(db/update! conn :profile {:is-blocked true} {:id (:id profile)})
|
||||
(db/delete! conn :http-session {:profile-id (:id profile)})
|
||||
(session/invalidate-all cfg (:id profile))
|
||||
|
||||
{::yres/status 200
|
||||
::yres/headers {"content-type" "text/plain"}
|
||||
|
||||
@ -306,16 +306,15 @@
|
||||
(let [decode-fn (get decoders type)]
|
||||
(if (or (= type :cookie) (= type :bearer))
|
||||
(let [metadata (tokens/decode-header token)]
|
||||
;; NOTE: we only proceed to decode claims on new
|
||||
;; cookie tokens. The old cookies dont need to be
|
||||
;; decoded because they use the token string as ID
|
||||
;; NOTE: only current (kid=1/ver=1) cookie tokens carry
|
||||
;; decodable claims. Anything else is left unauthenticated.
|
||||
(if (and (= (:kid metadata) 1)
|
||||
(= (:ver metadata) 1)
|
||||
(some? decode-fn))
|
||||
(assoc request ::http/auth-data (assoc auth
|
||||
:claims (decode-fn token)
|
||||
:metadata metadata))
|
||||
(assoc request ::http/auth-data (assoc auth :metadata {:ver 0}))))
|
||||
request))
|
||||
|
||||
(if decode-fn
|
||||
(assoc request ::http/auth-data (assoc auth :claims (decode-fn token)))
|
||||
|
||||
@ -81,14 +81,8 @@
|
||||
[pool]
|
||||
(reify ISessionManager
|
||||
(read-session [_ id]
|
||||
(if (string? id)
|
||||
;; Backward compatibility: http_session (v1) has no props column
|
||||
(let [session (db/exec-one! pool (sql/select :http-session {:id id}))]
|
||||
(-> session
|
||||
(assoc :modified-at (:updated-at session))
|
||||
(dissoc :updated-at)))
|
||||
(some-> (db/exec-one! pool (sql/select :http-session-v2 {:id id}))
|
||||
(decode-session))))
|
||||
(some-> (db/exec-one! pool (sql/select :http-session-v2 {:id id}))
|
||||
(decode-session)))
|
||||
|
||||
(create-session [_ params]
|
||||
(assert (valid-params? params) "expect valid session params")
|
||||
@ -103,23 +97,15 @@
|
||||
|
||||
(update-session [_ session]
|
||||
(let [modified-at (ct/now)]
|
||||
(if (string? (:id session))
|
||||
(db/insert! pool :http-session-v2
|
||||
(-> session
|
||||
(assoc :id (uuid/next))
|
||||
(assoc :created-at modified-at)
|
||||
(assoc :modified-at modified-at)))
|
||||
(db/update! pool :http-session-v2
|
||||
(cond-> {:modified-at modified-at}
|
||||
(some? (:props session))
|
||||
(assoc :props (db/tjson (:props session))))
|
||||
{:id (:id session)}
|
||||
{::db/return-keys true}))))
|
||||
(db/update! pool :http-session-v2
|
||||
(cond-> {:modified-at modified-at}
|
||||
(some? (:props session))
|
||||
(assoc :props (db/tjson (:props session))))
|
||||
{:id (:id session)}
|
||||
{::db/return-keys true})))
|
||||
|
||||
(delete-session [_ id]
|
||||
(if (string? id)
|
||||
(db/delete! pool :http-session {:id id} {::db/return-keys false})
|
||||
(db/delete! pool :http-session-v2 {:id id} {::db/return-keys false}))
|
||||
(db/delete! pool :http-session-v2 {:id id} {::db/return-keys false})
|
||||
nil)))
|
||||
|
||||
(defn inmemory-manager
|
||||
@ -256,25 +242,28 @@
|
||||
(db/exec! pool [sql:clear-organization-sso-sessions organization-key organization-key])))
|
||||
|
||||
(defn- renew-session?
|
||||
[{:keys [id modified-at] :as session}]
|
||||
(or (string? id)
|
||||
(and (ct/inst? modified-at)
|
||||
(let [elapsed (ct/diff modified-at (ct/now))]
|
||||
(neg? (compare default-renewal-max-age elapsed))))))
|
||||
[{:keys [modified-at]}]
|
||||
(and (ct/inst? modified-at)
|
||||
(let [elapsed (ct/diff modified-at (ct/now))]
|
||||
(neg? (compare default-renewal-max-age elapsed)))))
|
||||
|
||||
(defn- session-expired?
|
||||
[{:keys [modified-at]}]
|
||||
(and (ct/inst? modified-at)
|
||||
(let [max-age (cf/get :auth-token-cookie-max-age default-cookie-max-age)]
|
||||
(pos? (compare (ct/now) (ct/plus modified-at max-age))))))
|
||||
|
||||
(defn- wrap-authz
|
||||
[handler {:keys [::manager] :as cfg}]
|
||||
(assert (manager? manager) "expected valid session manager")
|
||||
(fn [request]
|
||||
(let [{:keys [type token claims metadata]} (get request ::http/auth-data)]
|
||||
(let [{:keys [type claims]} (get request ::http/auth-data)]
|
||||
(cond
|
||||
(= type :cookie)
|
||||
(let [session
|
||||
(case (:ver metadata)
|
||||
;; BACKWARD COMPATIBILITY WITH OLD TOKENS
|
||||
0 (read-session manager token)
|
||||
1 (some->> (:sid claims) (read-session manager))
|
||||
nil)
|
||||
(let [session (some->> (:sid claims) (read-session manager))
|
||||
;; Idle-expired sessions authenticate like missing ones:
|
||||
;; no profile, no renewal. The GC deletes the row later.
|
||||
session (when-not (session-expired? session) session)
|
||||
|
||||
request
|
||||
(cond-> request
|
||||
@ -294,11 +283,8 @@
|
||||
response))
|
||||
|
||||
(= type :bearer)
|
||||
(let [session (case (:ver metadata)
|
||||
;; BACKWARD COMPATIBILITY WITH OLD TOKENS
|
||||
0 (read-session manager token)
|
||||
1 (some->> (:sid claims) (read-session manager))
|
||||
nil)
|
||||
(let [session (some->> (:sid claims) (read-session manager))
|
||||
session (when-not (session-expired? session) session)
|
||||
request (cond-> request
|
||||
(some? session)
|
||||
(-> (assoc ::profile-id (:profile-id session))
|
||||
@ -317,9 +303,9 @@
|
||||
(defn- assign-session-cookie
|
||||
[response {token :token modified-at :modified-at}]
|
||||
(let [max-age (cf/get :auth-token-cookie-max-age default-cookie-max-age)
|
||||
created-at modified-at
|
||||
renewal (ct/plus created-at default-renewal-max-age)
|
||||
expires (ct/plus created-at max-age)
|
||||
renewal-at modified-at
|
||||
renewal (ct/plus renewal-at default-renewal-max-age)
|
||||
expires (ct/plus renewal-at max-age)
|
||||
secure? (contains? cf/flags :secure-session-cookies)
|
||||
strict? (contains? cf/flags :strict-session-cookies)
|
||||
cors? (contains? cf/flags :cors)
|
||||
@ -346,40 +332,46 @@
|
||||
(defmethod ig/assert-key ::tasks/gc
|
||||
[_ params]
|
||||
(assert (db/pool? (::db/pool params)) "expected valid database pool")
|
||||
(assert (ct/duration? (::tasks/max-age params))))
|
||||
(assert (ct/duration? (::tasks/max-age params)))
|
||||
(assert (ct/duration? (::tasks/max-age-absolute params)))
|
||||
;; NOTE: explicit throw (not `assert`) so the guard fires even when
|
||||
;; the `:backend-asserts` flag leaves `*assert*` false.
|
||||
(when (pos? (compare (::tasks/max-age params)
|
||||
(::tasks/max-age-absolute params)))
|
||||
(throw (IllegalArgumentException.
|
||||
"absolute session max-age must be greater than or equal to idle max-age"))))
|
||||
|
||||
(defmethod ig/expand-key ::tasks/gc
|
||||
[k v]
|
||||
(let [max-age (cf/get :auth-token-cookie-max-age default-cookie-max-age)]
|
||||
{k (merge {::tasks/max-age max-age} (d/without-nils v))}))
|
||||
|
||||
(def ^:private
|
||||
sql:delete-expired
|
||||
"DELETE FROM http_session
|
||||
WHERE updated_at < ?::timestamptz
|
||||
or (updated_at is null and
|
||||
created_at < ?::timestamptz)")
|
||||
(let [max-age (cf/get :auth-token-cookie-max-age default-cookie-max-age)
|
||||
max-age-absolute (cf/get :auth-token-cookie-max-age-absolute
|
||||
default-cookie-max-age-absolute)]
|
||||
{k (merge {::tasks/max-age max-age
|
||||
::tasks/max-age-absolute max-age-absolute}
|
||||
(d/without-nils v))}))
|
||||
|
||||
(def ^:private
|
||||
sql:delete-expired-v2
|
||||
"DELETE FROM http_session_v2
|
||||
WHERE created_at < ?::timestamptz")
|
||||
WHERE modified_at < ?::timestamptz
|
||||
OR created_at < ?::timestamptz")
|
||||
|
||||
(defn- collect-expired-tasks
|
||||
[{:keys [::db/conn ::tasks/max-age]}]
|
||||
(let [threshold (ct/minus (ct/now) max-age)
|
||||
result-legacy (-> (db/exec-one! conn [sql:delete-expired threshold threshold])
|
||||
(db/get-update-count))
|
||||
result-v2 (-> (db/exec-one! conn [sql:delete-expired-v2 threshold])
|
||||
(db/get-update-count))]
|
||||
[{:keys [::db/conn ::tasks/max-age ::tasks/max-age-absolute]}]
|
||||
(let [idle-threshold (ct/minus (ct/now) max-age)
|
||||
abs-threshold (ct/minus (ct/now) max-age-absolute)
|
||||
result (-> (db/exec-one! conn [sql:delete-expired-v2
|
||||
idle-threshold abs-threshold])
|
||||
(db/get-update-count))]
|
||||
(l/dbg :task "gc"
|
||||
:hint "clean http sessions"
|
||||
:deleted-legacy result-legacy
|
||||
:deleted-v2 result-v2)
|
||||
(+ result-legacy result-v2)))
|
||||
:deleted result)
|
||||
result))
|
||||
|
||||
(defmethod ig/init-key ::tasks/gc
|
||||
[_ {:keys [::tasks/max-age] :as cfg}]
|
||||
(l/dbg :hint "initializing session gc task" :max-age max-age)
|
||||
[_ {:keys [::tasks/max-age ::tasks/max-age-absolute] :as cfg}]
|
||||
(l/dbg :hint "initializing session gc task"
|
||||
:max-age max-age
|
||||
:max-age-absolute max-age-absolute)
|
||||
(fn [_]
|
||||
(db/tx-run! cfg collect-expired-tasks)))
|
||||
|
||||
@ -180,6 +180,12 @@
|
||||
::mdef/labels ["route" "backend" "bucket" "result"]
|
||||
::mdef/type :counter}
|
||||
|
||||
:storage-gc-poison
|
||||
{::mdef/name "penpot_storage_gc_poison_total"
|
||||
::mdef/help "Storage objects deferred by storage-gc-touched because their metadata is corrupt."
|
||||
::mdef/labels []
|
||||
::mdef/type :counter}
|
||||
|
||||
:http-server-dispatch-timing
|
||||
{::mdef/name "penpot_http_server_dispatch_timing"
|
||||
::mdef/help "Histogram of dispatch handler"
|
||||
@ -271,7 +277,8 @@
|
||||
::sto/storage (ig/ref ::sto/storage)}
|
||||
|
||||
::sto.gc-touched/handler
|
||||
{::db/pool (ig/ref ::db/pool)}
|
||||
{::db/pool (ig/ref ::db/pool)
|
||||
::mtx/metrics (ig/ref ::mtx/metrics)}
|
||||
|
||||
::sto.pending-gc/handler
|
||||
{::db/pool (ig/ref ::db/pool)
|
||||
|
||||
@ -505,7 +505,16 @@
|
||||
:fn (mg/resource "app/migrations/sql/0153-add-storage-object-status-and-deletion-attempts.sql")}
|
||||
|
||||
{:name "0154-add-upload-session-chunk-table"
|
||||
:fn (mg/resource "app/migrations/sql/0154-add-upload-session-chunk-table.sql")}])
|
||||
:fn (mg/resource "app/migrations/sql/0154-add-upload-session-chunk-table.sql")}
|
||||
|
||||
{:name "0155-normalize-storage-object-metadata"
|
||||
:fn (mg/resource "app/migrations/sql/0155-normalize-storage-object-metadata.sql")}
|
||||
|
||||
{:name "0155-drop-http-session-table"
|
||||
:fn (mg/resource "app/migrations/sql/0155-drop-http-session-table.sql")}
|
||||
|
||||
{:name "0156-add-storage-object-json-dedup-index"
|
||||
:fn (mg/resource "app/migrations/sql/0156-add-storage-object-json-dedup-index.sql")}])
|
||||
|
||||
(defn apply-migrations!
|
||||
[pool name migrations]
|
||||
|
||||
@ -0,0 +1,4 @@
|
||||
-- Legacy v1 HTTP sessions have been removed from the backend; the
|
||||
-- http_session table is no longer read or written by any code path.
|
||||
|
||||
DROP TABLE http_session;
|
||||
@ -0,0 +1,52 @@
|
||||
--- Normalize storage_object.metadata inside Transit (Phase 1).
|
||||
---
|
||||
--- Brings every row to a canonical shape WITHOUT changing the encoding
|
||||
--- family (still Transit JSON, "~:" keys), so old and new code keep
|
||||
--- reading while it runs. Each statement below documents its own rule.
|
||||
---
|
||||
--- NOTE: key-existence is tested with `-> ... IS [NOT] NULL` instead of
|
||||
--- the `?` operator because the migration runner prepares statements
|
||||
--- and a bare `?` is parsed as a bind placeholder.
|
||||
---
|
||||
--- Idempotent: re-running changes zero rows.
|
||||
--- NOTE for large instances (e.g. our production, ~33M rows): DO NOT run
|
||||
--- this inline. Mark the migration as executed (fake) and run the batched
|
||||
--- equivalent by hand: .agents/plans/2026-09-11-storage-normalize-phase1.sql
|
||||
|
||||
-- Drop leftovers from the old chunked-upload metadata: no code reads
|
||||
-- "~:upload-id" / "~:chunk-index" anymore (#11651 moved chunks to
|
||||
-- upload_session_chunk and objects-gc purges the orphan rows), and the
|
||||
-- Phase 1 readers already ignore them on decode.
|
||||
UPDATE storage_object
|
||||
SET metadata = metadata - '~:upload-id' - '~:chunk-index'
|
||||
WHERE (metadata -> '~:upload-id') IS NOT NULL
|
||||
OR (metadata -> '~:chunk-index') IS NOT NULL;
|
||||
|
||||
-- Rows that carry only the legacy "~:reference": promote it to "~:bucket"
|
||||
-- with the same value. The value comes as a Transit keyword ("~:xxx"), so
|
||||
-- the "~:" prefix is stripped; plain strings are kept as they are.
|
||||
UPDATE storage_object
|
||||
SET metadata = (metadata - '~:reference')
|
||||
|| jsonb_build_object('~:bucket',
|
||||
CASE WHEN metadata->>'~:reference' LIKE '~:%'
|
||||
THEN substr(metadata->>'~:reference', 3)
|
||||
ELSE metadata->>'~:reference'
|
||||
END)
|
||||
WHERE (metadata -> '~:reference') IS NOT NULL
|
||||
AND (metadata -> '~:bucket') IS NULL;
|
||||
|
||||
-- Rows that carry both keys: "~:bucket" wins and the residual
|
||||
-- "~:reference" is dropped.
|
||||
UPDATE storage_object
|
||||
SET metadata = metadata - '~:reference'
|
||||
WHERE (metadata -> '~:reference') IS NOT NULL
|
||||
AND (metadata -> '~:bucket') IS NOT NULL;
|
||||
|
||||
-- Rows with neither key get the historical default bucket
|
||||
-- "file-media-object" (the value `lookup-bucket` used to fall back to).
|
||||
-- NULL columns match this rule too (`->` on NULL is NULL) and are
|
||||
-- backfilled by coalesce.
|
||||
UPDATE storage_object
|
||||
SET metadata = coalesce(metadata, '{}') || '{"~:bucket": "file-media-object"}'
|
||||
WHERE (metadata -> '~:bucket') IS NULL
|
||||
AND (metadata -> '~:reference') IS NULL;
|
||||
@ -0,0 +1,11 @@
|
||||
-- Dedup lookup index for the plain-JSON encoding of
|
||||
-- storage_object.metadata ("hash" / "bucket").
|
||||
--
|
||||
-- The dedup lookup matches both metadata encodings while Transit rows
|
||||
-- still exist (a UNION ALL of two indexable branches). The legacy "~:"
|
||||
-- index (storage_object__hash_backend_bucket__idx, migration 0068) keeps
|
||||
-- covering the Transit branch until Transit support is removed.
|
||||
|
||||
CREATE INDEX storage_object__hash_backend_bucket_json__idx
|
||||
ON storage_object ((metadata->>'hash'), (metadata->>'bucket'), backend)
|
||||
WHERE deleted_at IS NULL AND status = 'valid';
|
||||
@ -125,7 +125,7 @@
|
||||
{:columns [:id :email]})]
|
||||
(when-not (:is-blocked profile)
|
||||
(db/update! conn :profile {:is-blocked true} {:id (:id profile)})
|
||||
(db/delete! conn :http-session {:profile-id (:id profile)})
|
||||
(session/invalidate-all system (:id profile))
|
||||
:blocked))))))
|
||||
|
||||
(defn reset-password!
|
||||
|
||||
@ -20,6 +20,7 @@
|
||||
[app.storage.fs :as sfs]
|
||||
[app.storage.impl :as impl]
|
||||
[app.storage.s3 :as ss3]
|
||||
[app.storage.schema :as stsch]
|
||||
[cuerdas.core :as str]
|
||||
[datoteka.fs :as fs]
|
||||
[integrant.core :as ig])
|
||||
@ -37,28 +38,18 @@
|
||||
nil)))
|
||||
|
||||
(def default-bucket
|
||||
"file-media-object")
|
||||
stsch/default-bucket)
|
||||
|
||||
(def tempfile-bucket
|
||||
"Bucket name for temporary file uploads (10-minute expiry)."
|
||||
"tempfile")
|
||||
stsch/tempfile-bucket)
|
||||
|
||||
(def upload-session-bucket
|
||||
"Bucket name for chunked-upload chunks."
|
||||
"upload-session")
|
||||
stsch/upload-session-bucket)
|
||||
|
||||
(def valid-buckets
|
||||
#{"file-media-object"
|
||||
"team-font-variant"
|
||||
"file-object-thumbnail"
|
||||
"file-thumbnail"
|
||||
"profile"
|
||||
"organization"
|
||||
tempfile-bucket
|
||||
upload-session-bucket
|
||||
"file-data"
|
||||
"file-data-fragment"
|
||||
"file-change"})
|
||||
stsch/metadata-buckets)
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; Storage Module State
|
||||
@ -118,18 +109,37 @@
|
||||
params
|
||||
params))
|
||||
|
||||
;; Matches both metadata encodings while Transit rows still exist. The
|
||||
;; UNION ALL keeps each branch indexable: migration 0068 covers the legacy
|
||||
;; "~:" keys and 0156 covers the plain JSON keys. Phase 3 drops the legacy
|
||||
;; branch once no "~:" rows remain.
|
||||
(def ^:private sql:get-database-object-by-hash
|
||||
"(
|
||||
select * from storage_object
|
||||
where (metadata->>'~:hash') = ?
|
||||
and (metadata->>'~:bucket') = ?
|
||||
and backend = ?
|
||||
and deleted_at is null
|
||||
and status = 'valid'
|
||||
limit 1
|
||||
) union all (
|
||||
select * from storage_object
|
||||
where (metadata->>'hash') = ?
|
||||
and (metadata->>'bucket') = ?
|
||||
and backend = ?
|
||||
and deleted_at is null
|
||||
and status = 'valid'
|
||||
limit 1
|
||||
) limit 1")
|
||||
|
||||
;; NOTE: metadata is left encoded; row->storage-object is responsible for
|
||||
;; decoding it.
|
||||
(defn- get-database-object-by-hash
|
||||
[connectable backend bucket hash]
|
||||
(let [sql (str "select * from storage_object "
|
||||
" where (metadata->>'~:hash') = ? "
|
||||
" and (metadata->>'~:bucket') = ? "
|
||||
" and backend = ?"
|
||||
" and deleted_at is null"
|
||||
" and status = 'valid'"
|
||||
" limit 1")]
|
||||
;; NOTE: metadata is left encoded; row->storage-object is
|
||||
;; responsible for decoding it.
|
||||
(db/exec-one! connectable [sql hash bucket (name backend)])))
|
||||
(let [backend (name backend)]
|
||||
(db/exec-one! connectable [sql:get-database-object-by-hash
|
||||
hash bucket backend
|
||||
hash bucket backend])))
|
||||
|
||||
(defn- promote-object!
|
||||
[storage object]
|
||||
@ -147,7 +157,7 @@
|
||||
res))
|
||||
|
||||
(defn row->storage-object [res]
|
||||
(let [mdata (or (some-> (:metadata res) (db/decode-transit-pgobject)) {})]
|
||||
(let [mdata (or (some-> (:metadata res) (stsch/decode-metadata)) {})]
|
||||
(impl/storage-object
|
||||
(:id res)
|
||||
(:size res)
|
||||
@ -289,7 +299,7 @@
|
||||
{:id id
|
||||
:size (impl/get-size content)
|
||||
:backend (name backend)
|
||||
:metadata (db/tjson mdata)
|
||||
:metadata (stsch/encode-metadata mdata)
|
||||
:deleted-at expired-at
|
||||
:touched-at touched-at
|
||||
:status "pending"})
|
||||
|
||||
@ -51,7 +51,7 @@
|
||||
"DELETE FROM storage_object
|
||||
WHERE id = ANY(?::uuid[])")
|
||||
|
||||
(defn- delete-sobjects!
|
||||
(defn- delete-sobjects
|
||||
[conn ids]
|
||||
(let [ids (db/create-array conn "uuid" ids)]
|
||||
(-> (db/exec-one! conn [sql:delete-sobjects ids])
|
||||
@ -61,7 +61,7 @@
|
||||
"DELETE FROM upload_session_chunk
|
||||
WHERE object_id = ANY(?::uuid[])")
|
||||
|
||||
(defn- delete-upload-session-chunks!
|
||||
(defn- delete-upload-session-chunks
|
||||
"Remove the chunk mappings for the given storage object ids. This must run
|
||||
before the storage_object rows are deleted: the upload_session_chunk
|
||||
foreign keys are ON DELETE NO ACTION."
|
||||
@ -75,7 +75,7 @@
|
||||
deleted_at = NOW() + INTERVAL '1 day'
|
||||
WHERE id = ANY(?::uuid[])")
|
||||
|
||||
(defn- increment-attempts-and-defer!
|
||||
(defn- increment-attempts-and-defer
|
||||
[conn ids]
|
||||
(let [ids (db/create-array conn "uuid" ids)]
|
||||
(db/exec-one! conn [sql:increment-attempts-and-defer ids])))
|
||||
@ -85,7 +85,7 @@
|
||||
WHERE id = ANY(?::uuid[])
|
||||
AND deletion_attempts >= ?")
|
||||
|
||||
(defn- delete-give-up!
|
||||
(defn- delete-give-up
|
||||
[conn ids]
|
||||
(let [ids (db/create-array conn "uuid" ids)]
|
||||
(db/exec-one! conn [sql:delete-give-up ids max-attempts])))
|
||||
@ -93,7 +93,7 @@
|
||||
(defn- process-chunk
|
||||
"Attempt to delete a chunk of storage objects from a specific backend.
|
||||
|
||||
This function runs inside the caller's transaction (clean-deleted!) —
|
||||
This function runs inside the caller's transaction (clean-deleted) —
|
||||
it does NOT open its own transaction. The caller is responsible for
|
||||
ensuring the rows are locked via FOR UPDATE SKIP LOCKED before calling.
|
||||
|
||||
@ -121,18 +121,18 @@
|
||||
;; storage_object rows (NO ACTION foreign keys). It only affects
|
||||
;; objects of the upload-session bucket; for any other bucket the
|
||||
;; delete matches no rows.
|
||||
(delete-upload-session-chunks! conn ok-ids)
|
||||
(delete-sobjects! conn ok-ids))
|
||||
(delete-upload-session-chunks conn ok-ids)
|
||||
(delete-sobjects conn ok-ids))
|
||||
|
||||
(when (seq fail-ids)
|
||||
(increment-attempts-and-defer! conn fail-ids)
|
||||
(increment-attempts-and-defer conn fail-ids)
|
||||
;; NOTE: same NO ACTION ordering as above: the give-up DELETE below
|
||||
;; removes storage_object rows, so chunk mappings must go first.
|
||||
;; Deferred objects keep their rows; only the mapping of a
|
||||
;; permanently given-up object disappears early, and that object is
|
||||
;; already deleted-marked.
|
||||
(delete-upload-session-chunks! conn fail-ids)
|
||||
(let [given-up (delete-give-up! conn fail-ids)]
|
||||
(delete-upload-session-chunks conn fail-ids)
|
||||
(let [given-up (delete-give-up conn fail-ids)]
|
||||
(when (pos? (db/get-update-count given-up))
|
||||
(l/wrn :hint "giving up on orphan blob after max attempts"
|
||||
:ids fail-ids
|
||||
@ -160,7 +160,7 @@
|
||||
[conn size]
|
||||
(db/exec! conn [sql:get-deleted-chunk (ct/now) size]))
|
||||
|
||||
(defn- clean-deleted!
|
||||
(defn- clean-deleted
|
||||
[cfg]
|
||||
(loop [total 0]
|
||||
(let [deleted (db/tx-run! cfg
|
||||
@ -184,6 +184,6 @@
|
||||
(defmethod ig/init-key ::handler
|
||||
[_ cfg]
|
||||
(fn [_]
|
||||
(let [total (clean-deleted! cfg)]
|
||||
(let [total (clean-deleted cfg)]
|
||||
(l/inf :hint "task finished" :total total)
|
||||
{:deleted total})))
|
||||
|
||||
@ -24,6 +24,7 @@
|
||||
[app.common.logging :as l]
|
||||
[app.common.time :as ct]
|
||||
[app.db :as db]
|
||||
[app.metrics :as mtx]
|
||||
[app.storage :as sto]
|
||||
[app.storage.impl :as impl]
|
||||
[integrant.core :as ig]))
|
||||
@ -95,7 +96,7 @@
|
||||
SET touched_at = NULL
|
||||
WHERE id = ANY(?::uuid[])")
|
||||
|
||||
(defn- mark-freeze-in-bulk!
|
||||
(defn- mark-freeze-in-bulk
|
||||
[conn ids]
|
||||
(let [ids (db/create-array conn "uuid" ids)]
|
||||
(db/exec-one! conn [sql:mark-freeze-in-bulk ids])))
|
||||
@ -106,11 +107,21 @@
|
||||
touched_at = NULL
|
||||
WHERE id = ANY(?::uuid[])")
|
||||
|
||||
(defn- mark-delete-in-bulk!
|
||||
(defn- mark-delete-in-bulk
|
||||
[conn ids]
|
||||
(let [ids (db/create-array conn "uuid" ids)]
|
||||
(db/exec-one! conn [sql:mark-delete-in-bulk (ct/now) ids])))
|
||||
|
||||
(def ^:private sql:defer-in-bulk
|
||||
"UPDATE storage_object
|
||||
SET touched_at = ?
|
||||
WHERE id = ANY(?::uuid[])")
|
||||
|
||||
(defn- defer-in-bulk
|
||||
[conn ids timestamp]
|
||||
(let [ids (db/create-array conn "uuid" ids)]
|
||||
(db/exec-one! conn [sql:defer-in-bulk timestamp ids])))
|
||||
|
||||
;; NOTE: A getter that retrieves the key which will be used for group
|
||||
;; ids; previously we have no value, then we introduced the
|
||||
;; `:reference` prop, and then it is renamed to `:bucket` and now is
|
||||
@ -125,12 +136,17 @@
|
||||
;; have value, it means :file-media-object.
|
||||
|
||||
(defn- lookup-bucket
|
||||
[{:keys [metadata]}]
|
||||
[{:keys [id metadata]}]
|
||||
(or (some-> metadata :bucket)
|
||||
(some-> metadata :reference d/name)
|
||||
sto/default-bucket))
|
||||
(do
|
||||
;; Only reachable when the metadata column is NULL: the decode
|
||||
;; always sets :bucket on non-nil metadata (0155 also backfills
|
||||
;; NULL columns). Keep working, but make it visible.
|
||||
(l/wrn :hint "storage object without bucket metadata, using fallback"
|
||||
:id (str id))
|
||||
sto/default-bucket)))
|
||||
|
||||
(defn- process-objects!
|
||||
(defn- process-objects
|
||||
[conn has-refs? bucket objects]
|
||||
(loop [to-freeze #{}
|
||||
to-delete #{}
|
||||
@ -148,31 +164,37 @@
|
||||
:bucket bucket)
|
||||
(recur to-freeze (conj to-delete id) (rest objects))))
|
||||
(do
|
||||
(some->> (seq to-freeze) (mark-freeze-in-bulk! conn))
|
||||
(some->> (seq to-delete) (mark-delete-in-bulk! conn))
|
||||
(some->> (seq to-freeze) (mark-freeze-in-bulk conn))
|
||||
(some->> (seq to-delete) (mark-delete-in-bulk conn))
|
||||
[(count to-freeze) (count to-delete)]))))
|
||||
|
||||
(defn- process-bucket!
|
||||
(defn- process-bucket
|
||||
[conn bucket objects]
|
||||
(cond
|
||||
(= bucket "file-media-object") (process-objects! conn has-file-media-object-refs? bucket objects)
|
||||
(= bucket "team-font-variant") (process-objects! conn has-team-font-variant-refs? bucket objects)
|
||||
(= bucket "file-object-thumbnail") (process-objects! conn has-file-object-thumbnails-refs? bucket objects)
|
||||
(= bucket "file-thumbnail") (process-objects! conn has-file-thumbnails-refs? bucket objects)
|
||||
(= bucket "profile") (process-objects! conn has-profile-refs? bucket objects)
|
||||
(= bucket "file-data") (process-objects! conn has-file-data-refs? bucket objects)
|
||||
(= bucket sto/tempfile-bucket) (process-objects! conn (constantly false) sto/tempfile-bucket objects)
|
||||
(= bucket sto/upload-session-bucket) (process-objects! conn (constantly false) sto/upload-session-bucket objects)
|
||||
(= bucket "organization") (process-objects! conn (constantly false) bucket objects)
|
||||
(= bucket "file-media-object") (process-objects conn has-file-media-object-refs? bucket objects)
|
||||
(= bucket "team-font-variant") (process-objects conn has-team-font-variant-refs? bucket objects)
|
||||
(= bucket "file-object-thumbnail") (process-objects conn has-file-object-thumbnails-refs? bucket objects)
|
||||
(= bucket "file-thumbnail") (process-objects conn has-file-thumbnails-refs? bucket objects)
|
||||
(= bucket "profile") (process-objects conn has-profile-refs? bucket objects)
|
||||
(= bucket "file-data") (process-objects conn has-file-data-refs? bucket objects)
|
||||
(= bucket sto/tempfile-bucket) (process-objects conn (constantly false) sto/tempfile-bucket objects)
|
||||
(= bucket sto/upload-session-bucket) (process-objects conn (constantly false) sto/upload-session-bucket objects)
|
||||
(= bucket "organization") (process-objects conn (constantly false) bucket objects)
|
||||
:else
|
||||
(ex/raise :type :internal
|
||||
:code :unexpected-unknown-reference
|
||||
:hint (dm/fmt "unknown reference '%'" bucket))))
|
||||
|
||||
(defn process-chunk!
|
||||
(defn- defer-poison
|
||||
"Defer corrupt rows by one day in their own transaction, separate from
|
||||
the healthy-chunk work."
|
||||
[{:keys [::db/conn]} poison-ids]
|
||||
(defer-in-bulk conn poison-ids (ct/plus (ct/now) {:days 1})))
|
||||
|
||||
(defn process-chunk
|
||||
[{:keys [::db/conn]} chunk]
|
||||
(reduce-kv (fn [[nfo ndo] bucket objects]
|
||||
(let [[nfo' ndo'] (process-bucket! conn bucket objects)]
|
||||
(let [[nfo' ndo'] (process-bucket conn bucket objects)]
|
||||
[(+ nfo nfo')
|
||||
(+ ndo ndo')]))
|
||||
[0 0]
|
||||
@ -189,21 +211,45 @@
|
||||
SKIP LOCKED
|
||||
LIMIT 10")
|
||||
|
||||
(defn- try-decode-row
|
||||
"Decode a touched row, capturing corrupt metadata as poison instead
|
||||
of aborting the whole chunk. Poison rows are deferred by the caller."
|
||||
[row]
|
||||
(try
|
||||
[:ok (impl/decode-row row)]
|
||||
;; Exception, not Throwable: JVM Errors (OOM, StackOverflow) must
|
||||
;; not be swallowed as a deferrable data problem.
|
||||
(catch Exception cause
|
||||
(l/err :hint "storage object with corrupt metadata, deferring evaluation"
|
||||
:id (str (:id row))
|
||||
:cause cause)
|
||||
[:poison (:id row)])))
|
||||
|
||||
(defn get-chunk
|
||||
[conn timestamp]
|
||||
(->> (db/exec! conn [sql:get-touched-storage-objects timestamp])
|
||||
(map impl/decode-row)
|
||||
(not-empty)))
|
||||
(let [grouped (->> (db/exec! conn [sql:get-touched-storage-objects timestamp])
|
||||
(map try-decode-row)
|
||||
(group-by first))]
|
||||
{:chunk (not-empty (mapv second (:ok grouped)))
|
||||
:poison (not-empty (mapv second (:poison grouped)))}))
|
||||
|
||||
(defn- process-touched!
|
||||
[{:keys [::db/pool ::timestamp] :as cfg}]
|
||||
(defn- process-touched
|
||||
[{:keys [::db/pool ::mtx/metrics ::timestamp] :as cfg}]
|
||||
(loop [freezed 0
|
||||
deleted 0]
|
||||
(if-let [chunk (get-chunk pool timestamp)]
|
||||
(let [[nfo ndo] (db/tx-run! cfg process-chunk! chunk)]
|
||||
(recur (long (+ freezed nfo))
|
||||
(long (+ deleted ndo))))
|
||||
{:freeze freezed :delete deleted})))
|
||||
(let [{:keys [chunk poison]} (get-chunk pool timestamp)]
|
||||
(when (seq poison)
|
||||
(mtx/run! metrics :id :storage-gc-poison :inc (count poison))
|
||||
(db/tx-run! cfg defer-poison poison))
|
||||
;; Keep draining after a poison-only batch: the deferred rows leave
|
||||
;; the selection and the next batch may hold healthy objects.
|
||||
(if (or (seq chunk) (seq poison))
|
||||
(let [[nfo ndo] (if (seq chunk)
|
||||
(db/tx-run! cfg process-chunk chunk)
|
||||
[0 0])]
|
||||
(recur (long (+ freezed nfo))
|
||||
(long (+ deleted ndo))))
|
||||
{:freeze freezed :delete deleted}))))
|
||||
|
||||
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
||||
;; HANDLER
|
||||
@ -211,7 +257,8 @@
|
||||
|
||||
(defmethod ig/assert-key ::handler
|
||||
[_ params]
|
||||
(assert (db/pool? (::db/pool params)) "expect valid storage"))
|
||||
(assert (db/pool? (::db/pool params)) "expect valid storage")
|
||||
(assert (mtx/metrics? (::mtx/metrics params)) "expect valid metrics"))
|
||||
|
||||
(defmethod ig/expand-key ::handler
|
||||
[k v]
|
||||
@ -223,5 +270,5 @@
|
||||
(let [threshold (if (:skip-delay props)
|
||||
(ct/now)
|
||||
(ct/minus (ct/now) min-age))]
|
||||
(process-touched! (assoc cfg ::timestamp threshold)))))
|
||||
(process-touched (assoc cfg ::timestamp threshold)))))
|
||||
|
||||
|
||||
@ -9,8 +9,8 @@
|
||||
(:require
|
||||
[app.common.data.macros :as dm]
|
||||
[app.common.exceptions :as ex]
|
||||
[app.db :as db]
|
||||
[app.storage :as-alias sto]
|
||||
[app.storage.schema :as stsch]
|
||||
[buddy.core.codecs :as bc]
|
||||
[buddy.core.hash :as bh]
|
||||
[clojure.java.io :as jio]
|
||||
@ -26,7 +26,7 @@
|
||||
[{:keys [metadata] :as row}]
|
||||
(cond-> row
|
||||
(some? metadata)
|
||||
(assoc :metadata (db/decode-transit-pgobject metadata))))
|
||||
(assoc :metadata (stsch/decode-metadata metadata))))
|
||||
|
||||
;; --- API Definition
|
||||
|
||||
|
||||
149
backend/src/app/storage/schema.clj
Normal file
149
backend/src/app/storage/schema.clj
Normal file
@ -0,0 +1,149 @@
|
||||
;; This Source Code Form is subject to the terms of the Mozilla Public
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
|
||||
(ns app.storage.schema
|
||||
"Schema, validation and encode/decode helpers for
|
||||
`storage_object.metadata`.
|
||||
|
||||
Phase 1: reads accept both the legacy Transit encoding and plain
|
||||
JSON (sniffed by the `\"~:` marker); writes validate + normalize
|
||||
but still serialize as Transit unless the
|
||||
`:storage-metadata-as-json` config flag is set."
|
||||
(:require
|
||||
[app.common.data :as d]
|
||||
[app.common.exceptions :as ex]
|
||||
[app.common.schema :as sm]
|
||||
[app.config :as cf]
|
||||
[app.db :as db]
|
||||
[cuerdas.core :as str])
|
||||
(:import
|
||||
org.postgresql.util.PGobject))
|
||||
|
||||
(def default-bucket
|
||||
"file-media-object")
|
||||
|
||||
(def tempfile-bucket
|
||||
"Bucket name for temporary file uploads (10-minute expiry)."
|
||||
"tempfile")
|
||||
|
||||
(def upload-session-bucket
|
||||
"Bucket name for chunked-upload chunks."
|
||||
"upload-session")
|
||||
|
||||
(def bucket-requirements
|
||||
"Canonical buckets and the extra keys each one must carry, so a new
|
||||
bucket and its contract live in one place. Listed keys are declared as
|
||||
optional in `schema:metadata` and only checked for presence; their type
|
||||
is enforced by the map. Only buckets whose keys are read somewhere need
|
||||
an entry: `file-data` ids are resolved by the GC, `organization-id` is
|
||||
the logo's owner."
|
||||
{"file-media-object" #{}
|
||||
"team-font-variant" #{}
|
||||
"file-object-thumbnail" #{}
|
||||
"file-thumbnail" #{}
|
||||
"profile" #{}
|
||||
"organization" #{:organization-id}
|
||||
tempfile-bucket #{}
|
||||
upload-session-bucket #{}
|
||||
"file-data" #{:file-id :id}
|
||||
"file-data-fragment" #{}
|
||||
"file-change" #{}})
|
||||
|
||||
(def metadata-buckets
|
||||
"Canonical bucket set. `app.storage/valid-buckets` aliases it so the
|
||||
list lives in exactly one place."
|
||||
(set (keys bucket-requirements)))
|
||||
|
||||
(defn- bucket-requirements-present?
|
||||
[{:keys [bucket] :as mdata}]
|
||||
(every? #(some? (get mdata %)) (get bucket-requirements bucket #{})))
|
||||
|
||||
(def schema:metadata
|
||||
"Closed schema for `storage_object.metadata`. A single shape shared by
|
||||
every bucket: `:bucket` is checked against `metadata-buckets` and the
|
||||
rest are typed optional keys. `bucket-requirements` adds the per-bucket
|
||||
presence checks."
|
||||
[:and
|
||||
[:map {:closed true}
|
||||
[:bucket [::sm/one-of {:format :string} metadata-buckets]]
|
||||
[:content-type :string]
|
||||
[:hash {:optional true} :string]
|
||||
[:profile-id {:optional true} ::sm/uuid]
|
||||
[:organization-id {:optional true} ::sm/uuid]
|
||||
[:file-id {:optional true} ::sm/uuid]
|
||||
[:id {:optional true} ::sm/uuid]]
|
||||
[:fn {:error/message "storage metadata is missing a required key for its bucket"}
|
||||
bucket-requirements-present?]])
|
||||
|
||||
(defn- ->bucket
|
||||
[v]
|
||||
(cond
|
||||
(string? v) v
|
||||
(keyword? v) (d/name v)
|
||||
:else (str v)))
|
||||
|
||||
(defn- normalize-metadata
|
||||
"Bring decoded (or incoming) metadata to its canonical shape:
|
||||
`:bucket` is always present (`:reference` is mapped to it, keyword
|
||||
or string, and defaults to `default-bucket`), and the legacy
|
||||
`:reference`, `:upload-id` and `:chunk-index` keys are dropped."
|
||||
[mdata]
|
||||
(let [bucket (or (not-empty (->bucket (:bucket mdata)))
|
||||
(not-empty (->bucket (:reference mdata)))
|
||||
default-bucket)]
|
||||
(-> mdata
|
||||
(dissoc :reference :upload-id :chunk-index)
|
||||
(assoc :bucket bucket))))
|
||||
|
||||
;; Decoder and encoder are built once per process: creating them
|
||||
;; compiles the closed schema, and `decode-metadata` runs on every read
|
||||
;; path (get-object, dedup probes, GC batches).
|
||||
(def ^:private decode-metadata-fn
|
||||
(sm/decode-fn schema:metadata (sm/json-transformer)))
|
||||
|
||||
(def ^:private encode-metadata-fn
|
||||
(sm/encoder schema:metadata (sm/json-transformer)))
|
||||
|
||||
(defn- transit-encoded?
|
||||
"Legacy metadata is Transit json-verbose, where every top-level key is
|
||||
a keyword, so the document starts with `{\"~:`. Anchoring the check to
|
||||
the start avoids reading a plain-JSON document as Transit just because
|
||||
one of its values contains `\"~:`."
|
||||
[value]
|
||||
(and (string? value)
|
||||
(str/starts-with? (str/trim value) "{\"~:")))
|
||||
|
||||
(defn decode-metadata
|
||||
"Decode storage metadata from a PGobject, accepting both the legacy
|
||||
Transit encoding (sniffed by the `\"~:` marker) and plain JSON.
|
||||
Always returns the normalized shape with id fields as UUIDs."
|
||||
[o]
|
||||
(when (some? o)
|
||||
(let [raw (if (transit-encoded? (.getValue ^PGobject o))
|
||||
(db/decode-transit-pgobject o)
|
||||
(db/decode-json-pgobject o))]
|
||||
(when-not (map? raw)
|
||||
(ex/raise :type :internal
|
||||
:code :invalid-storage-metadata
|
||||
:hint "expected a map on storage object metadata"))
|
||||
(decode-metadata-fn (normalize-metadata raw)))))
|
||||
|
||||
(def ^:private check-metadata
|
||||
(sm/check-fn schema:metadata
|
||||
:hint "invalid storage object metadata"
|
||||
:type :validation
|
||||
:code :invalid-storage-metadata))
|
||||
|
||||
(defn encode-metadata
|
||||
"Validate, normalize and encode storage metadata into a PGobject
|
||||
ready for the `metadata` column. Serializes as Transit while the
|
||||
`:storage-metadata-as-json` config flag is unset, as plain JSON
|
||||
when it is set."
|
||||
[mdata]
|
||||
(let [mdata (check-metadata (normalize-metadata mdata))]
|
||||
(if (cf/get :storage-metadata-as-json)
|
||||
(db/json (encode-metadata-fn mdata))
|
||||
(db/tjson mdata))))
|
||||
@ -56,7 +56,17 @@
|
||||
:bucket bucket
|
||||
:content-type "text/plain"}
|
||||
(some? profile-id)
|
||||
(assoc :profile-id profile-id)))))
|
||||
(assoc :profile-id profile-id)
|
||||
|
||||
;; file-data objects require file/id
|
||||
;; references for GC (has-file-data-refs?).
|
||||
(= bucket "file-data")
|
||||
(assoc :file-id (uuid/random)
|
||||
:id (uuid/random))
|
||||
|
||||
;; organization objects require their owner id.
|
||||
(= bucket "organization")
|
||||
(assoc :organization-id (uuid/random))))))
|
||||
|
||||
(defn- make-metrics
|
||||
[]
|
||||
|
||||
@ -8,6 +8,8 @@
|
||||
(:require
|
||||
[app.common.exceptions :as ex]
|
||||
[app.common.time :as ct]
|
||||
[app.common.uuid :as uuid]
|
||||
[app.config :as cf]
|
||||
[app.db :as db]
|
||||
[app.http :as-alias http]
|
||||
[app.http.access-token]
|
||||
@ -21,6 +23,7 @@
|
||||
[backend-tests.helpers :as th]
|
||||
[clojure.string :as str]
|
||||
[clojure.test :as t]
|
||||
[integrant.core :as ig]
|
||||
[mockery.core :refer [with-mocks]]
|
||||
[yetti.request :as yreq]
|
||||
[yetti.response :as yres])
|
||||
@ -56,12 +59,11 @@
|
||||
(handler (th/make-dummy-request {}))
|
||||
(t/is (nil? (::http/auth-data @request)))
|
||||
|
||||
;; A bearer token is only attached when it is a current session
|
||||
;; token (kid=1/ver=1) and a decoder is configured. Otherwise the
|
||||
;; request stays unauthenticated.
|
||||
(handler (th/make-dummy-request {:headers {"authorization" "Bearer aaaa"}}))
|
||||
|
||||
(let [{:keys [token claims] token-type :type} (get @request ::http/auth-data)]
|
||||
(t/is (= :bearer token-type))
|
||||
(t/is (= "aaaa" token))
|
||||
(t/is (nil? claims)))))
|
||||
(t/is (nil? (::http/auth-data @request)))))
|
||||
|
||||
(t/deftest auth-middleware-3
|
||||
(let [request (volatile! nil)
|
||||
@ -73,11 +75,7 @@
|
||||
(t/is (nil? (::http/auth-data @request)))
|
||||
|
||||
(handler (th/make-dummy-request {:cookies {"auth-token" "foobar"}}))
|
||||
|
||||
(let [{:keys [token claims] token-type :type} (get @request ::http/auth-data)]
|
||||
(t/is (= :cookie token-type))
|
||||
(t/is (= "foobar" token))
|
||||
(t/is (nil? claims)))))
|
||||
(t/is (nil? (::http/auth-data @request)))))
|
||||
|
||||
(t/deftest shared-key-auth
|
||||
(let [handler (#'app.http.middleware/wrap-shared-key-auth
|
||||
@ -230,56 +228,282 @@
|
||||
(t/is (ct/inst? exp) "exp should be an instant")))
|
||||
|
||||
(t/deftest session-token-exp-based-on-created-at
|
||||
(let [cfg th/*system*
|
||||
manager (session/inmemory-manager)
|
||||
profile (th/create-profile* 1)
|
||||
session (->> (session/create-session manager {:profile-id (:id profile)
|
||||
:user-agent "user agent"})
|
||||
(#'session/assign-token cfg))
|
||||
claims (tokens/decode cfg (:token session))
|
||||
expected-exp (ct/plus (:created-at session) (ct/duration {:days 30}))]
|
||||
(let [cfg th/*system*
|
||||
manager (session/inmemory-manager)
|
||||
profile (th/create-profile* 1)
|
||||
session (->> (session/create-session manager {:profile-id (:id profile)
|
||||
:user-agent "user agent"})
|
||||
(#'session/assign-token cfg))
|
||||
claims (tokens/decode cfg (:token session))
|
||||
expected-exp (ct/plus (:created-at session) (ct/duration {:days 30}))]
|
||||
(t/is (some? (:exp claims)) "session token should contain :exp claim")
|
||||
(t/is (= (inst-ms (:exp claims))
|
||||
(inst-ms expected-exp))
|
||||
"exp should equal created-at + 30 days")))
|
||||
|
||||
(t/deftest session-token-past-exp-is-rejected
|
||||
(let [cfg th/*system*
|
||||
manager (session/inmemory-manager)
|
||||
profile (th/create-profile* 1)
|
||||
session (->> (session/create-session manager {:profile-id (:id profile)
|
||||
:user-agent "user agent"})
|
||||
(#'session/assign-token cfg))
|
||||
claims (tokens/decode cfg (:token session))
|
||||
;; Manually create a token with exp in the past
|
||||
(let [cfg th/*system*
|
||||
manager (session/inmemory-manager)
|
||||
profile (th/create-profile* 1)
|
||||
session (->> (session/create-session manager {:profile-id (:id profile)
|
||||
:user-agent "user agent"})
|
||||
(#'session/assign-token cfg))
|
||||
claims (tokens/decode cfg (:token session))
|
||||
past-claims (assoc claims :exp (ct/minus (ct/now) (ct/duration {:days 1})))
|
||||
header {:kid 1 :ver 1}
|
||||
past-token (tokens/generate cfg past-claims header)]
|
||||
past-token (tokens/generate cfg past-claims {:kid 1 :ver 1})]
|
||||
(t/is (nil? (session/decode-token cfg past-token))
|
||||
"token with exp in the past should be rejected")))
|
||||
|
||||
(t/deftest session-renewal-preserves-original-exp
|
||||
(let [cfg th/*system*
|
||||
manager (session/inmemory-manager)
|
||||
profile (th/create-profile* 1)
|
||||
created (ct/minus (ct/now) (ct/duration {:days 1}))
|
||||
session {:id (uuid/random)
|
||||
:profile-id (:id profile)
|
||||
:user-agent "user agent"
|
||||
:created-at created
|
||||
:modified-at (ct/minus (ct/now) (ct/duration {:hours 7}))}
|
||||
manager (reify session/ISessionManager
|
||||
(read-session [_ _] session)
|
||||
(create-session [_ _] session)
|
||||
(update-session [_ s] (assoc s :modified-at (ct/now)))
|
||||
(delete-session [_ _] nil))
|
||||
|
||||
old-token (:token (#'session/assign-token cfg session))
|
||||
original-exp (:exp (tokens/decode cfg old-token))
|
||||
handler (-> (fn [req] req)
|
||||
(#'session/wrap-authz (assoc th/*system* ::session/manager manager))
|
||||
(#'mw/wrap-auth {:bearer (partial session/decode-token cfg)
|
||||
:cookie (partial session/decode-token cfg)}))
|
||||
response (handler (th/make-dummy-request {:cookies {"auth-token" old-token}}))
|
||||
renewed-token (get-in response [::yres/cookies "auth-token" :value])
|
||||
renewed-claims (tokens/decode cfg renewed-token)]
|
||||
(t/is (some? original-exp) "original token should have :exp")
|
||||
(t/is (not= old-token renewed-token) "renewal should issue a new token string")
|
||||
(t/is (= (inst-ms original-exp) (inst-ms (:exp renewed-claims)))
|
||||
"renewed token should preserve the original :exp, not extend it")))
|
||||
|
||||
(t/deftest session-renewal-preserves-exp-with-db-manager
|
||||
(let [cfg th/*system*
|
||||
manager (::session/manager th/*system*)
|
||||
profile (th/create-profile* 1)
|
||||
created (session/create-session manager {:profile-id (:id profile)
|
||||
:user-agent "user agent"})
|
||||
_ (th/db-exec-one! ["UPDATE http_session_v2
|
||||
SET modified_at = now() - interval '7 hours'
|
||||
WHERE id = ?" (:id created)])
|
||||
stale (session/read-session manager (:id created))
|
||||
old-token (:token (#'session/assign-token cfg stale))
|
||||
original-exp (:exp (tokens/decode cfg old-token))
|
||||
handler (-> (fn [req] req)
|
||||
(#'session/wrap-authz cfg)
|
||||
(#'mw/wrap-auth {:bearer (partial session/decode-token cfg)
|
||||
:cookie (partial session/decode-token cfg)}))
|
||||
response (handler (th/make-dummy-request {:cookies {"auth-token" old-token}}))
|
||||
renewed (get-in response [::yres/cookies "auth-token" :value])
|
||||
renewed-exp (:exp (tokens/decode cfg renewed))
|
||||
expected-exp (ct/plus (:created-at created) (ct/duration {:days 30}))
|
||||
current (session/read-session manager (:id created))]
|
||||
(t/is (some? original-exp) "original token should have :exp")
|
||||
(t/is (some? renewed) "renewal should issue a new cookie token")
|
||||
(t/is (not= old-token renewed) "renewal should issue a new token string")
|
||||
(t/is (= (inst-ms original-exp) (inst-ms renewed-exp))
|
||||
"renewed token should preserve the original :exp, not extend it")
|
||||
(t/is (= (inst-ms expected-exp) (inst-ms renewed-exp))
|
||||
"renewed :exp should equal created-at + 30 days")
|
||||
(t/is (some? current) "session row must still exist after renewal")
|
||||
(t/is (pos? (compare (:modified-at current) (:modified-at stale)))
|
||||
"persisted modified_at must move forward on renewal")))
|
||||
|
||||
(t/deftest session-renewal-cookie-expires-diverges-from-token-exp
|
||||
(let [cfg th/*system*
|
||||
manager (::session/manager th/*system*)
|
||||
profile (th/create-profile* 1)
|
||||
created (session/create-session manager {:profile-id (:id profile)
|
||||
:user-agent "user agent"})
|
||||
_ (th/db-exec-one! ["UPDATE http_session_v2
|
||||
SET created_at = now() - interval '29 days',
|
||||
modified_at = now() - interval '7 hours'
|
||||
WHERE id = ?" (:id created)])
|
||||
stale (session/read-session manager (:id created))
|
||||
old-token (:token (#'session/assign-token cfg stale))
|
||||
handler (-> (fn [req] req)
|
||||
(#'session/wrap-authz cfg)
|
||||
(#'mw/wrap-auth {:bearer (partial session/decode-token cfg)
|
||||
:cookie (partial session/decode-token cfg)}))
|
||||
response (handler (th/make-dummy-request {:cookies {"auth-token" old-token}}))
|
||||
cookie (get-in response [::yres/cookies "auth-token"])
|
||||
renewed (:value cookie)
|
||||
renewed-exp (:exp (tokens/decode cfg renewed))
|
||||
expected-exp (ct/plus (:created-at stale) (ct/duration {:days 30}))
|
||||
close-to? (fn [a b tolerance-ms]
|
||||
(<= (Math/abs (- (inst-ms a) (inst-ms b))) tolerance-ms))]
|
||||
(t/is (some? renewed) "renewal should issue a new cookie token")
|
||||
(t/is (not= old-token renewed) "renewal should issue a new token string")
|
||||
(t/is (= (inst-ms expected-exp) (inst-ms renewed-exp))
|
||||
"renewed :exp should equal created-at + 30 days")
|
||||
(t/is (close-to? renewed-exp (ct/plus (ct/now) (ct/duration {:days 1}))
|
||||
(* 10 60 1000))
|
||||
"renewed :exp should be ~1 day out (absolute cap is near)")
|
||||
(t/is (close-to? (:expires cookie) (ct/plus (ct/now) (ct/duration {:days 7}))
|
||||
(* 10 60 1000))
|
||||
"cookie Expires should slide ~7 days out from now")
|
||||
(t/is (pos? (compare (:expires cookie) renewed-exp))
|
||||
"cookie Expires should stay ahead of the token :exp")))
|
||||
|
||||
(t/deftest legacy-session-token-is-rejected
|
||||
(let [cfg th/*system*
|
||||
manager (session/inmemory-manager)
|
||||
handler (-> (fn [req] req)
|
||||
(#'session/wrap-authz {::session/manager manager})
|
||||
(#'session/wrap-authz {::session/manager manager})
|
||||
(#'mw/wrap-auth {:bearer (partial session/decode-token cfg)
|
||||
:cookie (partial session/decode-token cfg)}))
|
||||
session (->> (session/create-session manager {:profile-id (:id profile)
|
||||
:user-agent "user agent"})
|
||||
(#'session/assign-token cfg))
|
||||
original-exp (:exp (tokens/decode cfg (:token session)))
|
||||
;; Force renewal by setting modified-at to 7 hours ago
|
||||
old-session (assoc session :modified-at (ct/minus (ct/now) (ct/duration {:hours 7})))
|
||||
response (handler (th/make-dummy-request {:cookies {"auth-token" (:token old-session)}}))
|
||||
{:keys [token claims]} (get response ::http/auth-data)
|
||||
new-exp (:exp claims)]
|
||||
(t/is (some? original-exp) "original token should have :exp")
|
||||
(t/is (some? new-exp) "renewed token should have :exp")
|
||||
(t/is (= (inst-ms original-exp)
|
||||
(inst-ms new-exp))
|
||||
"renewed token should preserve original :exp, not extend it")))
|
||||
token (tokens/generate cfg {:sid "legacy-session-id"} {:kid 0 :ver 0})
|
||||
response (handler (th/make-dummy-request {:cookies {"auth-token" token}}))]
|
||||
(t/is (nil? (get response ::http/auth-data))
|
||||
"legacy tokens must not be attached as auth data")
|
||||
(t/is (nil? (::session/profile-id response))
|
||||
"legacy tokens must not authenticate")))
|
||||
|
||||
(t/deftest session-gc-deletes-idle-and-absolute-expired-rows
|
||||
(let [profile (th/create-profile* 1)
|
||||
fresh (uuid/random)
|
||||
idle (uuid/random)
|
||||
absolute (uuid/random)
|
||||
valid (uuid/random)]
|
||||
|
||||
(th/db-exec-one! ["INSERT INTO http_session_v2 (id, profile_id, created_at, modified_at)
|
||||
VALUES (?, ?, now(), now())"
|
||||
fresh (:id profile)])
|
||||
(th/db-exec-one! ["INSERT INTO http_session_v2 (id, profile_id, created_at, modified_at)
|
||||
VALUES (?, ?, now() - interval '1 day', now() - interval '8 days')"
|
||||
idle (:id profile)])
|
||||
(th/db-exec-one! ["INSERT INTO http_session_v2 (id, profile_id, created_at, modified_at)
|
||||
VALUES (?, ?, now() - interval '31 days', now())"
|
||||
absolute (:id profile)])
|
||||
(th/db-exec-one! ["INSERT INTO http_session_v2 (id, profile_id, created_at, modified_at)
|
||||
VALUES (?, ?, now() - interval '1 day', now() - interval '6 days')"
|
||||
valid (:id profile)])
|
||||
|
||||
(db/tx-run! th/*system*
|
||||
(fn [cfg]
|
||||
(#'session/collect-expired-tasks
|
||||
(assoc cfg
|
||||
:app.http.session.tasks/max-age (ct/duration {:days 7})
|
||||
:app.http.session.tasks/max-age-absolute (ct/duration {:days 30})))))
|
||||
|
||||
(let [ids (->> (th/db-exec! ["SELECT id FROM http_session_v2 WHERE profile_id = ?" (:id profile)])
|
||||
(map :id)
|
||||
(set))]
|
||||
(t/is (contains? ids fresh) "fresh session must be kept")
|
||||
(t/is (contains? ids valid) "session within both windows must be kept")
|
||||
(t/is (not (contains? ids idle)) "idle session must be deleted")
|
||||
(t/is (not (contains? ids absolute)) "session past the absolute cap must be deleted"))))
|
||||
|
||||
(t/deftest session-gc-config-wiring
|
||||
(let [idle (ct/duration {:days 3})
|
||||
absolute (ct/duration {:days 10})]
|
||||
(with-redefs [cf/get (fn
|
||||
([k] (case k
|
||||
:auth-token-cookie-max-age idle
|
||||
:auth-token-cookie-max-age-absolute absolute
|
||||
nil))
|
||||
([k default] (case k
|
||||
:auth-token-cookie-max-age idle
|
||||
:auth-token-cookie-max-age-absolute absolute
|
||||
default)))]
|
||||
(let [expanded (ig/expand-key :app.http.session.tasks/gc {})]
|
||||
(t/is (= idle
|
||||
(get-in expanded [:app.http.session.tasks/gc
|
||||
:app.http.session.tasks/max-age]))
|
||||
"task max-age should carry the configured idle window")
|
||||
(t/is (= absolute
|
||||
(get-in expanded [:app.http.session.tasks/gc
|
||||
:app.http.session.tasks/max-age-absolute]))
|
||||
"task max-age-absolute should carry the configured absolute cap")))
|
||||
(with-redefs [cf/get (fn
|
||||
([_k] nil)
|
||||
([_k default] default))]
|
||||
(let [expanded (ig/expand-key :app.http.session.tasks/gc {})]
|
||||
(t/is (= session/default-cookie-max-age
|
||||
(get-in expanded [:app.http.session.tasks/gc
|
||||
:app.http.session.tasks/max-age]))
|
||||
"task max-age should fall back to the default idle window")
|
||||
(t/is (= session/default-cookie-max-age-absolute
|
||||
(get-in expanded [:app.http.session.tasks/gc
|
||||
:app.http.session.tasks/max-age-absolute]))
|
||||
"task max-age-absolute should fall back to the default absolute cap")))))
|
||||
|
||||
(t/deftest session-gc-rejects-absolute-below-idle
|
||||
(let [pool (:app.db/pool th/*system*)
|
||||
params {:app.db/pool pool
|
||||
:app.http.session.tasks/max-age (ct/duration {:days 7})
|
||||
:app.http.session.tasks/max-age-absolute (ct/duration {:days 30})}]
|
||||
(t/is (nil? (ig/assert-key :app.http.session.tasks/gc params))
|
||||
"absolute cap above the idle window should pass")
|
||||
(t/is (nil? (ig/assert-key :app.http.session.tasks/gc
|
||||
(assoc params
|
||||
:app.http.session.tasks/max-age-absolute
|
||||
(ct/duration {:days 7}))))
|
||||
"absolute cap equal to the idle window should pass")
|
||||
(t/is (thrown? IllegalArgumentException
|
||||
(ig/assert-key :app.http.session.tasks/gc
|
||||
(assoc params
|
||||
:app.http.session.tasks/max-age-absolute
|
||||
(ct/duration {:days 3}))))
|
||||
"absolute cap below the idle window should fail fast")))
|
||||
|
||||
(t/deftest idle-expired-session-is-rejected
|
||||
(let [cfg th/*system*
|
||||
profile (th/create-profile* 1)
|
||||
updated? (atom false)
|
||||
session {:id (uuid/random)
|
||||
:profile-id (:id profile)
|
||||
:user-agent "user agent"
|
||||
:created-at (ct/now)
|
||||
:modified-at (ct/minus (ct/now) (ct/duration {:days 8}))}
|
||||
manager (reify session/ISessionManager
|
||||
(read-session [_ _] session)
|
||||
(create-session [_ _] session)
|
||||
(update-session [_ s] (reset! updated? true) s)
|
||||
(delete-session [_ _] nil))
|
||||
token (:token (#'session/assign-token cfg session))
|
||||
handler (-> (fn [req] req)
|
||||
(#'session/wrap-authz (assoc cfg ::session/manager manager))
|
||||
(#'mw/wrap-auth {:bearer (partial session/decode-token cfg)
|
||||
:cookie (partial session/decode-token cfg)}))
|
||||
cookie-r (handler (th/make-dummy-request {:cookies {"auth-token" token}}))
|
||||
bearer-r (handler (th/make-dummy-request {:headers {"authorization" (str "Bearer " token)}}))]
|
||||
(t/is (nil? (::session/profile-id cookie-r))
|
||||
"idle-expired session must not authenticate via cookie")
|
||||
(t/is (nil? (::session/session cookie-r))
|
||||
"idle-expired session must not be attached via cookie")
|
||||
(t/is (nil? (::session/profile-id bearer-r))
|
||||
"idle-expired session must not authenticate via bearer")
|
||||
(t/is (false? @updated?)
|
||||
"idle-expired session must not be renewed")))
|
||||
|
||||
(t/deftest idle-session-within-window-is-accepted
|
||||
(let [cfg th/*system*
|
||||
profile (th/create-profile* 1)
|
||||
session {:id (uuid/random)
|
||||
:profile-id (:id profile)
|
||||
:user-agent "user agent"
|
||||
:created-at (ct/now)
|
||||
:modified-at (ct/minus (ct/now) (ct/duration {:days 6}))}
|
||||
manager (reify session/ISessionManager
|
||||
(read-session [_ _] session)
|
||||
(create-session [_ _] session)
|
||||
(update-session [_ s] (assoc s :modified-at (ct/now)))
|
||||
(delete-session [_ _] nil))
|
||||
token (:token (#'session/assign-token cfg session))
|
||||
handler (-> (fn [req] req)
|
||||
(#'session/wrap-authz (assoc cfg ::session/manager manager))
|
||||
(#'mw/wrap-auth {:bearer (partial session/decode-token cfg)
|
||||
:cookie (partial session/decode-token cfg)}))
|
||||
response (handler (th/make-dummy-request {:cookies {"auth-token" token}}))]
|
||||
(t/is (= (:id profile) (::session/profile-id response))
|
||||
"session within the idle window must authenticate")))
|
||||
|
||||
(t/deftest parse-request-illegal-argument-exception
|
||||
;; clojure.data.json raises IllegalArgumentException (case
|
||||
|
||||
@ -85,8 +85,7 @@
|
||||
(configure-storage-backend))
|
||||
|
||||
sobject (sto/put-object! storage {::sto/content (sto/content "content")
|
||||
:content-type "text/plain"
|
||||
:other "data"})
|
||||
:content-type "text/plain"})
|
||||
profile (th/create-profile* 1 {:is-active true})
|
||||
project (th/create-project* 1 {:team-id (:default-team-id profile)
|
||||
:profile-id (:id profile)})
|
||||
@ -157,8 +156,7 @@
|
||||
(let [storage (-> (:app.storage/storage th/*system*)
|
||||
(configure-storage-backend))
|
||||
sobject (sto/put-object! storage {::sto/content (sto/content "content")
|
||||
:content-type "text/plain"
|
||||
:other "data"})
|
||||
:content-type "text/plain"})
|
||||
profile (th/create-profile* 1 {:is-active true})
|
||||
|
||||
project (th/create-project* 1 {:team-id (:default-team-id profile)
|
||||
@ -214,8 +212,7 @@
|
||||
(configure-storage-backend))
|
||||
|
||||
sobject (sto/put-object! storage {::sto/content (sto/content "content")
|
||||
:content-type "text/plain"
|
||||
:other "data"})
|
||||
:content-type "text/plain"})
|
||||
|
||||
profile (th/create-profile* 1 {:is-active true})
|
||||
project (th/create-project* 1 {:team-id (:default-team-id profile)
|
||||
@ -278,8 +275,7 @@
|
||||
(let [storage (-> (:app.storage/storage th/*system*)
|
||||
(configure-storage-backend))
|
||||
sobject (sto/put-object! storage {::sto/content (sto/content "content")
|
||||
:content-type "text/plain"
|
||||
:other "data"})
|
||||
:content-type "text/plain"})
|
||||
profile (th/create-profile* 1 {:is-active true})
|
||||
project (th/create-project* 1 {:team-id (:default-team-id profile)
|
||||
:profile-id (:id profile)})
|
||||
|
||||
276
backend/test/backend_tests/storage_metadata_test.clj
Normal file
276
backend/test/backend_tests/storage_metadata_test.clj
Normal file
@ -0,0 +1,276 @@
|
||||
;; This Source Code Form is subject to the terms of the Mozilla Public
|
||||
;; License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
;;
|
||||
;; Copyright (c) KALEIDOS SUBSIDIARY SL
|
||||
|
||||
(ns backend-tests.storage-metadata-test
|
||||
(:require
|
||||
[app.config :as cf]
|
||||
[app.db :as db]
|
||||
[app.storage.schema :as stsch]
|
||||
[clojure.string :as str]
|
||||
[clojure.test :as t])
|
||||
(:import
|
||||
org.postgresql.util.PGobject))
|
||||
|
||||
(defn- pgobject
|
||||
[^String value]
|
||||
(doto (PGobject.)
|
||||
(.setType "jsonb")
|
||||
(.setValue value)))
|
||||
|
||||
;; Raw Transit payloads shaped like the production rows sampled on
|
||||
;; 2026-09-11 (verbose transit: "~:key" keys, "~u<uuid>" uuid values,
|
||||
;; "~:xxx" keyword values).
|
||||
(def ^:private transit-media
|
||||
"{\"~:hash\":\"blake2b:9f1c2e\",\"~:bucket\":\"file-media-object\",\"~:content-type\":\"image/png\"}")
|
||||
|
||||
(def ^:private transit-tempfile
|
||||
"{\"~:hash\":\"blake2b:9f1c2e\",\"~:bucket\":\"tempfile\",\"~:content-type\":\"application/zip\",\"~:profile-id\":\"~u86907e95-1cb8-8122-8008-4eb7ba07d89d\"}")
|
||||
|
||||
(def ^:private transit-legacy-reference
|
||||
"{\"~:reference\":\"~:file-media-object\",\"~:content-type\":\"image/png\",\"~:hash\":\"blake2b:9f1c2e\"}")
|
||||
|
||||
(def ^:private transit-no-bucket
|
||||
"{\"~:content-type\":\"image/svg+xml\"}")
|
||||
|
||||
(def ^:private transit-chunk-leftovers
|
||||
"{\"~:bucket\":\"tempfile\",\"~:content-type\":\"application/zip\",\"~:upload-id\":\"~u86907e95-1cb8-8122-8008-4eb7ba07d89d\",\"~:chunk-index\":3}")
|
||||
|
||||
(def ^:private json-file-data
|
||||
"{\"bucket\":\"file-data\",\"content-type\":\"application/octet-stream\",\"file-id\":\"86907e95-1cb8-8122-8008-4eb7ba07d89d\",\"id\":\"83df2f92-6bd4-4e6d-9c9a-3f6d2b1a4c55\"}")
|
||||
|
||||
(t/deftest decode-transit-returns-native-types
|
||||
(let [mdata (stsch/decode-metadata (pgobject transit-tempfile))]
|
||||
(t/is (= "tempfile" (:bucket mdata)))
|
||||
(t/is (= "application/zip" (:content-type mdata)))
|
||||
(t/is (= "blake2b:9f1c2e" (:hash mdata)))
|
||||
(t/is (uuid? (:profile-id mdata)))
|
||||
(t/is (= (parse-uuid "86907e95-1cb8-8122-8008-4eb7ba07d89d")
|
||||
(:profile-id mdata)))))
|
||||
|
||||
(t/deftest decode-transit-maps-reference-to-bucket
|
||||
(let [mdata (stsch/decode-metadata (pgobject transit-legacy-reference))]
|
||||
(t/is (= "file-media-object" (:bucket mdata)))
|
||||
(t/is (= "image/png" (:content-type mdata)))
|
||||
(t/is (not (contains? mdata :reference)))))
|
||||
|
||||
(t/deftest decode-transit-defaults-missing-bucket
|
||||
(let [mdata (stsch/decode-metadata (pgobject transit-no-bucket))]
|
||||
(t/is (= "file-media-object" (:bucket mdata)))
|
||||
(t/is (= "image/svg+xml" (:content-type mdata)))))
|
||||
|
||||
(t/deftest decode-transit-drops-chunk-leftovers
|
||||
(let [mdata (stsch/decode-metadata (pgobject transit-chunk-leftovers))]
|
||||
(t/is (= "tempfile" (:bucket mdata)))
|
||||
(t/is (not (contains? mdata :upload-id)))
|
||||
(t/is (not (contains? mdata :chunk-index)))))
|
||||
|
||||
(t/deftest decode-json-coerces-uuids
|
||||
(let [mdata (stsch/decode-metadata (pgobject json-file-data))]
|
||||
(t/is (= "file-data" (:bucket mdata)))
|
||||
(t/is (uuid? (:file-id mdata)))
|
||||
(t/is (uuid? (:id mdata)))
|
||||
(t/is (= (parse-uuid "86907e95-1cb8-8122-8008-4eb7ba07d89d")
|
||||
(:file-id mdata)))))
|
||||
|
||||
(t/deftest sniff-has-no-false-positives-on-values
|
||||
;; "~:" inside a value (not anchored to a quote) must not trigger
|
||||
;; the transit branch.
|
||||
(let [mdata (stsch/decode-metadata
|
||||
(pgobject "{\"bucket\":\"tempfile\",\"content-type\":\"text/plain;~:x\"}"))]
|
||||
(t/is (= "tempfile" (:bucket mdata)))
|
||||
(t/is (= "text/plain;~:x" (:content-type mdata)))))
|
||||
|
||||
(t/deftest encode-writes-transit-by-default
|
||||
;; Pinned off: without the binding this test inherits the ambient
|
||||
;; config and proves nothing where the flag is set.
|
||||
(binding [cf/config (assoc cf/config :storage-metadata-as-json nil)]
|
||||
(let [encoded (stsch/encode-metadata {:bucket "file-media-object"
|
||||
:content-type "image/png"
|
||||
:hash "blake2b:9f1c2e"})
|
||||
value (.getValue ^PGobject encoded)]
|
||||
(t/is (string? value))
|
||||
(t/is (str/includes? value "\"~:bucket\""))
|
||||
(t/is (not (str/includes? value "\"~:reference\""))))))
|
||||
|
||||
(t/deftest encode-normalizes-legacy-input
|
||||
(binding [cf/config (assoc cf/config :storage-metadata-as-json nil)]
|
||||
(let [encoded (stsch/encode-metadata {:reference :file-media-object
|
||||
:content-type "image/png"})
|
||||
value (.getValue ^PGobject encoded)
|
||||
decoded (stsch/decode-metadata encoded)]
|
||||
(t/is (= "file-media-object" (:bucket decoded)))
|
||||
(t/is (not (str/includes? value "\"~:reference\""))))))
|
||||
|
||||
(t/deftest encode-writes-plain-json-with-flag
|
||||
(binding [cf/config (assoc cf/config :storage-metadata-as-json true)]
|
||||
(let [file-id (parse-uuid "86907e95-1cb8-8122-8008-4eb7ba07d89d")
|
||||
encoded (stsch/encode-metadata {:bucket "file-data"
|
||||
:content-type "application/octet-stream"
|
||||
:file-id file-id
|
||||
:id (parse-uuid "83df2f92-6bd4-4e6d-9c9a-3f6d2b1a4c55")})
|
||||
value (.getValue ^PGobject encoded)]
|
||||
(t/is (str/includes? value "\"bucket\""))
|
||||
(t/is (not (str/includes? value "\"~:")))
|
||||
;; uuids travel as plain strings on the JSON encoding
|
||||
(t/is (str/includes? value (str file-id)))
|
||||
(let [decoded (stsch/decode-metadata encoded)]
|
||||
(t/is (uuid? (:file-id decoded)))
|
||||
(t/is (= file-id (:file-id decoded)))))))
|
||||
|
||||
(t/deftest encode-rejects-string-uuids
|
||||
;; Encoding does not coerce input types: callers must pass native UUIDs
|
||||
;; (every producer does; reads still coerce on the way back).
|
||||
(t/is (thrown-with-msg? clojure.lang.ExceptionInfo
|
||||
#"invalid storage object metadata"
|
||||
(stsch/encode-metadata {:bucket "tempfile"
|
||||
:content-type "application/zip"
|
||||
:profile-id "86907e95-1cb8-8122-8008-4eb7ba07d89d"}))))
|
||||
|
||||
(t/deftest encode-rejects-unknown-bucket
|
||||
(t/is (thrown-with-msg? clojure.lang.ExceptionInfo
|
||||
#"invalid storage object metadata"
|
||||
(stsch/encode-metadata {:bucket "no-such-bucket"
|
||||
:content-type "image/png"}))))
|
||||
|
||||
(t/deftest encode-rejects-extra-key
|
||||
(t/is (thrown-with-msg? clojure.lang.ExceptionInfo
|
||||
#"invalid storage object metadata"
|
||||
(stsch/encode-metadata {:bucket "file-media-object"
|
||||
:content-type "image/png"
|
||||
:other "data"}))))
|
||||
|
||||
(t/deftest encode-rejects-malformed-uuid
|
||||
(t/is (thrown-with-msg? clojure.lang.ExceptionInfo
|
||||
#"invalid storage object metadata"
|
||||
(stsch/encode-metadata {:bucket "file-data"
|
||||
:content-type "application/octet-stream"
|
||||
:file-id "not-a-uuid"
|
||||
:id "83df2f92-6bd4-4e6d-9c9a-3f6d2b1a4c55"}))))
|
||||
|
||||
(t/deftest encode-rejects-missing-content-type
|
||||
(t/is (thrown-with-msg? clojure.lang.ExceptionInfo
|
||||
#"invalid storage object metadata"
|
||||
(stsch/encode-metadata {:bucket "file-media-object"}))))
|
||||
|
||||
(t/deftest encode-rejects-missing-file-data-ids
|
||||
;; Both ids are required for file-data; omitting one fails the bucket
|
||||
;; requirement check (the file-id here is a native UUID, so the map
|
||||
;; itself is valid).
|
||||
(t/is (thrown-with-msg? clojure.lang.ExceptionInfo
|
||||
#"invalid storage object metadata"
|
||||
(stsch/encode-metadata {:bucket "file-data"
|
||||
:content-type "application/octet-stream"
|
||||
:file-id (parse-uuid "86907e95-1cb8-8122-8008-4eb7ba07d89d")}))))
|
||||
|
||||
(t/deftest encode-rejects-missing-organization-id
|
||||
(t/is (thrown-with-msg? clojure.lang.ExceptionInfo
|
||||
#"invalid storage object metadata"
|
||||
(stsch/encode-metadata {:bucket "organization"
|
||||
:content-type "image/svg+xml"}))))
|
||||
|
||||
(t/deftest decode-real-transit-payload
|
||||
;; Same legacy shape as above but produced by the real transit
|
||||
;; encoder, so the test breaks if the encoding ever drifts.
|
||||
(let [mdata (stsch/decode-metadata
|
||||
(db/tjson {:reference :team-font-variant
|
||||
:content-type "font/woff2"}))]
|
||||
(t/is (= "team-font-variant" (:bucket mdata)))
|
||||
(t/is (not (contains? mdata :reference)))))
|
||||
|
||||
(t/deftest transit-and-json-encodings-decode-to-same-shape
|
||||
(let [mdata {:bucket "tempfile"
|
||||
:content-type "application/zip"
|
||||
:profile-id (parse-uuid "86907e95-1cb8-8122-8008-4eb7ba07d89d")}
|
||||
transit (binding [cf/config (assoc cf/config :storage-metadata-as-json nil)]
|
||||
(stsch/decode-metadata (stsch/encode-metadata mdata)))
|
||||
json (binding [cf/config (assoc cf/config :storage-metadata-as-json true)]
|
||||
(stsch/decode-metadata (stsch/encode-metadata mdata)))]
|
||||
(t/is (= transit json))
|
||||
(t/is (= mdata json))))
|
||||
|
||||
(t/deftest flag-on-and-off-differ-only-in-encoding-family
|
||||
;; The Phase 2 rollback contract: flipping the flag changes how the
|
||||
;; same logical metadata hits the disk, never what it means.
|
||||
(let [mdata {:bucket "file-media-object"
|
||||
:content-type "image/png"
|
||||
:hash "blake2b:9f1c2e"}
|
||||
transit (binding [cf/config (assoc cf/config :storage-metadata-as-json nil)]
|
||||
(.getValue ^PGobject (stsch/encode-metadata mdata)))
|
||||
json (binding [cf/config (assoc cf/config :storage-metadata-as-json true)]
|
||||
(.getValue ^PGobject (stsch/encode-metadata mdata)))]
|
||||
(t/is (str/includes? transit "\"~:bucket\""))
|
||||
(t/is (str/includes? json "\"bucket\""))
|
||||
(t/is (not (str/includes? json "\"~:")))
|
||||
(t/is (= (stsch/decode-metadata (pgobject transit))
|
||||
(stsch/decode-metadata (pgobject json))))))
|
||||
|
||||
(t/deftest decode-json-keeps-hash-byte-exact
|
||||
;; Dedup matches on the hash string; it must survive the JSON
|
||||
;; roundtrip untouched.
|
||||
(let [mdata (stsch/decode-metadata
|
||||
(pgobject "{\"bucket\":\"file-media-object\",\"content-type\":\"image/png\",\"hash\":\"blake2b:9f1c2e\"}"))]
|
||||
(t/is (= "blake2b:9f1c2e" (:hash mdata)))))
|
||||
|
||||
(def ^:private bucket-samples
|
||||
;; One representative payload per bucket, with every required key and
|
||||
;; one optional key where the bucket declares any.
|
||||
[["file-media-object"
|
||||
{:bucket "file-media-object" :content-type "image/png" :hash "blake2b:9f1c2e"}]
|
||||
["team-font-variant"
|
||||
{:bucket "team-font-variant" :content-type "font/woff2"}]
|
||||
["file-object-thumbnail"
|
||||
{:bucket "file-object-thumbnail" :content-type "image/png"}]
|
||||
["file-thumbnail"
|
||||
{:bucket "file-thumbnail" :content-type "image/png"}]
|
||||
["profile"
|
||||
{:bucket "profile" :content-type "image/png"}]
|
||||
["organization"
|
||||
{:bucket "organization" :content-type "image/svg+xml"
|
||||
:organization-id #uuid "11111111-2222-3333-4444-555555555555"}]
|
||||
["tempfile"
|
||||
{:bucket "tempfile" :content-type "application/zip"
|
||||
:profile-id #uuid "86907e95-1cb8-8122-8008-4eb7ba07d89d"}]
|
||||
["upload-session"
|
||||
{:bucket "upload-session" :content-type "application/octet-stream"}]
|
||||
["file-data"
|
||||
{:bucket "file-data" :content-type "application/octet-stream"
|
||||
:file-id #uuid "86907e95-1cb8-8122-8008-4eb7ba07d89d"
|
||||
:id #uuid "83df2f92-6bd4-4e6d-9c9a-3f6d2b1a4c55"}]
|
||||
["file-data-fragment"
|
||||
{:bucket "file-data-fragment" :content-type "application/octet-stream"}]
|
||||
["file-change"
|
||||
{:bucket "file-change" :content-type "application/octet-stream"}]])
|
||||
|
||||
(t/deftest encode-decode-roundtrips-every-bucket
|
||||
(binding [cf/config (assoc cf/config :storage-metadata-as-json nil)]
|
||||
(doseq [[bucket mdata] bucket-samples]
|
||||
(t/testing bucket
|
||||
(t/is (= mdata (stsch/decode-metadata (stsch/encode-metadata mdata))))))))
|
||||
|
||||
(t/deftest json-encoding-roundtrips-every-bucket
|
||||
(binding [cf/config (assoc cf/config :storage-metadata-as-json true)]
|
||||
(doseq [[bucket mdata] bucket-samples]
|
||||
(t/testing bucket
|
||||
(t/is (= mdata (stsch/decode-metadata (stsch/encode-metadata mdata))))))))
|
||||
|
||||
(t/deftest metadata-buckets-is-the-canonical-set
|
||||
(t/is (= #{"file-media-object" "team-font-variant" "file-object-thumbnail"
|
||||
"file-thumbnail" "profile" "organization" "tempfile"
|
||||
"upload-session" "file-data" "file-data-fragment" "file-change"}
|
||||
stsch/metadata-buckets)))
|
||||
|
||||
(t/deftest decode-metadata-returns-nil-for-nil
|
||||
(t/is (nil? (stsch/decode-metadata nil))))
|
||||
|
||||
(t/deftest sniff-treats-leading-tilde-value-as-json
|
||||
;; A plain-JSON value starting with `~:` must not switch the reader to
|
||||
;; the Transit branch: the sniff is anchored to the first key.
|
||||
(let [encoded (binding [cf/config (assoc cf/config :storage-metadata-as-json true)]
|
||||
(stsch/encode-metadata {:bucket "tempfile"
|
||||
:content-type "~:not-transit"}))
|
||||
decoded (stsch/decode-metadata encoded)]
|
||||
(t/is (= "~:not-transit" (:content-type decoded)))
|
||||
(t/is (= "tempfile" (:bucket decoded)))))
|
||||
@ -9,12 +9,16 @@
|
||||
[app.common.exceptions :as ex]
|
||||
[app.common.time :as ct]
|
||||
[app.common.uuid :as uuid]
|
||||
[app.config :as cf]
|
||||
[app.db :as db]
|
||||
[app.metrics :as mtx]
|
||||
[app.metrics.definition :as-alias mdef]
|
||||
[app.rpc :as-alias rpc]
|
||||
[app.storage :as sto]
|
||||
[app.storage.fs :as-alias sto.fs]
|
||||
[app.storage.impl :as impl]
|
||||
[app.storage.s3 :as-alias sto.s3]
|
||||
[app.storage.schema :as stsch]
|
||||
[backend-tests.helpers :as th]
|
||||
[clojure.test :as t]
|
||||
[cuerdas.core :as str]
|
||||
@ -23,6 +27,11 @@
|
||||
[mockery.core :refer [with-mocks]]
|
||||
[promesa.core :as p])
|
||||
(:import
|
||||
(io.prometheus.client
|
||||
Counter
|
||||
Counter$Child)
|
||||
(org.postgresql.util
|
||||
PGobject)
|
||||
(software.amazon.awssdk.services.s3
|
||||
S3AsyncClient)
|
||||
(software.amazon.awssdk.services.s3.model
|
||||
@ -41,24 +50,51 @@
|
||||
[storage]
|
||||
(assoc storage ::sto/backend :fs))
|
||||
|
||||
(defn- poison-counter-value
|
||||
"Read the process-wide gc-poison counter from the test system metrics."
|
||||
[]
|
||||
(let [metrics (:app.metrics/metrics th/*system*)
|
||||
collector (mtx/get-collector metrics :storage-gc-poison)
|
||||
instance (::mdef/instance collector)]
|
||||
(.get ^Counter$Child (.labels ^Counter instance (into-array String [])))))
|
||||
|
||||
(t/deftest put-and-retrieve-object
|
||||
(let [storage (-> (:app.storage/storage th/*system*)
|
||||
(configure-storage-backend))
|
||||
content (sto/content "content")
|
||||
object (sto/put-object! storage {::sto/content content
|
||||
:content-type "text/plain"
|
||||
:other "data"})]
|
||||
:content-type "text/plain"})]
|
||||
|
||||
(t/is (sto/object? object))
|
||||
(t/is (fs/path? (sto/get-object-path storage object)))
|
||||
|
||||
(t/is (nil? (:expired-at object)))
|
||||
(t/is (= :fs (:backend object)))
|
||||
(t/is (= "data" (:other (meta object))))
|
||||
(t/is (= "file-media-object" (:bucket (meta object))))
|
||||
(t/is (= "text/plain" (:content-type (meta object))))
|
||||
(t/is (= "content" (slurp (sto/get-object-data storage object))))
|
||||
(t/is (= "content" (slurp (sto/get-object-path storage object))))))
|
||||
|
||||
(t/deftest put-and-retrieve-with-json-metadata-flag
|
||||
;; End-to-end through the jsonb column: with the flag on, the row is
|
||||
;; stored as plain JSON and still decodes to native types.
|
||||
(binding [cf/config (assoc cf/config :storage-metadata-as-json true)]
|
||||
(let [storage (-> (:app.storage/storage th/*system*)
|
||||
(configure-storage-backend))
|
||||
profile (uuid/random)
|
||||
object (sto/put-object! storage {::sto/content (sto/content "content")
|
||||
:bucket "tempfile"
|
||||
:content-type "application/zip"
|
||||
:profile-id profile})
|
||||
row (th/db-exec-one! ["select metadata::text as metadata from storage_object where id = ?"
|
||||
(:id object)])]
|
||||
(t/is (not (str/includes? (:metadata row) "\"~:")))
|
||||
(t/is (str/includes? (:metadata row) "\"bucket\""))
|
||||
(let [loaded (sto/get-object storage (:id object))]
|
||||
(t/is (= "tempfile" (:bucket (meta loaded))))
|
||||
(t/is (= "application/zip" (:content-type (meta loaded))))
|
||||
(t/is (= profile (:profile-id (meta loaded))))))))
|
||||
|
||||
(t/deftest tempfile-objects-are-not-deduplicated
|
||||
(let [storage (-> (:app.storage/storage th/*system*)
|
||||
(configure-storage-backend))
|
||||
@ -94,8 +130,7 @@
|
||||
(configure-storage-backend))
|
||||
content (sto/content "content")
|
||||
object (sto/put-object! storage {::sto/content content
|
||||
:content-type "text/plain"
|
||||
:expired-at (ct/in-future {:seconds 1})})]
|
||||
:content-type "text/plain"})]
|
||||
(t/is (sto/object? object))
|
||||
(t/is (true? (sto/del-object! storage (:id object))))
|
||||
|
||||
@ -109,6 +144,18 @@
|
||||
;; marked as deleted/expired.
|
||||
(t/is (nil? (sto/get-object storage (:id object))))))
|
||||
|
||||
(t/deftest delete-expired-object-returns-false
|
||||
;; An object stored with `::sto/expired-at` is born with `deleted_at`
|
||||
;; set, so deleting it is a no-op and must report `false`.
|
||||
(let [storage (-> (:app.storage/storage th/*system*)
|
||||
(configure-storage-backend))
|
||||
content (sto/content "content")
|
||||
object (sto/put-object! storage {::sto/content content
|
||||
:content-type "text/plain"
|
||||
::sto/expired-at (ct/in-future {:hours 1})})]
|
||||
(t/is (some? (:expired-at object)))
|
||||
(t/is (false? (sto/del-object! storage (:id object))))))
|
||||
|
||||
(t/deftest deleted-gc-task
|
||||
(let [storage (-> (:app.storage/storage th/*system*)
|
||||
(configure-storage-backend))
|
||||
@ -438,6 +485,146 @@
|
||||
(let [row (th/db-exec-one! ["select deleted_at from storage_object where id = ?" (:id object1)])]
|
||||
(t/is (ct/is-before-or-equal? (:deleted-at row) (ct/plus now {:seconds 1}))))))
|
||||
|
||||
(t/deftest storage-gc-touched-null-metadata
|
||||
;; A NULL metadata column (predates any normalization) flows through
|
||||
;; the lookup fallback with a warning instead of breaking the GC loop.
|
||||
(let [storage (-> (:app.storage/storage th/*system*)
|
||||
(configure-storage-backend))
|
||||
content (sto/content "content")
|
||||
object (sto/put-object! storage {::sto/content content
|
||||
::sto/touch true
|
||||
:content-type "text/plain"})]
|
||||
(th/db-exec! ["update storage_object set metadata = null where id = ?" (:id object)])
|
||||
(let [res (th/run-task! :storage-gc-touched {:skip-delay true})]
|
||||
(t/is (= 0 (:freeze res)))
|
||||
(t/is (= 1 (:delete res))))))
|
||||
|
||||
(t/deftest storage-gc-touched-defers-corrupt-metadata
|
||||
;; A non-map metadata row neither blocks the chunk nor gets
|
||||
;; collected: it is logged and deferred exactly one day, keeping its
|
||||
;; metadata intact for a later repair.
|
||||
(let [now (ct/now)
|
||||
before (poison-counter-value)
|
||||
storage (-> (:app.storage/storage th/*system*)
|
||||
(configure-storage-backend))
|
||||
healthy (sto/put-object! storage {::sto/content (sto/content "healthy")
|
||||
::sto/touched-at now
|
||||
:content-type "text/plain"})
|
||||
poison (uuid/random)]
|
||||
(th/db-exec! ["insert into storage_object (id, backend, metadata, touched_at) values (?, 'fs', '[]'::jsonb, ?)"
|
||||
poison now])
|
||||
(binding [ct/*clock* (ct/fixed-clock now)]
|
||||
(let [res (th/run-task! :storage-gc-touched {:skip-delay true})]
|
||||
(t/is (= 0 (:freeze res)))
|
||||
(t/is (= 1 (:delete res)))))
|
||||
(let [row (th/db-exec-one! ["select metadata::text as metadata, touched_at from storage_object where id = ?" poison])]
|
||||
(t/is (= "[]" (:metadata row)))
|
||||
;; inst-ms: timestamptz keeps micros, the frozen clock has nanos.
|
||||
(t/is (= (inst-ms (ct/plus now {:days 1}))
|
||||
(inst-ms (:touched-at row)))))
|
||||
;; One poison row observed, one increment.
|
||||
(t/is (= (inc before) (poison-counter-value)))))
|
||||
|
||||
(t/deftest storage-gc-touched-poison-only
|
||||
;; A chunk made only of poison rows still terminates: the rows are
|
||||
;; deferred, the next scan returns nothing, and no delete is counted.
|
||||
(let [now (ct/now)
|
||||
poison (uuid/random)]
|
||||
(th/db-exec! ["insert into storage_object (id, backend, metadata, touched_at) values (?, 'fs', '[]'::jsonb, ?)"
|
||||
poison now])
|
||||
(binding [ct/*clock* (ct/fixed-clock now)]
|
||||
(let [res (th/run-task! :storage-gc-touched {:skip-delay true})]
|
||||
(t/is (= 0 (:freeze res)))
|
||||
(t/is (= 0 (:delete res)))))
|
||||
(let [row (th/db-exec-one! ["select touched_at from storage_object where id = ?" poison])]
|
||||
(t/is (= (inst-ms (ct/plus now {:days 1}))
|
||||
(inst-ms (:touched-at row)))))))
|
||||
|
||||
(t/deftest storage-gc-touched-drains-past-a-poison-only-batch
|
||||
;; A full batch of poison rows must not stop the run: they are deferred
|
||||
;; and the loop keeps draining, so a healthy row queued behind the
|
||||
;; LIMIT (later touched_at) still gets collected.
|
||||
(let [now (ct/now)
|
||||
storage (-> (:app.storage/storage th/*system*)
|
||||
(configure-storage-backend))
|
||||
earlier (ct/minus now {:seconds 1})
|
||||
healthy (sto/put-object! storage {::sto/content (sto/content "healthy")
|
||||
::sto/touched-at now
|
||||
:content-type "text/plain"})
|
||||
poisons (repeatedly 10 uuid/random)]
|
||||
(doseq [id poisons]
|
||||
(th/db-exec! ["insert into storage_object (id, backend, metadata, touched_at) values (?, 'fs', '[]'::jsonb, ?)"
|
||||
id earlier]))
|
||||
(binding [ct/*clock* (ct/fixed-clock now)]
|
||||
(let [res (th/run-task! :storage-gc-touched {:skip-delay true})]
|
||||
(t/is (= 0 (:freeze res)))
|
||||
(t/is (= 1 (:delete res)))))
|
||||
(doseq [id poisons]
|
||||
(let [row (th/db-exec-one! ["select touched_at from storage_object where id = ?" id])]
|
||||
(t/is (= (inst-ms (ct/plus now {:days 1}))
|
||||
(inst-ms (:touched-at row))))))))
|
||||
|
||||
(def ^:private migration-0155-fixtures
|
||||
;; [id transit-metadata]: production-shaped legacy rows (nil payload
|
||||
;; means a NULL column).
|
||||
[["11111111-1111-1111-1111-111111111111"
|
||||
"{\"~:reference\":\"~:file-media-object\",\"~:content-type\":\"image/png\",\"~:hash\":\"blake2b:aaa\"}"]
|
||||
["22222222-2222-2222-2222-222222222222"
|
||||
"{\"~:content-type\":\"image/svg+xml\"}"]
|
||||
["33333333-3333-3333-3333-333333333333"
|
||||
"{\"~:bucket\":\"tempfile\",\"~:reference\":\"~:tempfile\",\"~:content-type\":\"application/zip\"}"]
|
||||
["44444444-4444-4444-4444-444444444444"
|
||||
"{\"~:bucket\":\"tempfile\",\"~:content-type\":\"application/zip\",\"~:upload-id\":\"~u86907e95-1cb8-8122-8008-4eb7ba07d89d\",\"~:chunk-index\":3}"]
|
||||
["55555555-5555-5555-5555-555555555555"
|
||||
nil]])
|
||||
|
||||
(defn- run-migration-0155!
|
||||
;; Re-runs the 0155 statements (not migratus: it already applied at
|
||||
;; bootstrap) over the fixture rows above.
|
||||
[]
|
||||
(let [sql (-> (io/resource "app/migrations/sql/0155-normalize-storage-object-metadata.sql")
|
||||
(slurp))
|
||||
no-comments (->> (.split ^String sql "\n")
|
||||
(remove #(.startsWith ^String (str/trim %) "--"))
|
||||
(str/join "\n"))]
|
||||
(doseq [stmt (->> (.split ^String no-comments ";")
|
||||
(map str/trim)
|
||||
(remove str/blank?))]
|
||||
(th/db-exec! [stmt]))))
|
||||
|
||||
(defn- get-metadata-by-id
|
||||
[id]
|
||||
(:metadata (th/db-exec-one! ["select metadata from storage_object where id = ?"
|
||||
(parse-uuid id)])))
|
||||
|
||||
(t/deftest storage-migration-0155-normalizes-legacy-rows
|
||||
(doseq [[id mdata] migration-0155-fixtures]
|
||||
(th/db-exec! ["insert into storage_object (id, backend, metadata) values (?, 'fs', ?::jsonb)"
|
||||
(parse-uuid id) mdata]))
|
||||
(run-migration-0155!)
|
||||
(let [mdata (fn [id] (stsch/decode-metadata (get-metadata-by-id id)))]
|
||||
(t/is (= {:bucket "file-media-object"
|
||||
:content-type "image/png"
|
||||
:hash "blake2b:aaa"}
|
||||
(mdata "11111111-1111-1111-1111-111111111111")))
|
||||
(t/is (= {:bucket "file-media-object"
|
||||
:content-type "image/svg+xml"}
|
||||
(mdata "22222222-2222-2222-2222-222222222222")))
|
||||
(t/is (= {:bucket "tempfile"
|
||||
:content-type "application/zip"}
|
||||
(mdata "33333333-3333-3333-3333-333333333333")))
|
||||
(t/is (= {:bucket "tempfile"
|
||||
:content-type "application/zip"}
|
||||
(mdata "44444444-4444-4444-4444-444444444444")))
|
||||
(t/is (= {:bucket "file-media-object"}
|
||||
(mdata "55555555-5555-5555-5555-555555555555"))))
|
||||
;; second run changes nothing (idempotent)
|
||||
(let [raw (fn [] (mapv #(.getValue ^PGobject (get-metadata-by-id %))
|
||||
(map first migration-0155-fixtures)))
|
||||
before (raw)]
|
||||
(run-migration-0155!)
|
||||
(t/is (= before (raw)))))
|
||||
|
||||
(t/deftest storage-gc-deleted-immediate
|
||||
(let [storage (-> (:app.storage/storage th/*system*)
|
||||
(configure-storage-backend))
|
||||
@ -626,6 +813,43 @@
|
||||
(let [row (th/db-exec-one! ["select count(*) from storage_object"])]
|
||||
(t/is (= 1 (:count row))))))
|
||||
|
||||
(t/deftest dedup-reuses-json-encoded-blob
|
||||
;; With the JSON flag on, the dedup lookup must find the row it just
|
||||
;; wrote; a "~:"-only lookup used to miss it and duplicate the blob.
|
||||
(binding [cf/config (assoc cf/config :storage-metadata-as-json true)]
|
||||
(let [storage (-> (:app.storage/storage th/*system*)
|
||||
(configure-storage-backend))
|
||||
content (-> (sto/content "json-content")
|
||||
(sto/wrap-with-hash "json-hash"))
|
||||
params {::sto/content content
|
||||
::sto/deduplicate? true
|
||||
:bucket "file-media-object"
|
||||
:content-type "text/plain"}
|
||||
object1 (sto/put-object! storage params)
|
||||
object2 (sto/put-object! storage params)]
|
||||
(t/is (= (:id object1) (:id object2)))
|
||||
(let [row (th/db-exec-one! ["select count(*) from storage_object"])]
|
||||
(t/is (= 1 (:count row)))))))
|
||||
|
||||
(t/deftest dedup-json-put-finds-transit-row
|
||||
;; Both encodings coexist during the transition: a JSON write must
|
||||
;; reuse a blob already stored as Transit.
|
||||
(let [storage (-> (:app.storage/storage th/*system*)
|
||||
(configure-storage-backend))
|
||||
content (-> (sto/content "mixed-content")
|
||||
(sto/wrap-with-hash "mixed-hash"))
|
||||
params {::sto/content content
|
||||
::sto/deduplicate? true
|
||||
:bucket "file-media-object"
|
||||
:content-type "text/plain"}
|
||||
object1 (binding [cf/config (assoc cf/config :storage-metadata-as-json nil)]
|
||||
(sto/put-object! storage params))
|
||||
object2 (binding [cf/config (assoc cf/config :storage-metadata-as-json true)]
|
||||
(sto/put-object! storage params))]
|
||||
(t/is (= (:id object1) (:id object2)))
|
||||
(let [row (th/db-exec-one! ["select count(*) from storage_object"])]
|
||||
(t/is (= 1 (:count row))))))
|
||||
|
||||
(t/deftest dedup-repairs-stale-object
|
||||
(let [storage (-> (:app.storage/storage th/*system*)
|
||||
(configure-storage-backend))
|
||||
|
||||
@ -643,6 +643,44 @@ And configure it:
|
||||
PENPOT_SECRET_KEY: my-super-secure-key
|
||||
```
|
||||
|
||||
### Session expiration
|
||||
|
||||
__Since version 2.19.0__
|
||||
|
||||
User sessions are stored server-side and expire on two independent conditions: an
|
||||
**idle timeout** and an **absolute maximum lifetime**. Both are backend only.
|
||||
|
||||
```bash
|
||||
# Backend
|
||||
# Idle timeout: the session stops working after this much inactivity.
|
||||
# Default: 168h
|
||||
PENPOT_AUTH_TOKEN_COOKIE_MAX_AGE: 168h
|
||||
|
||||
# Absolute maximum lifetime from the moment the session was created,
|
||||
# regardless of activity. Default: 720h
|
||||
PENPOT_AUTH_TOKEN_COOKIE_MAX_AGE_ABSOLUTE: 720h
|
||||
```
|
||||
|
||||
Durations use the `<number><unit>` form with hour, minute or second units,
|
||||
for example `168h`, `30m` or `90s` (day units like `7d` are not accepted).
|
||||
|
||||
While a user is active the session is automatically renewed every 6 hours (not
|
||||
configurable). Renewal extends the cookie, but never the absolute maximum. The
|
||||
backend checks the idle timeout on every request, measured from the last
|
||||
renewal: once it passes, the session stops working at once, even for a copied
|
||||
token. A daily task (`session-gc`) then deletes the sessions that have exceeded
|
||||
either window. Legacy v1 sessions and the old `http_session` table are no
|
||||
longer used.
|
||||
|
||||
Sessions created before 2.19.0 carry no `:exp` in their token; they are still
|
||||
removed by the 30-day `created_at` cleanup and acquire `:exp` on their next
|
||||
renewal.
|
||||
|
||||
The `secure` and `same-site` attributes of the session cookie are controlled by
|
||||
the `disable-secure-session-cookies`, `strict-session-cookies` and `enable-cors`
|
||||
flags, and by whether `PENPOT_PUBLIC_URI` is served over HTTPS. See
|
||||
[Penpot URI](#penpot-uri).
|
||||
|
||||
### Database
|
||||
|
||||
Penpot only supports PostgreSQL and we highly recommend >=13 version. If you are using official
|
||||
|
||||
@ -94,14 +94,45 @@ Similarly as the OIDC backend, it checks if the profile exists, and calls
|
||||
|
||||
## Sessions
|
||||
|
||||
User sessions are created when a user logs in via any one of the backends. A
|
||||
session token is generated (a JWT token that does not currently contain any data)
|
||||
and returned to frontend as a cookie.
|
||||
User sessions are created when a user logs in via any one of the backends. The
|
||||
backend generates a signed JWT token and returns it to the frontend as an
|
||||
<code class="language-text">auth-token</code> cookie. A matching row is stored in
|
||||
the <code class="language-text">http_session_v2</code> table with the profile id
|
||||
and the session timestamps.
|
||||
|
||||
Normally the session is stored in a DB table with the information of the user
|
||||
profile and the session expiration. But if a frontend connects to the backend in
|
||||
"read only" mode (for example, to debug something in production with the local
|
||||
devenv), sessions are stored in memory (may be lost if the backend restarts).
|
||||
A request is authenticated only when the token verifies, its session row still
|
||||
exists and the session is within the idle timeout. The token claims carry the session row id (<code
|
||||
class="language-clojure">:sid</code>), the last activity instant (<code
|
||||
class="language-clojure">:iat</code>) and an absolute expiration (<code
|
||||
class="language-clojure">:exp</code>). The server enforces two independent
|
||||
limits:
|
||||
|
||||
* **Idle timeout:** a session that is not renewed within
|
||||
<code class="language-bash">PENPOT_AUTH_TOKEN_COOKIE_MAX_AGE</code> (default 7
|
||||
days) stops working at once. The backend checks the session's last renewal
|
||||
time on every request, so the row does not need to be deleted first.
|
||||
* **Absolute maximum:** a session cannot live longer than
|
||||
<code class="language-bash">PENPOT_AUTH_TOKEN_COOKIE_MAX_AGE_ABSOLUTE</code>
|
||||
(default 30 days) from its creation, no matter how much it is renewed. The
|
||||
<code class="language-clojure">:exp</code> claim enforces this even when the
|
||||
cookie is still present.
|
||||
|
||||
Sessions are automatically renewed every 6 hours of use (not configurable).
|
||||
Renewal issues a new token but keeps the same session row, so the absolute
|
||||
maximum is not extended. A daily garbage collector
|
||||
(<code class="language-text">session-gc</code>) deletes rows that exceed either
|
||||
the idle window or the absolute maximum.
|
||||
|
||||
Sessions created before 2.19.0 carry no <code
|
||||
class="language-clojure">:exp</code> in their token; they are still removed by
|
||||
the 30-day <code class="language-text">created_at</code> cleanup and acquire <code
|
||||
class="language-clojure">:exp</code> on their next renewal.
|
||||
|
||||
The normal storage is the database. When the backend uses a read-only database
|
||||
pool (for example, to debug something in production with the local devenv),
|
||||
sessions are kept in memory and are lost when the backend restarts. The
|
||||
organization SSO gate keeps an additional 4-hour entry inside the same session
|
||||
row, separate from the session token lifetime.
|
||||
|
||||
## Team invitations
|
||||
|
||||
|
||||
@ -54,8 +54,12 @@
|
||||
(let [el (dom/get-element "app")]
|
||||
(mf/create-root el)))
|
||||
|
||||
;; Root component of the last render; a hot reload of `app.main.ui` replaces it
|
||||
(defonce ^:private rendered-app (atom nil))
|
||||
|
||||
(defn init-ui
|
||||
[]
|
||||
(reset! rendered-app ui/app)
|
||||
(mf/render! app-root (mf/element ui/app)))
|
||||
|
||||
(defn- initialize-rasterizer
|
||||
@ -146,7 +150,10 @@
|
||||
(init-ui)))
|
||||
|
||||
(defn ^:dev/after-load after-load
|
||||
"Rerender the UI after a hot reload. When the reload replaced the root
|
||||
component, unmount the whole tree first so its cleanups run before
|
||||
the new tree mounts."
|
||||
[]
|
||||
(reinit))
|
||||
(reinit (not (identical? @rendered-app ui/app))))
|
||||
|
||||
(set! (.-stackTraceLimit js/Error) 50)
|
||||
|
||||
@ -1248,15 +1248,26 @@
|
||||
(not (r/check-permission plugin-id "library:write"))
|
||||
(u/reject-not-valid reject :connectLibrary "Plugin doesn't have 'library:write' permission")
|
||||
|
||||
(not (string? library-id))
|
||||
(not (uuid/parse* library-id))
|
||||
(u/reject-not-valid reject :connectLibrary library-id)
|
||||
|
||||
:else
|
||||
(let [file-id (:current-file-id @st/state)
|
||||
(let [file-id (:current-file-id @st/state)
|
||||
team-id (:current-team-id @st/state)
|
||||
library-id (uuid/parse library-id)]
|
||||
(->> st/stream
|
||||
(rx/filter (ptk/type? ::dwl/link-file-to-library-finished))
|
||||
(rx/take 1)
|
||||
(rx/subs! #(resolve (library-proxy plugin-id library-id)) reject))
|
||||
(st/emit! (-> (dwl/link-file-to-library file-id library-id)
|
||||
(se/add-event plugin-id))))))))))
|
||||
(->> (rp/cmd! :get-team-shared-files {:team-id team-id})
|
||||
(rx/subs!
|
||||
(fn [shared-files]
|
||||
;; Only published libraries of the team can be linked
|
||||
(if (or (= library-id file-id)
|
||||
(not (some #(= library-id (:id %)) shared-files)))
|
||||
(u/reject-not-valid reject :connectLibrary
|
||||
(dm/str library-id " is not a published library"))
|
||||
(do
|
||||
(->> st/stream
|
||||
(rx/filter (ptk/type? ::dwl/link-file-to-library-finished))
|
||||
(rx/take 1)
|
||||
(rx/subs! #(resolve (library-proxy plugin-id library-id)) reject))
|
||||
(st/emit! (-> (dwl/link-file-to-library file-id library-id)
|
||||
(se/add-event plugin-id))))))
|
||||
reject)))))))))
|
||||
|
||||
@ -2479,6 +2479,13 @@
|
||||
:cause cause)
|
||||
nil)))))
|
||||
|
||||
;; Stable listener identities, so `removeEventListener` matches across hot reloads
|
||||
(defonce ^:private webgl-context-lost-listener
|
||||
(fn [event] (on-webgl-context-lost event)))
|
||||
|
||||
(defonce ^:private webgl-context-restored-listener
|
||||
(fn [event] (on-webgl-context-restored event)))
|
||||
|
||||
(defn init-canvas-context
|
||||
[canvas]
|
||||
(if-not (wasm/module-ready?)
|
||||
@ -2523,8 +2530,8 @@
|
||||
;; Add event listeners for WebGL context lost
|
||||
(set! wasm/canvas canvas)
|
||||
(when can-listen?
|
||||
(.addEventListener canvas "webglcontextlost" on-webgl-context-lost)
|
||||
(.addEventListener canvas "webglcontextrestored" on-webgl-context-restored))
|
||||
(.addEventListener canvas "webglcontextlost" webgl-context-lost-listener)
|
||||
(.addEventListener canvas "webglcontextrestored" webgl-context-restored-listener))
|
||||
(start-canvas-snapshot-listener!)
|
||||
(reset! wasm/context-lost? false)
|
||||
(set! wasm/context-initialized? true)
|
||||
@ -2558,8 +2565,8 @@
|
||||
|
||||
;; Remove listener before losing/deleting context.
|
||||
(when wasm/canvas
|
||||
(.removeEventListener wasm/canvas "webglcontextlost" on-webgl-context-lost)
|
||||
(.removeEventListener wasm/canvas "webglcontextrestored" on-webgl-context-restored))
|
||||
(.removeEventListener wasm/canvas "webglcontextlost" webgl-context-lost-listener)
|
||||
(.removeEventListener wasm/canvas "webglcontextrestored" webgl-context-restored-listener))
|
||||
(stop-canvas-snapshot-listener!)
|
||||
|
||||
(when (wasm/module-ready?)
|
||||
|
||||
@ -8,16 +8,23 @@
|
||||
(:require
|
||||
[app.common.types.component :as ctk]
|
||||
[app.common.uuid :as uuid]
|
||||
[app.main.data.event :as ev]
|
||||
[app.main.data.workspace.libraries :as dwl]
|
||||
[app.main.data.workspace.texts :as dwt]
|
||||
[app.main.data.workspace.variants :as dwv]
|
||||
[app.main.repo :as rp]
|
||||
[app.main.store :as st]
|
||||
[app.plugins.library :as library]
|
||||
[app.plugins.register :as r]
|
||||
[app.plugins.text :as text]
|
||||
[app.plugins.utils :as u]
|
||||
[app.util.object :as obj]
|
||||
[beicon.v2.core :as rx]
|
||||
[cljs.test :as t :include-macros true]
|
||||
[frontend-tests.helpers.mock :as mock]))
|
||||
[cuerdas.core :as str]
|
||||
[frontend-tests.helpers.async :as async]
|
||||
[frontend-tests.helpers.mock :as mock]
|
||||
[potok.v2.core :as ptk]))
|
||||
|
||||
(def ^:private plugin-id "00000000-0000-0000-0000-000000000000")
|
||||
|
||||
@ -205,3 +212,103 @@
|
||||
(t/is (= 1 (count @errors)))
|
||||
(t/is (= [plugin-id :setVariantProperty "Plugin doesn't have 'library:write' permission"]
|
||||
(first @errors)))))))
|
||||
|
||||
;; ---------------------------------------------------------------------------
|
||||
;; connectLibrary
|
||||
;; ---------------------------------------------------------------------------
|
||||
|
||||
(defn- shared-files-cmd
|
||||
"Answers every RPC command asynchronously with `shared-files`, recording
|
||||
the calls in `mock/rpc-calls`."
|
||||
[shared-files]
|
||||
(mock/stub
|
||||
(fn [cmd params]
|
||||
(swap! mock/rpc-calls conj {:cmd cmd :params params})
|
||||
(rx/observe-on :async (rx/of shared-files)))))
|
||||
|
||||
(defn- settled
|
||||
"Resolves with `{:resolved value}` or `{:rejected cause}` once `p` settles,
|
||||
or with `{:timeout true}` when it stays pending for 2 seconds."
|
||||
[p]
|
||||
(js/Promise.race
|
||||
#js [(.then p (fn [v] {:resolved v}) (fn [e] {:rejected e}))
|
||||
(js/Promise. (fn [resolve] (js/setTimeout #(resolve {:timeout true}) 2000)))]))
|
||||
|
||||
(t/deftest ^:async connect-library-links-a-published-library
|
||||
(let [file-id (uuid/next)
|
||||
team-id (uuid/next)
|
||||
library-id (uuid/next)
|
||||
stream (rx/subject)
|
||||
events (atom [])]
|
||||
(await
|
||||
(mock/with-mocks*
|
||||
{st/state (atom {:current-file-id file-id :current-team-id team-id})
|
||||
st/stream stream
|
||||
st/emit! (mock/stub (fn [& emitted] (swap! events into emitted)))
|
||||
rp/cmd! (shared-files-cmd [{:id library-id :name "Published"}])
|
||||
r/check-permission (mock/stub (constantly true))}
|
||||
(let [context (library/library-subcontext plugin-id)
|
||||
result (settled (.connectLibrary context (str library-id)))
|
||||
link? #(= ::dwl/link-file-to-library (ptk/type %))]
|
||||
(await (async/wait-for #(some link? @events) "link event emitted"))
|
||||
(rx/push! stream (ptk/data-event ::dwl/link-file-to-library-finished))
|
||||
(let [{:keys [resolved]} (await result)
|
||||
link-data (ev/-data (first (filter link? @events)))]
|
||||
(t/is (= library-id (obj/get resolved "$id")))
|
||||
(t/is (= file-id (:file-id link-data)))
|
||||
(t/is (= library-id (:library-id link-data)))
|
||||
(t/is (= [{:cmd :get-team-shared-files :params {:team-id team-id}}]
|
||||
@mock/rpc-calls))))))))
|
||||
|
||||
(t/deftest ^:async connect-library-rejects-an-unpublished-file
|
||||
(let [file-id (uuid/next)
|
||||
library-id (uuid/next)
|
||||
events (atom [])]
|
||||
(await
|
||||
(mock/with-mocks*
|
||||
{st/state (atom {:current-file-id file-id :current-team-id (uuid/next)})
|
||||
st/emit! (mock/stub (fn [& emitted] (swap! events into emitted)))
|
||||
rp/cmd! (shared-files-cmd [{:id (uuid/next) :name "Other published"}])
|
||||
r/check-permission (mock/stub (constantly true))}
|
||||
(let [context (library/library-subcontext plugin-id)
|
||||
{:keys [rejected]} (await (settled (.connectLibrary context (str library-id))))]
|
||||
(t/is (str/includes? (str rejected) "is not a published library"))
|
||||
(t/is (empty? @events)))))))
|
||||
|
||||
(t/deftest ^:async connect-library-rejects-the-current-file
|
||||
(let [file-id (uuid/next)
|
||||
events (atom [])]
|
||||
(await
|
||||
(mock/with-mocks*
|
||||
{st/state (atom {:current-file-id file-id :current-team-id (uuid/next)})
|
||||
st/emit! (mock/stub (fn [& emitted] (swap! events into emitted)))
|
||||
rp/cmd! (shared-files-cmd [{:id file-id :name "Current file"}])
|
||||
r/check-permission (mock/stub (constantly true))}
|
||||
(let [context (library/library-subcontext plugin-id)
|
||||
{:keys [rejected]} (await (settled (.connectLibrary context (str file-id))))]
|
||||
(t/is (str/includes? (str rejected) "is not a published library"))
|
||||
(t/is (empty? @events)))))))
|
||||
|
||||
(t/deftest ^:async connect-library-rejects-an-invalid-id
|
||||
(await
|
||||
(mock/with-mocks*
|
||||
{st/state (atom {:current-file-id (uuid/next) :current-team-id (uuid/next)})
|
||||
st/emit! mock/noop
|
||||
rp/cmd! (shared-files-cmd [])
|
||||
r/check-permission (mock/stub (constantly true))}
|
||||
(let [context (library/library-subcontext plugin-id)
|
||||
{:keys [rejected]} (await (settled (.connectLibrary context "not-a-uuid")))]
|
||||
(t/is (str/includes? (str rejected) "not-a-uuid"))
|
||||
(t/is (empty? @mock/rpc-calls))))))
|
||||
|
||||
(t/deftest ^:async connect-library-checks-permission
|
||||
(await
|
||||
(mock/with-mocks*
|
||||
{st/state (atom {:current-file-id (uuid/next) :current-team-id (uuid/next)})
|
||||
st/emit! mock/noop
|
||||
rp/cmd! (shared-files-cmd [])
|
||||
r/check-permission (mock/stub (constantly false))}
|
||||
(let [context (library/library-subcontext plugin-id)
|
||||
{:keys [rejected]} (await (settled (.connectLibrary context (str (uuid/next)))))]
|
||||
(t/is (str/includes? (str rejected) "Plugin doesn't have 'library:write' permission"))
|
||||
(t/is (empty? @mock/rpc-calls))))))
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user