From af8477ea3c2c2907340a0958afb8138c19bc273a Mon Sep 17 00:00:00 2001 From: yousefed Date: Fri, 9 Oct 2026 09:20:54 +0200 Subject: [PATCH 1/7] =?UTF-8?q?=F0=9F=90=9B(compose)=20route=20named=20ver?= =?UTF-8?q?sions=20to=20the=20collaboration=20server?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The version history reads and writes named versions on /collaboration/version/. The helm ingress routes it, but the nginx template of the compose example did not, so named versions failed with 404 on a compose deployment. Signed-off-by: yousefed --- docker/files/production/etc/nginx/conf.d/default.conf.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/files/production/etc/nginx/conf.d/default.conf.template b/docker/files/production/etc/nginx/conf.d/default.conf.template index 0b2fee1bd8..bf306f0be5 100644 --- a/docker/files/production/etc/nginx/conf.d/default.conf.template +++ b/docker/files/production/etc/nginx/conf.d/default.conf.template @@ -80,7 +80,7 @@ server { # Document routes, guarded by the same document authorization as the # websocket, and the public keys the collaboration server signs with. - location ~ ^/collaboration/(ydoc|rollback|prune|changeset|activity|jwks)/ { + location ~ ^/collaboration/(ydoc|rollback|prune|changeset|activity|version|jwks)/ { proxy_pass http://${YHUB_HOST}:3002; proxy_set_header X-Forwarded-Proto https; proxy_set_header Origin $http_origin; From a4ba94be0b37f1ca77dc8c5179b6660dd0ab1466 Mon Sep 17 00:00:00 2001 From: yousefed Date: Fri, 9 Oct 2026 09:20:54 +0200 Subject: [PATCH 2/7] =?UTF-8?q?=E2=9C=A8(dev-instance)=20deploy=20a=20dev?= =?UTF-8?q?=20instance=20from=20the=20dev=20branch?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A push to the dev branch builds the frontend, yhub and y-provider images, pushes them to GHCR and deploys them on the dev server with docker compose. The server runs Docs with Keycloak, an S3 store and Caddy for TLS. The secrets are generated on the server, in a .env file that is not in the repository. Signed-off-by: yousefed --- .github/workflows/dev-instance.yml | 127 +++++++++++++++ deploy/dev-instance/Caddyfile | 20 +++ deploy/dev-instance/compose.yaml | 208 ++++++++++++++++++++++++ deploy/dev-instance/env.d/backend | 49 ++++++ deploy/dev-instance/env.d/common | 10 ++ deploy/dev-instance/env.d/postgresql | 9 + deploy/dev-instance/env.d/yhub | 8 + deploy/dev-instance/env.d/yprovider | 5 + deploy/dev-instance/keycloak/realm.json | 108 ++++++++++++ deploy/dev-instance/scripts/deploy.sh | 24 +++ deploy/dev-instance/scripts/init-env.sh | 54 ++++++ 11 files changed, 622 insertions(+) create mode 100644 .github/workflows/dev-instance.yml create mode 100644 deploy/dev-instance/Caddyfile create mode 100644 deploy/dev-instance/compose.yaml create mode 100644 deploy/dev-instance/env.d/backend create mode 100644 deploy/dev-instance/env.d/common create mode 100644 deploy/dev-instance/env.d/postgresql create mode 100644 deploy/dev-instance/env.d/yhub create mode 100644 deploy/dev-instance/env.d/yprovider create mode 100644 deploy/dev-instance/keycloak/realm.json create mode 100755 deploy/dev-instance/scripts/deploy.sh create mode 100755 deploy/dev-instance/scripts/init-env.sh diff --git a/.github/workflows/dev-instance.yml b/.github/workflows/dev-instance.yml new file mode 100644 index 0000000000..88ace0087c --- /dev/null +++ b/.github/workflows/dev-instance.yml @@ -0,0 +1,127 @@ +name: Dev instance +run-name: Dev instance (${{ github.event_name == 'workflow_dispatch' && inputs.seed && 'deploy and seed' || 'deploy' }}) + +# Builds the images that differ from upstream and deploys them to the dev +# server (deploy/dev-instance). A merge to the `dev` branch ships it. +on: + push: + branches: + - dev + workflow_dispatch: + inputs: + seed: + description: "Replay the meeting document into the instance after the deploy" + type: boolean + default: false + seed_force: + description: "Write a new copy of the document even if it exists" + type: boolean + default: false + +permissions: + contents: read + +# Never cancel a deploy halfway; queue the next one instead. +concurrency: + group: dev-instance + cancel-in-progress: false + +jobs: + build: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + strategy: + fail-fast: true + matrix: + include: + - name: frontend + file: src/frontend/Dockerfile + target: frontend-production + - name: yhub + file: src/yhub-server/Dockerfile + target: yhub + - name: y-provider + file: src/frontend/servers/y-provider/Dockerfile + target: y-provider + steps: + - uses: actions/checkout@v6 + - uses: docker/setup-buildx-action@v3 + - uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Image name + id: image + run: echo "name=ghcr.io/${GITHUB_REPOSITORY,,}/${{ matrix.name }}" >> "$GITHUB_OUTPUT" + - uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7 + with: + context: . + file: ${{ matrix.file }} + target: ${{ matrix.target }} + platforms: linux/amd64 + build-args: | + DOCKER_USER=1001:127 + PUBLISH_AS_MIT=false + push: true + provenance: false + tags: | + ${{ steps.image.outputs.name }}:${{ github.sha }} + ${{ steps.image.outputs.name }}:dev + cache-from: type=gha,scope=dev-${{ matrix.name }} + cache-to: type=gha,scope=dev-${{ matrix.name }},mode=max + + deploy: + needs: build + runs-on: ubuntu-latest + timeout-minutes: 150 + permissions: + contents: read + packages: read + steps: + - uses: actions/checkout@v6 + - name: Set up SSH + env: + DEPLOY_SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }} + DEPLOY_KNOWN_HOSTS: ${{ secrets.DEPLOY_KNOWN_HOSTS }} + DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }} + DEPLOY_USER: ${{ secrets.DEPLOY_USER }} + run: | + install -d -m 700 ~/.ssh + printf '%s\n' "$DEPLOY_SSH_KEY" > ~/.ssh/id_ed25519 + printf '%s\n' "$DEPLOY_KNOWN_HOSTS" > ~/.ssh/known_hosts + chmod 600 ~/.ssh/id_ed25519 ~/.ssh/known_hosts + cat > ~/.ssh/config </dev/null \ + && trap 'docker logout ghcr.io >/dev/null' EXIT \ + && /opt/docs/scripts/deploy.sh '${{ github.sha }}'" + - name: Check the instance answers + run: | + host=$(ssh dev-instance "grep '^DOCS_HOST=' /opt/docs/.env | cut -d= -f2") + curl -fsS --retry 10 --retry-delay 6 --retry-all-errors -o /dev/null "https://$host/" + curl -fsS --retry 10 --retry-delay 6 --retry-all-errors "https://$host/api/v1.0/config/" -o /dev/null + echo "Deployed to https://$host/" >> "$GITHUB_STEP_SUMMARY" + - name: Seed the meeting document + if: github.event_name == 'workflow_dispatch' && inputs.seed + run: | + ssh dev-instance "SEED_FORCE=${{ inputs.seed_force && '1' || '' }} /opt/docs/scripts/seed.sh" \ + | tee seed.log + grep -Eo 'https://[^ ]+/docs/[^ ]+' seed.log | tail -1 | sed 's/^/Seeded document: /' >> "$GITHUB_STEP_SUMMARY" diff --git a/deploy/dev-instance/Caddyfile b/deploy/dev-instance/Caddyfile new file mode 100644 index 0000000000..8f2a0007a9 --- /dev/null +++ b/deploy/dev-instance/Caddyfile @@ -0,0 +1,20 @@ +# TLS for the three public host names, with certificates from Let's Encrypt. +# The host names come from the .env file on the server. +{ + email {$ACME_EMAIL} +} + +{$DOCS_HOST} { + encode zstd gzip + reverse_proxy frontend:8083 +} + +{$KEYCLOAK_HOST} { + reverse_proxy keycloak:8080 +} + +# The media bucket. Access is signed by the backend (see /media-auth in the +# nginx template), the bucket itself is private. +{$S3_HOST} { + reverse_proxy minio:9000 +} diff --git a/deploy/dev-instance/compose.yaml b/deploy/dev-instance/compose.yaml new file mode 100644 index 0000000000..49ccc6c4fb --- /dev/null +++ b/deploy/dev-instance/compose.yaml @@ -0,0 +1,208 @@ +# Dev instance of Docs on one server, deployed by +# .github/workflows/dev-instance.yml. It follows the production example in +# documentation/examples/compose/compose.yaml, with the identity provider, the +# object storage and the TLS proxy on the same machine. +# +# Every value that is specific to the server (host names, passwords, keys) is +# in the `.env` file next to this one on the server, which +# scripts/init-env.sh generates. Nothing secret is in the repository. +# +# Only caddy publishes ports (80 and 443). Docker bypasses the host firewall +# for published ports, so do not publish ports on other services. + +name: docs + +services: + caddy: + image: caddy:2 + restart: unless-stopped + ports: + - "80:80" + - "443:443" + - "443:443/udp" + environment: + - DOCS_HOST=${DOCS_HOST} + - KEYCLOAK_HOST=${KEYCLOAK_HOST} + - S3_HOST=${S3_HOST} + - ACME_EMAIL=${ACME_EMAIL:-} + volumes: + - ./Caddyfile:/etc/caddy/Caddyfile:ro + - caddy-data:/data + - caddy-config:/config + + postgresql: + image: postgres:16 + restart: unless-stopped + healthcheck: + test: ["CMD-SHELL", "pg_isready -d $${POSTGRES_DB} -U $${POSTGRES_USER}"] + interval: 1s + timeout: 2s + retries: 300 + env_file: + - env.d/postgresql + environment: + - PGDATA=/var/lib/postgresql/data/pgdata + volumes: + - postgresql:/var/lib/postgresql/data + + redis: + image: redis:8 + restart: unless-stopped + + # Holds the updates that no yhub worker has persisted yet: a store, not a + # cache. Hence the append-only file. + yhub-valkey: + image: valkey/valkey:8-alpine + restart: unless-stopped + command: ["valkey-server", "--maxmemory-policy", "volatile-lru", + "--appendonly", "yes", "--appendfsync", "everysec"] + healthcheck: + test: ["CMD", "valkey-cli", "ping"] + interval: 1s + timeout: 2s + retries: 60 + volumes: + - yhub-valkey:/data + + minio: + image: pgsty/silo + restart: unless-stopped + command: server /data + environment: + - MINIO_ROOT_USER=${S3_ACCESS_KEY} + - MINIO_ROOT_PASSWORD=${S3_SECRET_KEY} + healthcheck: + test: ["CMD", "mc", "ready", "local"] + interval: 2s + timeout: 20s + retries: 150 + volumes: + - minio:/data + + # One-shot, run by deploy.sh + createbuckets: + image: pgsty/mc + profiles: ["setup"] + depends_on: + minio: + condition: service_healthy + environment: + - S3_ACCESS_KEY=${S3_ACCESS_KEY} + - S3_SECRET_KEY=${S3_SECRET_KEY} + - BUCKET_NAME=${BUCKET_NAME:-docs-media-storage} + entrypoint: > + sh -c " + mc alias set docs http://minio:9000 $${S3_ACCESS_KEY} $${S3_SECRET_KEY} && + mc mb --ignore-existing docs/$${BUCKET_NAME} && + mc version enable docs/$${BUCKET_NAME}" + + keycloak: + image: quay.io/keycloak/keycloak:26.7 + restart: unless-stopped + # Development mode with the embedded database: enough for a dev instance. + # The realm and its users are imported on the first start only. + command: ["start-dev", "--import-realm"] + environment: + - KC_HOSTNAME=https://${KEYCLOAK_HOST} + - KC_PROXY_HEADERS=xforwarded + - KC_HTTP_ENABLED=true + - KC_BOOTSTRAP_ADMIN_USERNAME=admin + - KC_BOOTSTRAP_ADMIN_PASSWORD=${KEYCLOAK_ADMIN_PASSWORD} + # Read by keycloak/realm.json + - DOCS_HOST=${DOCS_HOST} + - OIDC_CLIENT_SECRET=${OIDC_CLIENT_SECRET} + - SEED_PASSWORD_ADMIN=${SEED_PASSWORD_ADMIN} + - SEED_PASSWORD_ALICE=${SEED_PASSWORD_ALICE} + - SEED_PASSWORD_BOB=${SEED_PASSWORD_BOB} + - SEED_PASSWORD_CAROL=${SEED_PASSWORD_CAROL} + - SEED_PASSWORD_DAVE=${SEED_PASSWORD_DAVE} + volumes: + - ./keycloak/realm.json:/opt/keycloak/data/import/realm.json:ro + - keycloak:/opt/keycloak/data/h2 + + backend: + # Upstream image of the commit this branch is based on: Docs' backend is not + # changed here. Update it when merging a newer upstream main. + image: ${BACKEND_IMAGE:-lasuite/impress-backend:main@sha256:525fbb920e7ad8de69ccfde490030db608d2f11693cbcc6af81e681850700558} + user: "1000" + restart: unless-stopped + environment: + - DJANGO_CONFIGURATION=Production + env_file: + - env.d/common + - env.d/backend + - env.d/yprovider + - env.d/postgresql + volumes: + - ./keys/private.pem:/keys/private.pem:ro + healthcheck: + test: ["CMD", "python", "manage.py", "check"] + interval: 15s + timeout: 30s + retries: 20 + start_period: 10s + depends_on: + postgresql: + condition: service_healthy + restart: true + redis: + condition: service_started + + # The collaboration server. Run `yarn init-db` before it starts (deploy.sh + # does it on every deploy, it is idempotent). + yhub: + image: ${IMAGE_REGISTRY:-ghcr.io/typecellos/docs}/yhub:${IMAGE_TAG} + user: "1000" + restart: unless-stopped + env_file: + - env.d/common + - env.d/yhub + volumes: + - ./keys/yhub-private.pem:/keys/yhub-private.pem:ro + healthcheck: + test: ["CMD", "node", "-e", "fetch('http://localhost:3002/collaboration/ready/v1').then(r=>process.exit(r.ok?0:1),()=>process.exit(1))"] + interval: 15s + timeout: 5s + retries: 20 + start_period: 10s + depends_on: + postgresql: + condition: service_healthy + restart: true + yhub-valkey: + condition: service_healthy + + # The conversion service (markdown, html, pdf, docx). + y-provider: + image: ${IMAGE_REGISTRY:-ghcr.io/typecellos/docs}/y-provider:${IMAGE_TAG} + user: "1000" + restart: unless-stopped + env_file: + - env.d/common + - env.d/yprovider + + # Serves the editor and routes everything else, on the same origin. + frontend: + image: ${IMAGE_REGISTRY:-ghcr.io/typecellos/docs}/frontend:${IMAGE_TAG} + user: "101" + restart: unless-stopped + entrypoint: + - /docker-entrypoint.sh + command: ["nginx", "-g", "daemon off;"] + env_file: + - env.d/common + volumes: + - ./default.conf.template:/etc/nginx/templates/docs.conf.template:ro + depends_on: + backend: + condition: service_healthy + yhub: + condition: service_healthy + +volumes: + caddy-data: + caddy-config: + postgresql: + yhub-valkey: + minio: + keycloak: diff --git a/deploy/dev-instance/env.d/backend b/deploy/dev-instance/env.d/backend new file mode 100644 index 0000000000..a4d3562137 --- /dev/null +++ b/deploy/dev-instance/env.d/backend @@ -0,0 +1,49 @@ +# Django +DJANGO_ALLOWED_HOSTS=${DOCS_HOST} +DJANGO_SECRET_KEY=${DJANGO_SECRET_KEY} +DJANGO_SETTINGS_MODULE=impress.settings +DJANGO_CONFIGURATION=Production +DJANGO_CSRF_TRUSTED_ORIGINS=https://${DOCS_HOST} + +LOGGING_LEVEL_HANDLERS_CONSOLE=INFO +LOGGING_LEVEL_LOGGERS_ROOT=INFO +LOGGING_LEVEL_LOGGERS_APP=INFO + +PYTHONPATH=/app + +# Signs the tokens the backend presents to yhub and to y-provider +JWT_PRIVATE_KEY_FILE=/keys/private.pem + +# Collaboration server +YHUB_API_BASE_URL=http://yhub:3002 +COLLABORATION_API_URL=http://yhub:3002/collaboration +COLLABORATION_WS_URL=wss://${DOCS_HOST}/collaboration/ws/v1/docs + +# No mail server on the dev instance +DJANGO_EMAIL_BRAND_NAME="Docs dev" +DJANGO_EMAIL_URL_APP=https://${DOCS_HOST} + +# Media +AWS_S3_ENDPOINT_URL=https://${S3_HOST} +AWS_S3_ACCESS_KEY_ID=${S3_ACCESS_KEY} +AWS_S3_SECRET_ACCESS_KEY=${S3_SECRET_KEY} +AWS_STORAGE_BUCKET_NAME=${BUCKET_NAME} +MEDIA_BASE_URL=https://${DOCS_HOST} + +# OIDC (Keycloak on the same server, realm keycloak/realm.json) +OIDC_OP_JWKS_ENDPOINT=https://${KEYCLOAK_HOST}/realms/docs/protocol/openid-connect/certs +OIDC_OP_AUTHORIZATION_ENDPOINT=https://${KEYCLOAK_HOST}/realms/docs/protocol/openid-connect/auth +OIDC_OP_TOKEN_ENDPOINT=https://${KEYCLOAK_HOST}/realms/docs/protocol/openid-connect/token +OIDC_OP_USER_ENDPOINT=https://${KEYCLOAK_HOST}/realms/docs/protocol/openid-connect/userinfo +OIDC_OP_LOGOUT_ENDPOINT=https://${KEYCLOAK_HOST}/realms/docs/protocol/openid-connect/logout +OIDC_RP_CLIENT_ID=docs +OIDC_RP_CLIENT_SECRET=${OIDC_CLIENT_SECRET} +OIDC_RP_SIGN_ALGO=RS256 +OIDC_RP_SCOPES="openid email profile" +OIDC_USERINFO_FULLNAME_FIELDS=given_name,family_name +OIDC_USERINFO_SHORTNAME_FIELD=given_name + +LOGIN_REDIRECT_URL=https://${DOCS_HOST} +LOGIN_REDIRECT_URL_FAILURE=https://${DOCS_HOST} +LOGOUT_REDIRECT_URL=https://${DOCS_HOST} +OIDC_REDIRECT_ALLOWED_HOSTS=["https://${DOCS_HOST}"] diff --git a/deploy/dev-instance/env.d/common b/deploy/dev-instance/env.d/common new file mode 100644 index 0000000000..ed616f4c39 --- /dev/null +++ b/deploy/dev-instance/env.d/common @@ -0,0 +1,10 @@ +# Values come from the .env file on the server (scripts/init-env.sh). +DOCS_HOST=${DOCS_HOST} +KEYCLOAK_HOST=${KEYCLOAK_HOST} +S3_HOST=${S3_HOST} +BACKEND_HOST=backend +FRONTEND_HOST=frontend +YPROVIDER_HOST=y-provider +YHUB_HOST=yhub +BUCKET_NAME=${BUCKET_NAME} +REALM_NAME=docs diff --git a/deploy/dev-instance/env.d/postgresql b/deploy/dev-instance/env.d/postgresql new file mode 100644 index 0000000000..04db025bee --- /dev/null +++ b/deploy/dev-instance/env.d/postgresql @@ -0,0 +1,9 @@ +DB_HOST=postgresql +DB_NAME=docs +DB_USER=docs +DB_PASSWORD=${DB_PASSWORD} +DB_PORT=5432 + +POSTGRES_DB=docs +POSTGRES_USER=docs +POSTGRES_PASSWORD=${DB_PASSWORD} diff --git a/deploy/dev-instance/env.d/yhub b/deploy/dev-instance/env.d/yhub new file mode 100644 index 0000000000..138aa29d5c --- /dev/null +++ b/deploy/dev-instance/env.d/yhub @@ -0,0 +1,8 @@ +PORT=3002 +REDIS=redis://yhub-valkey:6379 +REDIS_PREFIX=yhub +POSTGRES=postgres://docs:${DB_PASSWORD}@postgresql:5432/yhub +COLLABORATION_BACKEND_BASE_URL=https://${DOCS_HOST} +COLLABORATION_SERVER_ORIGIN=https://${DOCS_HOST} +Y_PROVIDER_API_KEY=${Y_PROVIDER_API_KEY} +YHUB_JWT_PRIVATE_KEY_FILE=/keys/yhub-private.pem diff --git a/deploy/dev-instance/env.d/yprovider b/deploy/dev-instance/env.d/yprovider new file mode 100644 index 0000000000..70644cf900 --- /dev/null +++ b/deploy/dev-instance/env.d/yprovider @@ -0,0 +1,5 @@ +Y_PROVIDER_API_BASE_URL=http://y-provider:4444/api/ +Y_PROVIDER_API_KEY=${Y_PROVIDER_API_KEY} +COLLABORATION_SERVER_ORIGIN=https://${DOCS_HOST} +COLLABORATION_BACKEND_BASE_URL=https://${DOCS_HOST} +COLLABORATION_LOGGING=true diff --git a/deploy/dev-instance/keycloak/realm.json b/deploy/dev-instance/keycloak/realm.json new file mode 100644 index 0000000000..71a1f5c601 --- /dev/null +++ b/deploy/dev-instance/keycloak/realm.json @@ -0,0 +1,108 @@ +{ + "realm": "docs", + "enabled": true, + "sslRequired": "external", + "registrationAllowed": false, + "loginWithEmailAllowed": true, + "resetPasswordAllowed": false, + "bruteForceProtected": true, + "clients": [ + { + "clientId": "docs", + "name": "Docs", + "enabled": true, + "protocol": "openid-connect", + "publicClient": false, + "clientAuthenticatorType": "client-secret", + "secret": "${OIDC_CLIENT_SECRET}", + "standardFlowEnabled": true, + "directAccessGrantsEnabled": false, + "redirectUris": [ + "https://${DOCS_HOST}/*" + ], + "webOrigins": [ + "https://${DOCS_HOST}" + ], + "attributes": { + "post.logout.redirect.uris": "https://${DOCS_HOST}/*" + } + } + ], + "users": [ + { + "username": "admin", + "email": "admin@example.com", + "emailVerified": true, + "firstName": "Admin", + "lastName": "Dev", + "enabled": true, + "credentials": [ + { + "type": "password", + "value": "${SEED_PASSWORD_ADMIN}", + "temporary": false + } + ] + }, + { + "username": "alice", + "email": "alice@example.com", + "emailVerified": true, + "firstName": "Alice", + "lastName": "Martin", + "enabled": true, + "credentials": [ + { + "type": "password", + "value": "${SEED_PASSWORD_ALICE}", + "temporary": false + } + ] + }, + { + "username": "bob", + "email": "bob@example.com", + "emailVerified": true, + "firstName": "Bob", + "lastName": "Bernard", + "enabled": true, + "credentials": [ + { + "type": "password", + "value": "${SEED_PASSWORD_BOB}", + "temporary": false + } + ] + }, + { + "username": "carol", + "email": "carol@example.com", + "emailVerified": true, + "firstName": "Carol", + "lastName": "Dubois", + "enabled": true, + "credentials": [ + { + "type": "password", + "value": "${SEED_PASSWORD_CAROL}", + "temporary": false + } + ] + }, + { + "username": "dave", + "email": "dave@example.com", + "emailVerified": true, + "firstName": "Dave", + "lastName": "Moreau", + "enabled": true, + "credentials": [ + { + "type": "password", + "value": "${SEED_PASSWORD_DAVE}", + "temporary": false + } + ] + } + ] +} diff --git a/deploy/dev-instance/scripts/deploy.sh b/deploy/dev-instance/scripts/deploy.sh new file mode 100755 index 0000000000..9e350ad3fd --- /dev/null +++ b/deploy/dev-instance/scripts/deploy.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Deploys the images of one commit: ./scripts/deploy.sh +# Every step is idempotent, so a failed deploy can be run again. +set -euo pipefail +cd "$(dirname "$0")/.." + +tag="${1:?usage: deploy.sh }" +./scripts/init-env.sh +sed -i "s/^IMAGE_TAG=.*/IMAGE_TAG=${tag}/" .env + +docker compose pull --quiet +docker compose up -d --wait postgresql redis yhub-valkey minio +docker compose run --rm createbuckets +# Schemas: the collaboration server's, then the backend's +docker compose run --rm --no-deps yhub yarn init-db +docker compose run --rm --no-deps backend python manage.py migrate --noinput +docker compose up -d --remove-orphans --wait --wait-timeout 600 + +set -a; . ./.env; set +a +docker compose exec -T backend python manage.py createsuperuser \ + --email admin@example.com --password "${SEED_PASSWORD_ADMIN}" + +docker image prune -f >/dev/null +docker compose ps --format 'table {{.Service}}\t{{.Image}}\t{{.Status}}' diff --git a/deploy/dev-instance/scripts/init-env.sh b/deploy/dev-instance/scripts/init-env.sh new file mode 100755 index 0000000000..acc0e06745 --- /dev/null +++ b/deploy/dev-instance/scripts/init-env.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# Creates the server configuration on the first deploy: the .env file with the +# host names and random secrets, and the two RS256 keys. It never changes +# existing values, so it is safe to run on every deploy. +# +# The host names derive from DEV_DOMAIN (default: .sslip.io). To use +# a real domain later, edit DOCS_HOST, KEYCLOAK_HOST and S3_HOST in .env and +# update the OIDC client of the "docs" realm in Keycloak. +set -euo pipefail +cd "$(dirname "$0")/.." + +random() { openssl rand -base64 48 | tr -dc 'A-Za-z0-9' | head -c "${1:-40}"; } + +if [ ! -f .env ]; then + domain="${DEV_DOMAIN:-$(curl -fsS4 https://ifconfig.me | tr . -).sslip.io}" + umask 077 + cat > .env </dev/null + echo "Created keys/${key}.pem" + fi +done +# Read by the containers that run as uid 1000 (the deploy user) +chmod 600 keys/*.pem From 47cbb9849577e8e1c8ea2d12061432195a15b7fc Mon Sep 17 00:00:00 2001 From: yousefed Date: Fri, 9 Oct 2026 09:20:54 +0200 Subject: [PATCH 3/7] =?UTF-8?q?=E2=9C=A8(dev-instance)=20replay=20a=20reco?= =?UTF-8?q?rded=20document=20history?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The seed script signs in as the dev users and writes a recorded history version by version through the collaboration routes, so the history panel shows several versions by several authors. It runs from the dev instance workflow on request. Signed-off-by: yousefed --- deploy/dev-instance/scripts/seed.sh | 32 ++ deploy/dev-instance/seed/package-lock.json | 569 +++++++++++++++++++++ deploy/dev-instance/seed/package.json | 13 + deploy/dev-instance/seed/replay.mjs | 49 ++ deploy/dev-instance/seed/seed.mjs | 271 ++++++++++ 5 files changed, 934 insertions(+) create mode 100755 deploy/dev-instance/scripts/seed.sh create mode 100644 deploy/dev-instance/seed/package-lock.json create mode 100644 deploy/dev-instance/seed/package.json create mode 100644 deploy/dev-instance/seed/replay.mjs create mode 100644 deploy/dev-instance/seed/seed.mjs diff --git a/deploy/dev-instance/scripts/seed.sh b/deploy/dev-instance/scripts/seed.sh new file mode 100755 index 0000000000..c0844f0096 --- /dev/null +++ b/deploy/dev-instance/scripts/seed.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Replays a recorded document history into the dev instance, as the dev users +# (see seed/seed.mjs). Idempotent: an existing document is kept. +# +# ./scripts/seed.sh [path to versions JSON] (default: seed-data/meetingVersions.json) +# SEED_FORCE=1 ./scripts/seed.sh writes a new copy +set -euo pipefail +cd "$(dirname "$0")/.." + +data="$(realpath "${1:-seed-data/meetingVersions.json}")" +if [ ! -f "${data}" ]; then + echo "No seed data at ${data}: copy the versions JSON to the server first." >&2 + exit 1 +fi + +set -a; . ./.env; set +a +users="alice:${SEED_PASSWORD_ALICE},bob:${SEED_PASSWORD_BOB},carol:${SEED_PASSWORD_CAROL},dave:${SEED_PASSWORD_DAVE}" + +docker run --rm \ + -v "$(pwd)/seed:/seed:ro" \ + -v "${data}:/data/versions.json:ro" \ + -v docs-seed-modules:/work/node_modules \ + -e DOCS_URL="https://${DOCS_HOST}" \ + -e SEED_USERS="${users}" \ + -e SEED_DATA=/data/versions.json \ + -e SEED_FORCE="${SEED_FORCE:-}" \ + -e SEED_TITLE="${SEED_TITLE:-}" \ + -e SEED_VERSION_GAP_S="${SEED_VERSION_GAP_S:-}" \ + -e SEED_STEP_GAP_MS="${SEED_STEP_GAP_MS:-}" \ + node:24-alpine sh -c ' + cp /seed/package.json /seed/package-lock.json /seed/*.mjs /work/ && + cd /work && npm ci --no-audit --no-fund --loglevel=error && node seed.mjs' diff --git a/deploy/dev-instance/seed/package-lock.json b/deploy/dev-instance/seed/package-lock.json new file mode 100644 index 0000000000..2ce861e06d --- /dev/null +++ b/deploy/dev-instance/seed/package-lock.json @@ -0,0 +1,569 @@ +{ + "name": "docs-dev-instance-seed", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "docs-dev-instance-seed", + "dependencies": { + "@blocknote/core": "https://pkg.pr.new/TypeCellOS/BlockNote/@blocknote/core@bb192fe8fc", + "@y/prosemirror": "2.0.0-14", + "@y/protocols": "1.0.6-rc.1", + "@y/y": "14.0.0-rc.26", + "lib0": "1.0.0-rc.36" + } + }, + "node_modules/@blocknote/core": { + "version": "0.55.0", + "resolved": "https://pkg.pr.new/TypeCellOS/BlockNote/@blocknote/core@bb192fe8fc", + "integrity": "sha512-uxuA/QEckkOmI+Hye1kIIyyd9YIqcSm+mj95uccd7WeV6zAKyYkRcW69v/TObyfZ4c+k+sGcKBKzN7teiEhXkw==", + "license": "MPL-2.0", + "dependencies": { + "@emoji-mart/data": "^1.2.1", + "@handlewithcare/prosemirror-inputrules": "^0.1.4", + "@shikijs/types": "^4.4.3", + "@tiptap/core": "^3.31.3", + "@tiptap/extension-bold": "^3.31.3", + "@tiptap/extension-code": "^3.31.3", + "@tiptap/extension-italic": "^3.31.3", + "@tiptap/extension-strike": "^3.31.3", + "@tiptap/extension-text": "^3.31.3", + "@tiptap/extension-underline": "^3.31.3", + "@tiptap/extensions": "^3.31.3", + "@tiptap/pm": "^3.31.3", + "emoji-mart": "^5.6.0", + "fast-deep-equal": "^3.1.3", + "lib0": "1.0.0-rc.36", + "prosemirror-highlight": "^0.15.3", + "prosemirror-model": "^1.25.11", + "prosemirror-state": "^1.4.4", + "prosemirror-tables": "^1.8.5", + "prosemirror-transform": "^1.12.1", + "prosemirror-view": "^1.42.5" + }, + "peerDependencies": { + "@y/prosemirror": "^2.0.0-14", + "@y/protocols": "^1.0.6-rc.1", + "@y/y": "^14.0.0-rc.26", + "y-prosemirror": "^1.3.7", + "y-protocols": "^1.0.6", + "yjs": "^13.6.33" + }, + "peerDependenciesMeta": { + "@y/prosemirror": { + "optional": true + }, + "@y/protocols": { + "optional": true + }, + "@y/y": { + "optional": true + }, + "y-prosemirror": { + "optional": true + }, + "y-protocols": { + "optional": true + }, + "yjs": { + "optional": true + } + } + }, + "node_modules/@emoji-mart/data": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@emoji-mart/data/-/data-1.2.1.tgz", + "integrity": "sha512-no2pQMWiBy6gpBEiqGeU77/bFejDqUTRY7KX+0+iur13op3bqUsXdnwoZs6Xb1zbv0gAj5VvS1PWoUUckSr5Dw==", + "license": "MIT" + }, + "node_modules/@handlewithcare/prosemirror-inputrules": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/@handlewithcare/prosemirror-inputrules/-/prosemirror-inputrules-0.1.4.tgz", + "integrity": "sha512-GMqlBeG2MKM+tXEFd2N+wIv5z4VvJTg8JtfJUrdjvFq2W6v+AW8oTgiWyFw8L3iEQwvtQcVJxU873iB0LXUNNw==", + "license": "MIT", + "dependencies": { + "prosemirror-history": "^1.4.1", + "prosemirror-transform": "^1.0.0" + }, + "peerDependencies": { + "prosemirror-model": "^1.0.0", + "prosemirror-state": "^1.0.0", + "prosemirror-view": "^1.0.0" + } + }, + "node_modules/@shikijs/types": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/@shikijs/types/-/types-4.5.0.tgz", + "integrity": "sha512-drtPrtD8N0DENyD2cyt6Ajy+IWsiCLgMxODwLTpvH1UsN6a9jf90tNTPjqwHvA+0Lzu9e0txk8DjfqH1p9UJRw==", + "license": "MIT", + "dependencies": { + "@shikijs/vscode-textmate": "^10.0.2", + "@types/hast": "^3.0.5" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@shikijs/vscode-textmate": { + "version": "10.0.2", + "resolved": "https://registry.npmjs.org/@shikijs/vscode-textmate/-/vscode-textmate-10.0.2.tgz", + "integrity": "sha512-83yeghZ2xxin3Nj8z1NMd/NCuca+gsYXswywDy5bHvwlWL8tpTQmzGeUuHd9FC3E/SBEMvzJRwWEOz5gGes9Qg==", + "license": "MIT" + }, + "node_modules/@tiptap/core": { + "version": "3.31.4", + "resolved": "https://registry.npmjs.org/@tiptap/core/-/core-3.31.4.tgz", + "integrity": "sha512-cIVUQYBycighzsk6GbmCef8ZiEDVxoT5EuAe4xg2q9dNkDqCXVqI4YGr9aavCQEiAS4uHiRwqvcsPqJHipFDfg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/ueberdosis" + }, + "peerDependencies": { + "@tiptap/pm": "3.31.4" + } + }, + "node_modules/@tiptap/extension-bold": { + "version": "3.31.4", + "resolved": "https://registry.npmjs.org/@tiptap/extension-bold/-/extension-bold-3.31.4.tgz", + "integrity": "sha512-QX1xoXvO0gz6uVsoLegumqQ4MJXL7lcUDQZ4J7la7LBqiIh4efrCmp2EXbhpr37M/YQL7w8q3g7H/VovSh7xIg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/ueberdosis" + }, + "peerDependencies": { + "@tiptap/core": "3.31.4" + } + }, + "node_modules/@tiptap/extension-code": { + "version": "3.31.4", + "resolved": "https://registry.npmjs.org/@tiptap/extension-code/-/extension-code-3.31.4.tgz", + "integrity": "sha512-pzi8GNDs5J1DEoO1HNYiJ5WZpgOZzlc/2VIa77mMpEuD4hZxuK0BQzlF/O6Ps/O5P1lGJT4xS0Yowpzb9mD9RA==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/ueberdosis" + }, + "peerDependencies": { + "@tiptap/core": "3.31.4" + } + }, + "node_modules/@tiptap/extension-italic": { + "version": "3.31.4", + "resolved": "https://registry.npmjs.org/@tiptap/extension-italic/-/extension-italic-3.31.4.tgz", + "integrity": "sha512-pftTQBPYZocNDAj/u6OyDIp7YxmN0+N895cQPogJQ6qMHdLiRhzgawiEewJkniyN1w8k3Wx/Etp8pTKrxaOOBQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/ueberdosis" + }, + "peerDependencies": { + "@tiptap/core": "3.31.4" + } + }, + "node_modules/@tiptap/extension-strike": { + "version": "3.31.4", + "resolved": "https://registry.npmjs.org/@tiptap/extension-strike/-/extension-strike-3.31.4.tgz", + "integrity": "sha512-4MBr63TJIMqxCjE+gAPdlY1IdELgvbAP516K3Ntn/BkcpEbEoPMEYVz+XTxULyHQEuARDO6jsXPPxRXpA74jaw==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/ueberdosis" + }, + "peerDependencies": { + "@tiptap/core": "3.31.4" + } + }, + "node_modules/@tiptap/extension-text": { + "version": "3.31.4", + "resolved": "https://registry.npmjs.org/@tiptap/extension-text/-/extension-text-3.31.4.tgz", + "integrity": "sha512-WL2ZRpFksgQcXJN05/UQH09YCH9LlhdLrSEcviASvEH/aAfsKiJlDrouN8PET+YAWK/F5xFc5/Q1z0xj+I23jw==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/ueberdosis" + }, + "peerDependencies": { + "@tiptap/core": "3.31.4" + } + }, + "node_modules/@tiptap/extension-underline": { + "version": "3.31.4", + "resolved": "https://registry.npmjs.org/@tiptap/extension-underline/-/extension-underline-3.31.4.tgz", + "integrity": "sha512-0iFGPIqmqGkCrOFh1OPo7Dzv5s86dVN3bZaugsz0jK04otkb8bn04ZDKP2/6Wq+lcK0M9DfJ0QqfW5hC2nBZxQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/ueberdosis" + }, + "peerDependencies": { + "@tiptap/core": "3.31.4" + } + }, + "node_modules/@tiptap/extensions": { + "version": "3.31.4", + "resolved": "https://registry.npmjs.org/@tiptap/extensions/-/extensions-3.31.4.tgz", + "integrity": "sha512-nIupIne/sHpV2Kw1M4MOmr4+G454KI8BXM5+LWKOU2EpzfALZWVSy7koF9lVRna4Zp5Gv8hfg5Z8kvmrYQvhkw==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/ueberdosis" + }, + "peerDependencies": { + "@tiptap/core": "3.31.4", + "@tiptap/pm": "3.31.4" + } + }, + "node_modules/@tiptap/pm": { + "version": "3.31.4", + "resolved": "https://registry.npmjs.org/@tiptap/pm/-/pm-3.31.4.tgz", + "integrity": "sha512-x7dwth79cQp4QGae66/RJqMDJQv0+n7b7dD7cyFQBWqzPFLJxN/4paHY4qbWedZjNLhRyIOAD9a5q4AYM1zmPA==", + "license": "MIT", + "dependencies": { + "prosemirror-changeset": "^2.4.1", + "prosemirror-commands": "^1.7.1", + "prosemirror-dropcursor": "^1.8.2", + "prosemirror-gapcursor": "^1.4.1", + "prosemirror-history": "^1.5.0", + "prosemirror-inputrules": "^1.5.1", + "prosemirror-keymap": "^1.2.3", + "prosemirror-model": "^1.25.11", + "prosemirror-schema-list": "^1.5.1", + "prosemirror-state": "^1.4.4", + "prosemirror-tables": "^1.8.5", + "prosemirror-transform": "^1.12.0", + "prosemirror-view": "^1.42.3" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/ueberdosis" + } + }, + "node_modules/@types/hast": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@types/hast/-/hast-3.0.5.tgz", + "integrity": "sha512-rp/ezSWaD1m44dPKICGhiskI13nVr7qTloFwDa/IYkhhf5nzwP+zIQcIJh3WIFSBOy/H1PzB40jPjMDksN4F+g==", + "license": "MIT", + "dependencies": { + "@types/unist": "*" + } + }, + "node_modules/@types/unist": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.3.tgz", + "integrity": "sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q==", + "license": "MIT" + }, + "node_modules/@y/prosemirror": { + "version": "2.0.0-14", + "resolved": "https://registry.npmjs.org/@y/prosemirror/-/prosemirror-2.0.0-14.tgz", + "integrity": "sha512-h0j2KWoVig9TxJ/1FjGMAPgLgugUffLCQL7bJvRzahfsNzGICiknGrQevvKpfTQkKcclKXsvmH+/lkFLMzhOhQ==", + "license": "MIT", + "dependencies": { + "@y/y": "^14.0.0-rc.26", + "lib0": "^1.0.0-rc.32" + }, + "engines": { + "node": ">=16.0.0", + "npm": ">=8.0.0" + }, + "funding": { + "type": "GitHub Sponsors ❤", + "url": "https://github.com/sponsors/dmonad" + }, + "peerDependencies": { + "@y/protocols": "^1.0.6-rc.1", + "prosemirror-model": "^1.7.1", + "prosemirror-state": "^1.2.3", + "prosemirror-transform": "^1.8.0", + "prosemirror-view": "^1.9.10" + } + }, + "node_modules/@y/protocols": { + "version": "1.0.6-rc.1", + "resolved": "https://registry.npmjs.org/@y/protocols/-/protocols-1.0.6-rc.1.tgz", + "integrity": "sha512-e/qs7hXcLk/SeNitxMXv2ymozyWFTULwbJEi7cAf/K/iXw9nGwGXHrR5TNluQ/bMwOX1cwuUT0hjEojkfH0gsA==", + "license": "MIT", + "dependencies": { + "lib0": "^1.0.0-rc.1" + }, + "engines": { + "node": ">=16.0.0", + "npm": ">=8.0.0" + }, + "funding": { + "type": "GitHub Sponsors ❤", + "url": "https://github.com/sponsors/dmonad" + }, + "peerDependencies": { + "@y/y": "*" + } + }, + "node_modules/@y/y": { + "version": "14.0.0-rc.26", + "resolved": "https://registry.npmjs.org/@y/y/-/y-14.0.0-rc.26.tgz", + "integrity": "sha512-sJ0N5qkYFtFTIdHWV77Fx+JU7bPqNXUmoFC807FscFtsxMSXp6zjPwHQ1PdUCYv8PJlNcPjsuGGyVpOT+FF5SQ==", + "license": "MIT", + "dependencies": { + "lib0": "^1.0.0-rc.29" + }, + "engines": { + "node": ">=22.0.0", + "npm": ">=8.0.0" + }, + "funding": { + "type": "GitHub Sponsors ❤", + "url": "https://github.com/sponsors/dmonad" + } + }, + "node_modules/emoji-mart": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/emoji-mart/-/emoji-mart-5.6.0.tgz", + "integrity": "sha512-eJp3QRe79pjwa+duv+n7+5YsNhRcMl812EcFVwrnRvYKoNPoQb5qxU8DG6Bgwji0akHdp6D4Ln6tYLG58MFSow==", + "license": "MIT" + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "license": "MIT" + }, + "node_modules/lib0": { + "version": "1.0.0-rc.36", + "resolved": "https://registry.npmjs.org/lib0/-/lib0-1.0.0-rc.36.tgz", + "integrity": "sha512-7t/I+PCQqakal22rhXq5giOUtDP/RfecQtdeKvzCtnAvPkxSLAE+/Zvzi/kJDopg+hBAYKo46qmxmlkhX0tYlQ==", + "license": "MIT", + "bin": { + "0ecdsa-generate-keypair": "src/bin/0ecdsa-generate-keypair.js", + "0gentesthtml": "src/bin/gentesthtml.js", + "0serve": "src/bin/0serve.js" + }, + "engines": { + "node": ">=22" + }, + "funding": { + "type": "GitHub Sponsors ❤", + "url": "https://github.com/sponsors/dmonad" + } + }, + "node_modules/orderedmap": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/orderedmap/-/orderedmap-2.1.1.tgz", + "integrity": "sha512-TvAWxi0nDe1j/rtMcWcIj94+Ffe6n7zhow33h40SKxmsmozs6dz/e+EajymfoFcHd7sxNn8yHM8839uixMOV6g==", + "license": "MIT" + }, + "node_modules/prosemirror-changeset": { + "version": "2.4.4", + "resolved": "https://registry.npmjs.org/prosemirror-changeset/-/prosemirror-changeset-2.4.4.tgz", + "integrity": "sha512-Gyuz9jYdrAeA9I7AholyOes9KUmv/lEwsC5O9s5xF5VB3EgTPsXMGUxz4+H1B+wKVPdZbvRcg1Cp3x8qX+8ofA==", + "license": "MIT", + "dependencies": { + "prosemirror-transform": "^1.0.0" + } + }, + "node_modules/prosemirror-commands": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/prosemirror-commands/-/prosemirror-commands-1.7.2.tgz", + "integrity": "sha512-q6Q6szxqdu9Xd6EcdKsqXghu5nQdZTpB4Q9yd04WRc7/jt763e/rT60Owh0L1GYY+T46o5rD+9lEN36dZS43tw==", + "license": "MIT", + "dependencies": { + "prosemirror-model": "^1.0.0", + "prosemirror-state": "^1.0.0", + "prosemirror-transform": "^1.10.2" + } + }, + "node_modules/prosemirror-dropcursor": { + "version": "1.8.4", + "resolved": "https://registry.npmjs.org/prosemirror-dropcursor/-/prosemirror-dropcursor-1.8.4.tgz", + "integrity": "sha512-VmKxhcA6a+h5oPSYmfF1ZNeQqPbyYaIxqZsKqeYCyTlPtrjUgNaHL3P8vVaO19oqAV0zuaM/axkowZHrB1P3Tg==", + "license": "MIT", + "dependencies": { + "prosemirror-state": "^1.0.0", + "prosemirror-transform": "^1.1.0", + "prosemirror-view": "^1.1.0" + } + }, + "node_modules/prosemirror-gapcursor": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/prosemirror-gapcursor/-/prosemirror-gapcursor-1.4.1.tgz", + "integrity": "sha512-pMdYaEnjNMSwl11yjEGtgTmLkR08m/Vl+Jj443167p9eB3HVQKhYCc4gmHVDsLPODfZfjr/MmirsdyZziXbQKw==", + "license": "MIT", + "dependencies": { + "prosemirror-keymap": "^1.0.0", + "prosemirror-model": "^1.0.0", + "prosemirror-state": "^1.0.0", + "prosemirror-view": "^1.0.0" + } + }, + "node_modules/prosemirror-highlight": { + "version": "0.15.3", + "resolved": "https://registry.npmjs.org/prosemirror-highlight/-/prosemirror-highlight-0.15.3.tgz", + "integrity": "sha512-WVV2st0fX1w2TkAgmTmdbj77BlWYuLfW4BGXPo8JfIWsSly5xYcfID8QJQL+GT8kisMRuu4jgT6JAqj1DOwvvg==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ocavue" + }, + "peerDependencies": { + "@lezer/common": "^1.0.0", + "@lezer/highlight": "^1.0.0", + "@shikijs/types": "^1.29.2 || ^2.0.0 || ^3.0.0 || ^4.0.0", + "@types/hast": "^3.0.0", + "highlight.js": "^11.9.0", + "lowlight": "^3.1.0", + "prosemirror-model": "^1.19.3", + "prosemirror-state": "^1.4.3", + "prosemirror-transform": "^1.8.0", + "prosemirror-view": "^1.32.4", + "refractor": "^5.0.0", + "sugar-high": "^0.6.1 || ^0.7.0 || ^0.8.0 || ^0.9.0 || ^1.0.0" + }, + "peerDependenciesMeta": { + "@lezer/common": { + "optional": true + }, + "@lezer/highlight": { + "optional": true + }, + "@shikijs/types": { + "optional": true + }, + "@types/hast": { + "optional": true + }, + "highlight.js": { + "optional": true + }, + "lowlight": { + "optional": true + }, + "prosemirror-model": { + "optional": true + }, + "prosemirror-state": { + "optional": true + }, + "prosemirror-transform": { + "optional": true + }, + "prosemirror-view": { + "optional": true + }, + "refractor": { + "optional": true + }, + "sugar-high": { + "optional": true + } + } + }, + "node_modules/prosemirror-history": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/prosemirror-history/-/prosemirror-history-1.5.1.tgz", + "integrity": "sha512-onlwnqKvgtFg6srZOmDOAhlAy1FIHQpOExORybX0S+DvJepE5tPALrB9iGet/2sqqqrelSnB10OoiOJqAvu2DA==", + "license": "MIT", + "dependencies": { + "prosemirror-state": "^1.2.2", + "prosemirror-transform": "^1.0.0", + "prosemirror-view": "^1.31.0", + "rope-sequence": "^1.3.0" + } + }, + "node_modules/prosemirror-inputrules": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/prosemirror-inputrules/-/prosemirror-inputrules-1.5.1.tgz", + "integrity": "sha512-7wj4uMjKaXWAQ1CDgxNzNtR9AlsuwzHfdFH1ygEHA2KHF2DOEaXl1CJfNPAKCg9qNEh4rum975QLaCiQPyY6Fw==", + "license": "MIT", + "dependencies": { + "prosemirror-state": "^1.0.0", + "prosemirror-transform": "^1.0.0" + } + }, + "node_modules/prosemirror-keymap": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/prosemirror-keymap/-/prosemirror-keymap-1.2.3.tgz", + "integrity": "sha512-4HucRlpiLd1IPQQXNqeo81BGtkY8Ai5smHhKW9jjPKRc2wQIxksg7Hl1tTI2IfT2B/LgX6bfYvXxEpJl7aKYKw==", + "license": "MIT", + "dependencies": { + "prosemirror-state": "^1.0.0", + "w3c-keyname": "^2.2.0" + } + }, + "node_modules/prosemirror-model": { + "version": "1.25.12", + "resolved": "https://registry.npmjs.org/prosemirror-model/-/prosemirror-model-1.25.12.tgz", + "integrity": "sha512-Ue2gTmXMa7EhpLNhC7J+h4+ykD8ha12K6rrZFFKKJHBForfIStw5gJ6Zrf1mqqAa9NmpmEB4wp9bKmA4eUVjWg==", + "license": "MIT", + "dependencies": { + "orderedmap": "^2.0.0" + } + }, + "node_modules/prosemirror-schema-list": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/prosemirror-schema-list/-/prosemirror-schema-list-1.5.1.tgz", + "integrity": "sha512-927lFx/uwyQaGwJxLWCZRkjXG0p48KpMj6ueoYiu4JX05GGuGcgzAy62dfiV8eFZftgyBUvLx76RsMe20fJl+Q==", + "license": "MIT", + "dependencies": { + "prosemirror-model": "^1.0.0", + "prosemirror-state": "^1.0.0", + "prosemirror-transform": "^1.7.3" + } + }, + "node_modules/prosemirror-state": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/prosemirror-state/-/prosemirror-state-1.4.4.tgz", + "integrity": "sha512-6jiYHH2CIGbCfnxdHbXZ12gySFY/fz/ulZE333G6bPqIZ4F+TXo9ifiR86nAHpWnfoNjOb3o5ESi7J8Uz1jXHw==", + "license": "MIT", + "dependencies": { + "prosemirror-model": "^1.0.0", + "prosemirror-transform": "^1.0.0", + "prosemirror-view": "^1.27.0" + } + }, + "node_modules/prosemirror-tables": { + "version": "1.8.5", + "resolved": "https://registry.npmjs.org/prosemirror-tables/-/prosemirror-tables-1.8.5.tgz", + "integrity": "sha512-V/0cDCsHKHe/tfWkeCmthNUcEp1IVO3p6vwN8XtwE9PZQLAZJigbw3QoraAdfJPir4NKJtNvOB8oYGKRl+t0Dw==", + "license": "MIT", + "dependencies": { + "prosemirror-keymap": "^1.2.3", + "prosemirror-model": "^1.25.4", + "prosemirror-state": "^1.4.4", + "prosemirror-transform": "^1.10.5", + "prosemirror-view": "^1.41.4" + } + }, + "node_modules/prosemirror-transform": { + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/prosemirror-transform/-/prosemirror-transform-1.12.2.tgz", + "integrity": "sha512-PE/aY0HEY4zczvmqrilgkUK/WautF0chvMqkmM/iN5/aPHvwrCWaQZjS6CIZp+K84YrvPFqrL+YArrfJX8xX7g==", + "license": "MIT", + "dependencies": { + "prosemirror-model": "^1.21.0" + } + }, + "node_modules/prosemirror-view": { + "version": "1.42.6", + "resolved": "https://registry.npmjs.org/prosemirror-view/-/prosemirror-view-1.42.6.tgz", + "integrity": "sha512-3g25f6hKV2bBxo3puH3EcYWmmVNSH0iWmNsMVT7sugDSpA025jgpOLarcDWr39SdzPjYMaiYtRdNtXialPSjEQ==", + "license": "MIT", + "dependencies": { + "prosemirror-model": "^1.25.12", + "prosemirror-state": "^1.0.0", + "prosemirror-transform": "^1.1.0" + } + }, + "node_modules/rope-sequence": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/rope-sequence/-/rope-sequence-1.3.4.tgz", + "integrity": "sha512-UT5EDe2cu2E/6O4igUr5PSFs23nvvukicWHx6GnOPlHAiiYbzNuCRQCuiUdHJQcqKalLKlrYJnjY0ySGsXNQXQ==", + "license": "MIT" + }, + "node_modules/w3c-keyname": { + "version": "2.2.8", + "resolved": "https://registry.npmjs.org/w3c-keyname/-/w3c-keyname-2.2.8.tgz", + "integrity": "sha512-dpojBhNsCNN7T82Tm7k26A6G9ML3NkhDsnw9n/eoxSRlVBB4CEtIQ/KTCLI2Fwf3ataSXRhYFkQi3SlnFwPvPQ==", + "license": "MIT" + } + } +} diff --git a/deploy/dev-instance/seed/package.json b/deploy/dev-instance/seed/package.json new file mode 100644 index 0000000000..3196113d99 --- /dev/null +++ b/deploy/dev-instance/seed/package.json @@ -0,0 +1,13 @@ +{ + "name": "docs-dev-instance-seed", + "private": true, + "type": "module", + "description": "Replays a recorded document history into a Docs dev instance, as real users.", + "dependencies": { + "@blocknote/core": "https://pkg.pr.new/TypeCellOS/BlockNote/@blocknote/core@bb192fe8fc", + "@y/prosemirror": "2.0.0-14", + "@y/protocols": "1.0.6-rc.1", + "@y/y": "14.0.0-rc.26", + "lib0": "1.0.0-rc.36" + } +} diff --git a/deploy/dev-instance/seed/replay.mjs b/deploy/dev-instance/seed/replay.mjs new file mode 100644 index 0000000000..2e972b3ee2 --- /dev/null +++ b/deploy/dev-instance/seed/replay.mjs @@ -0,0 +1,49 @@ +// Turns a recorded document history into Yjs updates for the Docs editor. +// +// The input is a list of versions. Each version is a list of steps, and each +// step is the full BlockNote document after a burst of editing. Each step is +// applied to a headless editor and written to Y the way the collaboration +// binding writes an edit (`docDiffToDelta`), so the history reflects the +// BlockNote build Docs runs. +import { BlockNoteEditor } from '@blocknote/core'; +import { docDiffToDelta } from '@blocknote/core/y'; +import { docToDelta } from '@y/prosemirror'; +import * as Y from '@y/y'; + +/** The shared type the Docs editor reads (`provider.doc.get('document-store')`). */ +export const FRAGMENT = 'document-store'; + +/** + * Prepares the replay. `next()` applies one step and returns the Yjs update + * (V1, the format yhub speaks) that the step adds to the document. + */ +export function createReplay() { + const editor = BlockNoteEditor.create(); + const ydoc = new Y.Doc({ gc: false }); + const yType = ydoc.get(FRAGMENT); + + function capture(write) { + const vector = Y.encodeStateVector(ydoc); + ydoc.transact(write); + return Y.encodeStateAsUpdate(ydoc, vector); + } + + return { + /** The update that writes the empty editor document. */ + base() { + return capture(() => { + yType.applyDelta(docToDelta(editor.prosemirrorState.doc)); + }); + }, + /** The update that turns the previous step into `blocks`. */ + next(blocks) { + const before = editor.prosemirrorState.doc; + editor.replaceBlocks(editor.document, blocks); + const delta = docDiffToDelta(before, editor.prosemirrorState.doc); + return capture(() => { + yType.applyDelta(delta); + }); + }, + ydoc, + }; +} diff --git a/deploy/dev-instance/seed/seed.mjs b/deploy/dev-instance/seed/seed.mjs new file mode 100644 index 0000000000..ece3a62705 --- /dev/null +++ b/deploy/dev-instance/seed/seed.mjs @@ -0,0 +1,271 @@ +// Seeds the dev instance with a document whose history has several versions by +// several authors. +// +// Every write goes through the routes a browser uses, as a real user: each user +// signs in through the identity provider, the first user creates the document +// and gives the others access, then the recorded history is written with +// `PATCH /collaboration/ydoc` by the author of each version. yhub attributes and +// timestamps the edits itself, so the script waits between versions for the +// history panel to show them apart, and names each version with +// `POST /collaboration/version`. +// +// It is idempotent: when the document already exists with content, it only +// makes sure the accesses are there and prints the document url. Set +// SEED_FORCE=1 to write a new copy. +// +// Environment: +// DOCS_URL https:// +// SEED_USERS user:password pairs, comma separated. The first one owns +// the document. Authors take turns in this order. +// SEED_DATA path to the recorded versions (JSON: [{name, steps}]) +// SEED_TITLE document title +// SEED_VERSION_GAP_S pause between two versions, in seconds (default 330) +// SEED_STEP_GAP_MS pause between two edits of a version (default 1500) +// SEED_FORCE 1 to write a new copy even if the document exists +import fs from 'node:fs'; +import { setTimeout as sleep } from 'node:timers/promises'; + +import * as Y from '@y/y'; +import { decodeAny, encodeAny } from 'lib0/buffer'; + +import { FRAGMENT, createReplay } from './replay.mjs'; + +const ORG = 'docs'; +const DOCS_URL = required('DOCS_URL').replace(/\/$/, ''); +const TITLE = process.env.SEED_TITLE || '07/10 meeting agenda (replay)'; +const VERSION_GAP_MS = Number(process.env.SEED_VERSION_GAP_S || 330) * 1000; +const STEP_GAP_MS = Number(process.env.SEED_STEP_GAP_MS || 1500); +const FORCE = process.env.SEED_FORCE === '1'; + +function required(name) { + const value = process.env[name]; + if (!value) { + throw new Error(`${name} is not set`); + } + return value; +} + +function log(message) { + console.log(`${new Date().toISOString()} ${message}`); +} + +/** A cookie jar per host, enough for the sign-in redirects. */ +function createSession() { + const jars = new Map(); + + function jarFor(url) { + const { host } = new URL(url); + if (!jars.has(host)) { + jars.set(host, new Map()); + } + return jars.get(host); + } + + async function request(url, init = {}) { + const jar = jarFor(url); + const headers = new Headers(init.headers); + if (jar.size) { + headers.set( + 'cookie', + [...jar].map(([name, value]) => `${name}=${value}`).join('; '), + ); + } + const response = await fetch(url, { + ...init, + headers, + redirect: 'manual', + }); + for (const line of response.headers.getSetCookie()) { + const [pair] = line.split(';'); + const index = pair.indexOf('='); + const name = pair.slice(0, index).trim(); + const value = pair.slice(index + 1).trim(); + if (value === '' || /max-age=0/i.test(line)) { + jar.delete(name); + } else { + jar.set(name, value); + } + } + return response; + } + + /** Follows redirects, carrying the cookies of each host. */ + async function follow(url, init = {}) { + let response = await request(url, init); + let current = url; + for (let i = 0; i < 20 && [301, 302, 303, 307, 308].includes(response.status); i++) { + current = new URL(response.headers.get('location'), current).toString(); + response = await request(current); + } + return { response, url: current }; + } + + return { + request, + follow, + cookie(url, name) { + return jarFor(url).get(name); + }, + }; +} + +/** Signs in through the OIDC flow of the backend and the Keycloak login form. */ +async function signIn(username, password) { + const session = createSession(); + const { response, url } = await session.follow( + `${DOCS_URL}/api/v1.0/authenticate/`, + ); + const html = await response.text(); + const action = html.match(/]*id="kc-form-login"[^>]*action="([^"]+)"/)?.[1]; + if (!action) { + throw new Error(`No login form for ${username} at ${url} (${response.status})`); + } + const done = await session.follow(action.replaceAll('&', '&'), { + method: 'POST', + headers: { 'content-type': 'application/x-www-form-urlencoded' }, + body: new URLSearchParams({ username, password, credentialId: '' }), + }); + await done.response.arrayBuffer(); + + async function api(path, init = {}) { + const headers = new Headers(init.headers); + const csrf = session.cookie(DOCS_URL, 'csrftoken'); + if (csrf) { + headers.set('x-csrftoken', csrf); + } + headers.set('referer', `${DOCS_URL}/`); + return session.request(`${DOCS_URL}${path}`, { ...init, headers }); + } + + const me = await api('/api/v1.0/users/me/'); + if (!me.ok) { + throw new Error(`Sign-in failed for ${username}: ${me.status}`); + } + const user = await me.json(); + log(`signed in as ${username} (${user.id})`); + return { username, user, api }; +} + +async function json(response, what) { + if (!response.ok) { + throw new Error(`${what} failed: ${response.status} ${await response.text()}`); + } + return response.json(); +} + +async function findDocument(owner) { + const response = await owner.api( + `/api/v1.0/documents/?is_creator_me=true&title=${encodeURIComponent(TITLE)}`, + ); + const { results } = await json(response, 'Listing documents'); + return results.find((doc) => doc.title === TITLE); +} + +async function readFragmentLength(user, docId) { + const response = await user.api( + `/collaboration/ydoc/v1/${ORG}/${docId}?gc=true`, + ); + if (!response.ok) { + throw new Error(`Reading the document failed: ${response.status}`); + } + const { doc } = decodeAny(new Uint8Array(await response.arrayBuffer())); + const ydoc = new Y.Doc(); + Y.applyUpdate(ydoc, doc); + return ydoc.get(FRAGMENT).length; +} + +async function ensureAccesses(owner, users, docId) { + const response = await owner.api(`/api/v1.0/documents/${docId}/accesses/`); + const existing = new Set( + (await json(response, 'Listing accesses')).map((access) => access.user?.id), + ); + for (const user of users) { + if (existing.has(user.user.id)) { + continue; + } + await json( + await owner.api(`/api/v1.0/documents/${docId}/accesses/`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ user_id: user.user.id, role: 'editor' }), + }), + `Giving access to ${user.username}`, + ); + log(`gave ${user.username} editor access`); + } +} + +async function patch(user, docId, update) { + const response = await user.api(`/collaboration/ydoc/v1/${ORG}/${docId}`, { + method: 'PATCH', + headers: { 'content-type': 'application/octet-stream' }, + body: encodeAny({ update }), + }); + if (!response.ok) { + throw new Error(`PATCH as ${user.username} failed: ${response.status} ${await response.text()}`); + } +} + +async function nameVersion(user, docId, name) { + const response = await user.api(`/collaboration/version/v1/${ORG}/${docId}`, { + method: 'POST', + headers: { 'content-type': 'application/octet-stream' }, + body: encodeAny({ type: 'version:v1', name }), + }); + if (!response.ok) { + throw new Error(`Naming "${name}" failed: ${response.status} ${await response.text()}`); + } +} + +async function main() { + const versions = JSON.parse(fs.readFileSync(required('SEED_DATA'), 'utf8')); + const users = []; + for (const pair of required('SEED_USERS').split(',')) { + const index = pair.indexOf(':'); + users.push(await signIn(pair.slice(0, index), pair.slice(index + 1))); + } + const [owner, ...others] = users; + + let doc = FORCE ? undefined : await findDocument(owner); + if (doc && (await readFragmentLength(owner, doc.id)) > 0) { + await ensureAccesses(owner, others, doc.id); + log(`already seeded: ${DOCS_URL}/docs/${doc.id}/`); + return; + } + if (!doc) { + doc = await json( + await owner.api('/api/v1.0/documents/', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ title: TITLE }), + }), + 'Creating the document', + ); + log(`created ${DOCS_URL}/docs/${doc.id}/`); + } + await ensureAccesses(owner, others, doc.id); + // History is only shown from the date of each access on: start after them. + await sleep(2000); + + const replay = createReplay(); + await patch(owner, doc.id, replay.base()); + for (const [i, version] of versions.entries()) { + const author = users[i % users.length]; + if (i > 0) { + log(`waiting ${VERSION_GAP_MS / 1000}s before the next version`); + await sleep(VERSION_GAP_MS); + } + for (const [j, blocks] of version.steps.entries()) { + if (j > 0) { + await sleep(STEP_GAP_MS); + } + await patch(author, doc.id, replay.next(blocks)); + } + await sleep(STEP_GAP_MS); + await nameVersion(author, doc.id, version.name); + log(`version ${i + 1}/${versions.length} "${version.name}" by ${author.username}: ${version.steps.length} edits`); + } + log(`done: ${DOCS_URL}/docs/${doc.id}/`); +} + +await main(); From 7c9c91168398bd347ffb3c8cd8c02a2ac7d8da9b Mon Sep 17 00:00:00 2001 From: yousefed Date: Fri, 9 Oct 2026 09:22:27 +0200 Subject: [PATCH 4/7] =?UTF-8?q?=F0=9F=90=9B(dev-instance)=20fix=20the=20ca?= =?UTF-8?q?ddy=20and=20keycloak=20configuration?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Caddy refused an empty email in its global options, and Keycloak could not write its database to a volume that root owned. Caddy now uses no global options, and the Keycloak volume holds its whole data folder. Signed-off-by: yousefed --- deploy/dev-instance/Caddyfile | 3 --- deploy/dev-instance/compose.yaml | 3 +-- deploy/dev-instance/scripts/init-env.sh | 1 - 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/deploy/dev-instance/Caddyfile b/deploy/dev-instance/Caddyfile index 8f2a0007a9..7885e9ab69 100644 --- a/deploy/dev-instance/Caddyfile +++ b/deploy/dev-instance/Caddyfile @@ -1,8 +1,5 @@ # TLS for the three public host names, with certificates from Let's Encrypt. # The host names come from the .env file on the server. -{ - email {$ACME_EMAIL} -} {$DOCS_HOST} { encode zstd gzip diff --git a/deploy/dev-instance/compose.yaml b/deploy/dev-instance/compose.yaml index 49ccc6c4fb..3c4ab850aa 100644 --- a/deploy/dev-instance/compose.yaml +++ b/deploy/dev-instance/compose.yaml @@ -24,7 +24,6 @@ services: - DOCS_HOST=${DOCS_HOST} - KEYCLOAK_HOST=${KEYCLOAK_HOST} - S3_HOST=${S3_HOST} - - ACME_EMAIL=${ACME_EMAIL:-} volumes: - ./Caddyfile:/etc/caddy/Caddyfile:ro - caddy-data:/data @@ -117,8 +116,8 @@ services: - SEED_PASSWORD_CAROL=${SEED_PASSWORD_CAROL} - SEED_PASSWORD_DAVE=${SEED_PASSWORD_DAVE} volumes: + - keycloak:/opt/keycloak/data - ./keycloak/realm.json:/opt/keycloak/data/import/realm.json:ro - - keycloak:/opt/keycloak/data/h2 backend: # Upstream image of the commit this branch is based on: Docs' backend is not diff --git a/deploy/dev-instance/scripts/init-env.sh b/deploy/dev-instance/scripts/init-env.sh index acc0e06745..d4eb604d8d 100755 --- a/deploy/dev-instance/scripts/init-env.sh +++ b/deploy/dev-instance/scripts/init-env.sh @@ -19,7 +19,6 @@ if [ ! -f .env ]; then DOCS_HOST=${domain} KEYCLOAK_HOST=id.${domain} S3_HOST=s3.${domain} -ACME_EMAIL= BUCKET_NAME=docs-media-storage # Set by deploy.sh From b6b3045a3446e07ef605b27a50bfd9d3246096a3 Mon Sep 17 00:00:00 2001 From: yousefed Date: Fri, 9 Oct 2026 09:36:43 +0200 Subject: [PATCH 5/7] =?UTF-8?q?=E2=9C=A8(dev-instance)=20deploy=20a=20chos?= =?UTF-8?q?en=20ref=20from=20the=20Actions=20tab?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The dev instance workflow takes a ref input when it is run by hand, so a branch that is not merged can go on the server for a short time. The images are tagged with the commit only, so such a deploy does not change the images of the dev branch. Signed-off-by: yousefed --- .github/workflows/dev-instance.yml | 38 +++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/.github/workflows/dev-instance.yml b/.github/workflows/dev-instance.yml index 88ace0087c..0ed20b5e25 100644 --- a/.github/workflows/dev-instance.yml +++ b/.github/workflows/dev-instance.yml @@ -1,14 +1,23 @@ name: Dev instance -run-name: Dev instance (${{ github.event_name == 'workflow_dispatch' && inputs.seed && 'deploy and seed' || 'deploy' }}) +run-name: Dev instance (${{ github.event_name == 'workflow_dispatch' && format('{0}{1}', inputs.ref, inputs.seed && ', seed' || '') || github.ref_name }}) # Builds the images that differ from upstream and deploys them to the dev # server (deploy/dev-instance). A merge to the `dev` branch ships it. +# +# From the Actions tab, "Run workflow" can also deploy another branch, tag or +# commit (`ref`), for example a pull request that is not merged yet. There is +# only one server: the last deploy wins, and the next merge to `dev` replaces +# it. The ref must contain deploy/dev-instance. on: push: branches: - dev workflow_dispatch: inputs: + ref: + description: "Branch, tag or commit to deploy. The last deploy wins: the next merge to dev replaces it." + type: string + default: dev seed: description: "Replay the meeting document into the instance after the deploy" type: boolean @@ -27,7 +36,22 @@ concurrency: cancel-in-progress: false jobs: + resolve: + runs-on: ubuntu-latest + outputs: + sha: ${{ steps.sha.outputs.sha }} + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ inputs.ref || github.sha }} + - name: Commit to deploy + id: sha + run: | + echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + echo "Commit: $(git log -1 --format='%h %s')" >> "$GITHUB_STEP_SUMMARY" + build: + needs: resolve runs-on: ubuntu-latest permissions: contents: read @@ -47,6 +71,8 @@ jobs: target: y-provider steps: - uses: actions/checkout@v6 + with: + ref: ${{ needs.resolve.outputs.sha }} - uses: docker/setup-buildx-action@v3 - uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4 with: @@ -67,14 +93,12 @@ jobs: PUBLISH_AS_MIT=false push: true provenance: false - tags: | - ${{ steps.image.outputs.name }}:${{ github.sha }} - ${{ steps.image.outputs.name }}:dev + tags: ${{ steps.image.outputs.name }}:${{ needs.resolve.outputs.sha }} cache-from: type=gha,scope=dev-${{ matrix.name }} cache-to: type=gha,scope=dev-${{ matrix.name }},mode=max deploy: - needs: build + needs: [resolve, build] runs-on: ubuntu-latest timeout-minutes: 150 permissions: @@ -82,6 +106,8 @@ jobs: packages: read steps: - uses: actions/checkout@v6 + with: + ref: ${{ needs.resolve.outputs.sha }} - name: Set up SSH env: DEPLOY_SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }} @@ -112,7 +138,7 @@ jobs: printf '%s' "$GHCR_TOKEN" | ssh dev-instance \ "docker login ghcr.io -u '${{ github.actor }}' --password-stdin >/dev/null \ && trap 'docker logout ghcr.io >/dev/null' EXIT \ - && /opt/docs/scripts/deploy.sh '${{ github.sha }}'" + && /opt/docs/scripts/deploy.sh '${{ needs.resolve.outputs.sha }}'" - name: Check the instance answers run: | host=$(ssh dev-instance "grep '^DOCS_HOST=' /opt/docs/.env | cut -d= -f2") From 8acb25ff92cda10a9a4e5218f24fb969adb8bdf3 Mon Sep 17 00:00:00 2001 From: yousefed Date: Fri, 9 Oct 2026 09:59:37 +0200 Subject: [PATCH 6/7] =?UTF-8?q?=E2=9A=A1=EF=B8=8F(dev-instance)=20build=20?= =?UTF-8?q?the=20sample=20document=20history=20offline?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The seed no longer writes the history in real time. It builds the history offline, with each edit attributed to its author at its recorded time, and stores it in yhub the way the full migration does. Every deploy runs it, and it does nothing when the document exists. It takes less than a minute instead of more than an hour. Signed-off-by: yousefed --- .github/workflows/dev-instance.yml | 16 +- deploy/dev-instance/scripts/seed.sh | 59 +++-- deploy/dev-instance/seed/import.mjs | 41 ++++ deploy/dev-instance/seed/package-lock.json | 10 +- deploy/dev-instance/seed/package.json | 4 +- deploy/dev-instance/seed/seed.mjs | 272 +++++++++++---------- 6 files changed, 231 insertions(+), 171 deletions(-) create mode 100644 deploy/dev-instance/seed/import.mjs diff --git a/.github/workflows/dev-instance.yml b/.github/workflows/dev-instance.yml index 0ed20b5e25..cf6bd5ac4c 100644 --- a/.github/workflows/dev-instance.yml +++ b/.github/workflows/dev-instance.yml @@ -1,5 +1,5 @@ name: Dev instance -run-name: Dev instance (${{ github.event_name == 'workflow_dispatch' && format('{0}{1}', inputs.ref, inputs.seed && ', seed' || '') || github.ref_name }}) +run-name: Dev instance (${{ inputs.ref || github.ref_name }}) # Builds the images that differ from upstream and deploys them to the dev # server (deploy/dev-instance). A merge to the `dev` branch ships it. @@ -18,14 +18,6 @@ on: description: "Branch, tag or commit to deploy. The last deploy wins: the next merge to dev replaces it." type: string default: dev - seed: - description: "Replay the meeting document into the instance after the deploy" - type: boolean - default: false - seed_force: - description: "Write a new copy of the document even if it exists" - type: boolean - default: false permissions: contents: read @@ -145,9 +137,3 @@ jobs: curl -fsS --retry 10 --retry-delay 6 --retry-all-errors -o /dev/null "https://$host/" curl -fsS --retry 10 --retry-delay 6 --retry-all-errors "https://$host/api/v1.0/config/" -o /dev/null echo "Deployed to https://$host/" >> "$GITHUB_STEP_SUMMARY" - - name: Seed the meeting document - if: github.event_name == 'workflow_dispatch' && inputs.seed - run: | - ssh dev-instance "SEED_FORCE=${{ inputs.seed_force && '1' || '' }} /opt/docs/scripts/seed.sh" \ - | tee seed.log - grep -Eo 'https://[^ ]+/docs/[^ ]+' seed.log | tail -1 | sed 's/^/Seeded document: /' >> "$GITHUB_STEP_SUMMARY" diff --git a/deploy/dev-instance/scripts/seed.sh b/deploy/dev-instance/scripts/seed.sh index c0844f0096..2524a33bd7 100755 --- a/deploy/dev-instance/scripts/seed.sh +++ b/deploy/dev-instance/scripts/seed.sh @@ -1,32 +1,57 @@ #!/usr/bin/env bash -# Replays a recorded document history into the dev instance, as the dev users -# (see seed/seed.mjs). Idempotent: an existing document is kept. +# Puts the sample document on the instance: "07/10 meeting agenda (sample)", +# owned by alice, with 15 named versions by alice, bob, carol and dave. +# deploy.sh runs it on every deploy. It does nothing when the document is +# already there, and it takes about a minute at most: the history is built +# offline with its past timestamps, then stored in yhub. # # ./scripts/seed.sh [path to versions JSON] (default: seed-data/meetingVersions.json) -# SEED_FORCE=1 ./scripts/seed.sh writes a new copy set -euo pipefail cd "$(dirname "$0")/.." -data="$(realpath "${1:-seed-data/meetingVersions.json}")" +data="${1:-seed-data/meetingVersions.json}" if [ ! -f "${data}" ]; then - echo "No seed data at ${data}: copy the versions JSON to the server first." >&2 - exit 1 + echo "seed: no ${data} on the server, so no sample document" >&2 + exit 0 fi +data="$(realpath "${data}")" +out="$(pwd)/seed-data/out" +rm -rf "${out}" && mkdir -p "${out}" && chmod 777 "${out}" -set -a; . ./.env; set +a -users="alice:${SEED_PASSWORD_ALICE},bob:${SEED_PASSWORD_BOB},carol:${SEED_PASSWORD_CAROL},dave:${SEED_PASSWORD_DAVE}" +host="$(grep '^DOCS_HOST=' .env | cut -d= -f2)" +# 1. Sign in as the demo users, create the document, build its history docker run --rm \ -v "$(pwd)/seed:/seed:ro" \ -v "${data}:/data/versions.json:ro" \ - -v docs-seed-modules:/work/node_modules \ - -e DOCS_URL="https://${DOCS_HOST}" \ - -e SEED_USERS="${users}" \ + -v "${out}:/out" \ + -v docs-seed-npm:/root/.npm \ + -e DOCS_URL="https://${host}" \ + -e SEED_USERS="alice:alice,bob:bob,carol:carol,dave:dave" \ -e SEED_DATA=/data/versions.json \ - -e SEED_FORCE="${SEED_FORCE:-}" \ - -e SEED_TITLE="${SEED_TITLE:-}" \ - -e SEED_VERSION_GAP_S="${SEED_VERSION_GAP_S:-}" \ - -e SEED_STEP_GAP_MS="${SEED_STEP_GAP_MS:-}" \ + -e OUT=/out/history.bin \ node:24-alpine sh -c ' - cp /seed/package.json /seed/package-lock.json /seed/*.mjs /work/ && - cd /work && npm ci --no-audit --no-fund --loglevel=error && node seed.mjs' + mkdir -p /work && cp /seed/package.json /seed/package-lock.json /seed/*.mjs /work/ && + cd /work && npm ci --prefer-offline --no-audit --no-fund --loglevel=error && node seed.mjs' + +if [ ! -f "${out}/history.bin" ]; then + exit 0 # already there +fi +docid="$(sed -E 's/.*"docid":"([^"]+)".*/\1/' "${out}/history.bin.json")" +from_ms="$(sed -E 's/.*"accessesFrom":([0-9]+).*/\1/' "${out}/history.bin.json")" + +# 2. Users only see the history after the date of their access: date the +# accesses before the recorded meeting. +docker compose exec -T backend python manage.py shell -c " +import datetime +from core.models import DocumentAccess +DocumentAccess.objects.filter(document_id='${docid}').update( + created_at=datetime.datetime.fromtimestamp(${from_ms} / 1000, datetime.timezone.utc)) +" + +# 3. Store the history in yhub +docker compose run --rm --no-deps \ + -v "$(pwd)/seed:/seed:ro" -v "${out}:/out:ro" \ + yhub node /seed/import.mjs /out/history.bin + +echo "seed: sample document at $(sed -E 's/.*"url":"([^"]+)".*/\1/' "${out}/history.bin.json")" diff --git a/deploy/dev-instance/seed/import.mjs b/deploy/dev-instance/seed/import.mjs new file mode 100644 index 0000000000..1ab3f528ef --- /dev/null +++ b/deploy/dev-instance/seed/import.mjs @@ -0,0 +1,41 @@ +// Stores the history built by seed.mjs in yhub: step 3 of scripts/seed.sh. +// +// It runs in the yhub image, with the @y/hub of the deployed server, and writes +// the document the way Docs' fullMigrate does: one persisted row at clock 0, +// and the named versions. It refuses a document that yhub already holds. +// +// Usage: node import.mjs +import fs from 'node:fs'; + +import { decodeAny, encodeAny } from '/app/node_modules/lib0/src/buffer.js'; +import { createPersistence } from '/app/node_modules/@y/hub/src/persistence.js'; + +const history = decodeAny(new Uint8Array(fs.readFileSync(process.argv[2]))); +const docRef = { org: history.org, docid: history.docid, branch: 'main' }; +const persistence = await createPersistence(process.env.POSTGRES, []); + +const current = await persistence.retrieveDoc(docRef, {}); +if (current.lastClock !== '0') { + console.log(`import: yhub already holds ${docRef.docid}, nothing to do`); +} else { + await persistence.store(docRef, { + lastClock: '0', + gcDoc: history.gcDoc, + nongcDoc: history.nongcDoc, + contentmap: history.contentmap, + contentids: history.contentids, + }); + for (const version of history.versions) { + await persistence.storeVersion(docRef, { + t: version.t, + name: version.name, + // what the version API stores for a version with no custom data + custom: encodeAny(null), + published: false, + at: version.t, + by: version.by, + }); + } + console.log(`import: stored ${history.versions.length} versions of ${docRef.docid}`); +} +process.exit(0); diff --git a/deploy/dev-instance/seed/package-lock.json b/deploy/dev-instance/seed/package-lock.json index 2ce861e06d..79c3efb721 100644 --- a/deploy/dev-instance/seed/package-lock.json +++ b/deploy/dev-instance/seed/package-lock.json @@ -9,7 +9,7 @@ "@blocknote/core": "https://pkg.pr.new/TypeCellOS/BlockNote/@blocknote/core@bb192fe8fc", "@y/prosemirror": "2.0.0-14", "@y/protocols": "1.0.6-rc.1", - "@y/y": "14.0.0-rc.26", + "@y/y": "14.0.0-rc.28", "lib0": "1.0.0-rc.36" } }, @@ -301,12 +301,12 @@ } }, "node_modules/@y/y": { - "version": "14.0.0-rc.26", - "resolved": "https://registry.npmjs.org/@y/y/-/y-14.0.0-rc.26.tgz", - "integrity": "sha512-sJ0N5qkYFtFTIdHWV77Fx+JU7bPqNXUmoFC807FscFtsxMSXp6zjPwHQ1PdUCYv8PJlNcPjsuGGyVpOT+FF5SQ==", + "version": "14.0.0-rc.28", + "resolved": "https://registry.npmjs.org/@y/y/-/y-14.0.0-rc.28.tgz", + "integrity": "sha512-Hzw2sSch5RpT4aKz0643LQnoN7qSm63VyWb5D3zzCDEgSe5G0CKu5W7zcAXKDlVwkXg5puKTNX5k/wdVg4LtMA==", "license": "MIT", "dependencies": { - "lib0": "^1.0.0-rc.29" + "lib0": "^1.0.0-rc.35" }, "engines": { "node": ">=22.0.0", diff --git a/deploy/dev-instance/seed/package.json b/deploy/dev-instance/seed/package.json index 3196113d99..f51309a35e 100644 --- a/deploy/dev-instance/seed/package.json +++ b/deploy/dev-instance/seed/package.json @@ -2,12 +2,12 @@ "name": "docs-dev-instance-seed", "private": true, "type": "module", - "description": "Replays a recorded document history into a Docs dev instance, as real users.", + "description": "Builds the sample document of the dev instance (scripts/seed.sh).", "dependencies": { "@blocknote/core": "https://pkg.pr.new/TypeCellOS/BlockNote/@blocknote/core@bb192fe8fc", "@y/prosemirror": "2.0.0-14", "@y/protocols": "1.0.6-rc.1", - "@y/y": "14.0.0-rc.26", + "@y/y": "14.0.0-rc.28", "lib0": "1.0.0-rc.36" } } diff --git a/deploy/dev-instance/seed/seed.mjs b/deploy/dev-instance/seed/seed.mjs index ece3a62705..48bc0ace58 100644 --- a/deploy/dev-instance/seed/seed.mjs +++ b/deploy/dev-instance/seed/seed.mjs @@ -1,29 +1,21 @@ -// Seeds the dev instance with a document whose history has several versions by -// several authors. +// Prepares the sample document of the dev instance: step 1 of scripts/seed.sh. // -// Every write goes through the routes a browser uses, as a real user: each user -// signs in through the identity provider, the first user creates the document -// and gives the others access, then the recorded history is written with -// `PATCH /collaboration/ydoc` by the author of each version. yhub attributes and -// timestamps the edits itself, so the script waits between versions for the -// history panel to show them apart, and names each version with -// `POST /collaboration/version`. +// It signs in as the demo users (which creates them in Docs), lets the first +// one create the document and give the others access, and builds the recorded +// history offline: every edit attributed to its author at a past timestamp, +// the way yhub records live edits (`insert`/`insertAt` in the contentmap). The +// result is written to OUT for import.mjs, which stores it in yhub. // -// It is idempotent: when the document already exists with content, it only -// makes sure the accesses are there and prints the document url. Set -// SEED_FORCE=1 to write a new copy. +// Nothing waits in real time: the timestamps are in the data, not the clock. // // Environment: -// DOCS_URL https:// -// SEED_USERS user:password pairs, comma separated. The first one owns -// the document. Authors take turns in this order. -// SEED_DATA path to the recorded versions (JSON: [{name, steps}]) -// SEED_TITLE document title -// SEED_VERSION_GAP_S pause between two versions, in seconds (default 330) -// SEED_STEP_GAP_MS pause between two edits of a version (default 1500) -// SEED_FORCE 1 to write a new copy even if the document exists +// DOCS_URL https:// +// SEED_USERS demo users, user:password pairs, comma separated. The first one +// owns the document; the authors of the versions take turns. +// SEED_DATA recorded versions (JSON: [{name, steps}]) +// SEED_DATE day of the recorded meeting, YYYY-MM-DD (default 2026-10-07) +// OUT where to write the history to import import fs from 'node:fs'; -import { setTimeout as sleep } from 'node:timers/promises'; import * as Y from '@y/y'; import { decodeAny, encodeAny } from 'lib0/buffer'; @@ -32,10 +24,8 @@ import { FRAGMENT, createReplay } from './replay.mjs'; const ORG = 'docs'; const DOCS_URL = required('DOCS_URL').replace(/\/$/, ''); -const TITLE = process.env.SEED_TITLE || '07/10 meeting agenda (replay)'; -const VERSION_GAP_MS = Number(process.env.SEED_VERSION_GAP_S || 330) * 1000; -const STEP_GAP_MS = Number(process.env.SEED_STEP_GAP_MS || 1500); -const FORCE = process.env.SEED_FORCE === '1'; +const TITLE = process.env.SEED_TITLE || '07/10 meeting agenda (sample)'; +const DATE = process.env.SEED_DATE || '2026-10-07'; function required(name) { const value = process.env[name]; @@ -46,7 +36,7 @@ function required(name) { } function log(message) { - console.log(`${new Date().toISOString()} ${message}`); + console.log(`seed: ${message}`); } /** A cookie jar per host, enough for the sign-in redirects. */ @@ -70,11 +60,7 @@ function createSession() { [...jar].map(([name, value]) => `${name}=${value}`).join('; '), ); } - const response = await fetch(url, { - ...init, - headers, - redirect: 'manual', - }); + const response = await fetch(url, { ...init, headers, redirect: 'manual' }); for (const line of response.headers.getSetCookie()) { const [pair] = line.split(';'); const index = pair.indexOf('='); @@ -89,12 +75,16 @@ function createSession() { return response; } - /** Follows redirects, carrying the cookies of each host. */ async function follow(url, init = {}) { let response = await request(url, init); let current = url; - for (let i = 0; i < 20 && [301, 302, 303, 307, 308].includes(response.status); i++) { + for ( + let i = 0; + i < 20 && [301, 302, 303, 307, 308].includes(response.status); + i++ + ) { current = new URL(response.headers.get('location'), current).toString(); + await response.arrayBuffer(); response = await request(current); } return { response, url: current }; @@ -103,9 +93,7 @@ function createSession() { return { request, follow, - cookie(url, name) { - return jarFor(url).get(name); - }, + cookie: (url, name) => jarFor(url).get(name), }; } @@ -116,7 +104,9 @@ async function signIn(username, password) { `${DOCS_URL}/api/v1.0/authenticate/`, ); const html = await response.text(); - const action = html.match(/]*id="kc-form-login"[^>]*action="([^"]+)"/)?.[1]; + const action = html.match( + /]*id="kc-form-login"[^>]*action="([^"]+)"/, + )?.[1]; if (!action) { throw new Error(`No login form for ${username} at ${url} (${response.status})`); } @@ -141,9 +131,7 @@ async function signIn(username, password) { if (!me.ok) { throw new Error(`Sign-in failed for ${username}: ${me.status}`); } - const user = await me.json(); - log(`signed in as ${username} (${user.id})`); - return { username, user, api }; + return { username, user: await me.json(), api }; } async function json(response, what) { @@ -153,119 +141,139 @@ async function json(response, what) { return response.json(); } -async function findDocument(owner) { - const response = await owner.api( - `/api/v1.0/documents/?is_creator_me=true&title=${encodeURIComponent(TITLE)}`, - ); - const { results } = await json(response, 'Listing documents'); - return results.find((doc) => doc.title === TITLE); -} - -async function readFragmentLength(user, docId) { - const response = await user.api( - `/collaboration/ydoc/v1/${ORG}/${docId}?gc=true`, - ); +async function hasContent(user, docId) { + const response = await user.api(`/collaboration/ydoc/v1/${ORG}/${docId}?gc=true`); if (!response.ok) { throw new Error(`Reading the document failed: ${response.status}`); } const { doc } = decodeAny(new Uint8Array(await response.arrayBuffer())); const ydoc = new Y.Doc(); Y.applyUpdate(ydoc, doc); - return ydoc.get(FRAGMENT).length; + return ydoc.get(FRAGMENT).length > 0; } async function ensureAccesses(owner, users, docId) { - const response = await owner.api(`/api/v1.0/documents/${docId}/accesses/`); - const existing = new Set( - (await json(response, 'Listing accesses')).map((access) => access.user?.id), + const listed = await json( + await owner.api(`/api/v1.0/documents/${docId}/accesses/`), + 'Listing accesses', ); + const existing = new Set(listed.map((access) => access.user?.id)); for (const user of users) { - if (existing.has(user.user.id)) { - continue; + if (!existing.has(user.user.id)) { + await json( + await owner.api(`/api/v1.0/documents/${docId}/accesses/`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ user_id: user.user.id, role: 'editor' }), + }), + `Giving access to ${user.username}`, + ); } - await json( - await owner.api(`/api/v1.0/documents/${docId}/accesses/`, { - method: 'POST', - headers: { 'content-type': 'application/json' }, - body: JSON.stringify({ user_id: user.user.id, role: 'editor' }), - }), - `Giving access to ${user.username}`, - ); - log(`gave ${user.username} editor access`); } } -async function patch(user, docId, update) { - const response = await user.api(`/collaboration/ydoc/v1/${ORG}/${docId}`, { - method: 'PATCH', - headers: { 'content-type': 'application/octet-stream' }, - body: encodeAny({ update }), - }); - if (!response.ok) { - throw new Error(`PATCH as ${user.username} failed: ${response.status} ${await response.text()}`); - } +/** "08:35:02–08:38:51 UTC" in a version name: when it was edited. */ +function timeRange(name) { + const [start, end] = name.match(/\d\d:\d\d:\d\d/g); + return [Date.parse(`${DATE}T${start}Z`), Date.parse(`${DATE}T${end}Z`)]; } -async function nameVersion(user, docId, name) { - const response = await user.api(`/collaboration/version/v1/${ORG}/${docId}`, { - method: 'POST', - headers: { 'content-type': 'application/octet-stream' }, - body: encodeAny({ type: 'version:v1', name }), - }); - if (!response.ok) { - throw new Error(`Naming "${name}" failed: ${response.status} ${await response.text()}`); - } -} - -async function main() { - const versions = JSON.parse(fs.readFileSync(required('SEED_DATA'), 'utf8')); - const users = []; - for (const pair of required('SEED_USERS').split(',')) { - const index = pair.indexOf(':'); - users.push(await signIn(pair.slice(0, index), pair.slice(index + 1))); - } - const [owner, ...others] = users; +/** + * Replays the versions into one gc:false document and attributes what each + * edit added to its author at its time, like Docs' fullMigrate does. + */ +function buildHistory(versions, authors) { + const replay = createReplay(); + let seen = Y.createContentIds(); + const contentmaps = []; - let doc = FORCE ? undefined : await findDocument(owner); - if (doc && (await readFragmentLength(owner, doc.id)) > 0) { - await ensureAccesses(owner, others, doc.id); - log(`already seeded: ${DOCS_URL}/docs/${doc.id}/`); - return; - } - if (!doc) { - doc = await json( - await owner.api('/api/v1.0/documents/', { - method: 'POST', - headers: { 'content-type': 'application/json' }, - body: JSON.stringify({ title: TITLE }), - }), - 'Creating the document', + function attribute(by, at) { + const all = Y.createContentIdsFromDoc(replay.ydoc, true); + const fresh = Y.excludeContentIds(all, seen); + seen = all; + const attrs = (verb) => [ + Y.createContentAttribute(verb, by), + Y.createContentAttribute(`${verb}At`, at), + ]; + contentmaps.push( + Y.createContentMapFromContentIds(fresh, attrs('insert'), attrs('delete')), ); - log(`created ${DOCS_URL}/docs/${doc.id}/`); } - await ensureAccesses(owner, others, doc.id); - // History is only shown from the date of each access on: start after them. - await sleep(2000); - const replay = createReplay(); - await patch(owner, doc.id, replay.base()); - for (const [i, version] of versions.entries()) { - const author = users[i % users.length]; - if (i > 0) { - log(`waiting ${VERSION_GAP_MS / 1000}s before the next version`); - await sleep(VERSION_GAP_MS); - } - for (const [j, blocks] of version.steps.entries()) { - if (j > 0) { - await sleep(STEP_GAP_MS); - } - await patch(author, doc.id, replay.next(blocks)); + const named = []; + versions.forEach((version, i) => { + const by = authors[i % authors.length]; + const [start, end] = timeRange(version.name); + if (i === 0) { + replay.base(); + attribute(by, start - 1000); } - await sleep(STEP_GAP_MS); - await nameVersion(author, doc.id, version.name); - log(`version ${i + 1}/${versions.length} "${version.name}" by ${author.username}: ${version.steps.length} edits`); - } - log(`done: ${DOCS_URL}/docs/${doc.id}/`); + const count = version.steps.length; + let at = start; + version.steps.forEach((blocks, j) => { + replay.next(blocks); + at = count > 1 ? Math.round(start + ((end - start) * j) / (count - 1)) : start; + attribute(by, at); + }); + named.push({ t: at, name: version.name, by }); + }); + + const nongcDoc = Y.encodeStateAsUpdate(replay.ydoc); + const gc = new Y.Doc({ gc: true }); + Y.applyUpdate(gc, nongcDoc); + return { + nongcDoc, + gcDoc: Y.encodeStateAsUpdate(gc), + contentmap: Y.encodeContentMap(Y.mergeContentMaps(contentmaps)), + contentids: Y.encodeContentIds(seen), + versions: named, + first: timeRange(versions[0].name)[0], + }; +} + +const started = Date.now(); +const users = []; +for (const pair of required('SEED_USERS').split(',')) { + const index = pair.indexOf(':'); + users.push(await signIn(pair.slice(0, index), pair.slice(index + 1))); +} +const [owner, ...others] = users; + +const found = await json( + await owner.api( + `/api/v1.0/documents/?is_creator_me=true&title=${encodeURIComponent(TITLE)}`, + ), + 'Listing documents', +); +let doc = found.results.find((d) => d.title === TITLE); +if (doc && (await hasContent(owner, doc.id))) { + await ensureAccesses(owner, others, doc.id); + log(`already there: ${DOCS_URL}/docs/${doc.id}/`); + process.exit(0); +} +if (!doc) { + doc = await json( + await owner.api('/api/v1.0/documents/', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ title: TITLE }), + }), + 'Creating the document', + ); } +await ensureAccesses(owner, others, doc.id); -await main(); +const versions = JSON.parse(fs.readFileSync(required('SEED_DATA'), 'utf8')); +const history = buildHistory( + versions, + users.map((u) => u.user.id), +); +fs.writeFileSync( + required('OUT'), + encodeAny({ org: ORG, docid: doc.id, ...history }), +); +fs.writeFileSync( + `${required('OUT')}.json`, + JSON.stringify({ docid: doc.id, url: `${DOCS_URL}/docs/${doc.id}/`, accessesFrom: history.first - 24 * 3600 * 1000 }), +); +log(`built ${versions.length} versions for ${DOCS_URL}/docs/${doc.id}/ in ${Date.now() - started} ms`); From 8d8c54b74384bfedf7f9aa13d6d2ea1d228d5d06 Mon Sep 17 00:00:00 2001 From: yousefed Date: Fri, 9 Oct 2026 09:59:37 +0200 Subject: [PATCH 7/7] =?UTF-8?q?=F0=9F=94=A7(dev-instance)=20use=20the=20us?= =?UTF-8?q?er=20name=20as=20demo=20user=20password?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The demo users alice, bob, carol and dave have their user name as password, also on a Keycloak that already exists. The admin accounts keep random passwords on the server. The README lists the users and tells how to read the admin passwords. Signed-off-by: yousefed --- deploy/dev-instance/README.md | 56 +++++++++++++++++++++++++ deploy/dev-instance/compose.yaml | 4 -- deploy/dev-instance/keycloak/realm.json | 8 ++-- deploy/dev-instance/scripts/deploy.sh | 17 ++++++++ deploy/dev-instance/scripts/init-env.sh | 7 +--- 5 files changed, 79 insertions(+), 13 deletions(-) create mode 100644 deploy/dev-instance/README.md diff --git a/deploy/dev-instance/README.md b/deploy/dev-instance/README.md new file mode 100644 index 0000000000..ca9a4b2423 --- /dev/null +++ b/deploy/dev-instance/README.md @@ -0,0 +1,56 @@ +# Dev instance + +A dev instance of Docs on one server, with the BlockNote version history work. +`.github/workflows/dev-instance.yml` deploys it. + +- Docs: https://167-233-225-101.sslip.io +- Keycloak: https://id.167-233-225-101.sslip.io + +## Demo users + +The password of each demo user is the user name. + +| User | Password | +| --- | --- | +| alice | alice | +| bob | bob | +| carol | carol | +| dave | dave | + +Every deploy makes sure that the sample document "07/10 meeting agenda (sample)" +exists. alice owns it, and bob, carol and dave can edit it. Its history has 15 +named versions by the four users (`scripts/seed.sh`). + +## Admin accounts + +The admin accounts have random passwords. They are only on the server, in +`/opt/docs/.env`: + +```bash +ssh deploy@167.233.225.101 'grep -E "^(SEED_PASSWORD_ADMIN|KEYCLOAK_ADMIN_PASSWORD)=" /opt/docs/.env' +``` + +| Account | User | Password variable | +| --- | --- | --- | +| Django admin (`/admin`) | admin@example.com | `SEED_PASSWORD_ADMIN` | +| "admin" user of the "docs" realm | admin | `SEED_PASSWORD_ADMIN` | +| Keycloak administration console | admin | `KEYCLOAK_ADMIN_PASSWORD` | + +## Deploy a change + +1. Open a pull request against `dev`. +2. Merge it. The workflow builds the frontend, yhub and y-provider images and + deploys them. + +To deploy a branch that is not merged, run the "Dev instance" workflow from the +Actions tab and set `ref`. There is only one server: the last deploy wins. + +## Server + +- The stack is in `/opt/docs` (`compose.yaml`). The `deploy` user owns it. +- The first deploy writes `/opt/docs/.env` with random secrets. It never + changes them after that. +- The data of the sample document is `/opt/docs/seed-data/meetingVersions.json`. + It is not in the repository. Without it, the deploy skips the sample + document. +- There are no backups. diff --git a/deploy/dev-instance/compose.yaml b/deploy/dev-instance/compose.yaml index 3c4ab850aa..6322164855 100644 --- a/deploy/dev-instance/compose.yaml +++ b/deploy/dev-instance/compose.yaml @@ -111,10 +111,6 @@ services: - DOCS_HOST=${DOCS_HOST} - OIDC_CLIENT_SECRET=${OIDC_CLIENT_SECRET} - SEED_PASSWORD_ADMIN=${SEED_PASSWORD_ADMIN} - - SEED_PASSWORD_ALICE=${SEED_PASSWORD_ALICE} - - SEED_PASSWORD_BOB=${SEED_PASSWORD_BOB} - - SEED_PASSWORD_CAROL=${SEED_PASSWORD_CAROL} - - SEED_PASSWORD_DAVE=${SEED_PASSWORD_DAVE} volumes: - keycloak:/opt/keycloak/data - ./keycloak/realm.json:/opt/keycloak/data/import/realm.json:ro diff --git a/deploy/dev-instance/keycloak/realm.json b/deploy/dev-instance/keycloak/realm.json index 71a1f5c601..52d76ad454 100644 --- a/deploy/dev-instance/keycloak/realm.json +++ b/deploy/dev-instance/keycloak/realm.json @@ -54,7 +54,7 @@ "credentials": [ { "type": "password", - "value": "${SEED_PASSWORD_ALICE}", + "value": "alice", "temporary": false } ] @@ -69,7 +69,7 @@ "credentials": [ { "type": "password", - "value": "${SEED_PASSWORD_BOB}", + "value": "bob", "temporary": false } ] @@ -84,7 +84,7 @@ "credentials": [ { "type": "password", - "value": "${SEED_PASSWORD_CAROL}", + "value": "carol", "temporary": false } ] @@ -99,7 +99,7 @@ "credentials": [ { "type": "password", - "value": "${SEED_PASSWORD_DAVE}", + "value": "dave", "temporary": false } ] diff --git a/deploy/dev-instance/scripts/deploy.sh b/deploy/dev-instance/scripts/deploy.sh index 9e350ad3fd..2c403e2c83 100755 --- a/deploy/dev-instance/scripts/deploy.sh +++ b/deploy/dev-instance/scripts/deploy.sh @@ -20,5 +20,22 @@ set -a; . ./.env; set +a docker compose exec -T backend python manage.py createsuperuser \ --email admin@example.com --password "${SEED_PASSWORD_ADMIN}" +# Demo users: the password is the user name. The realm import sets it on the +# first start only, so set it again for a Keycloak volume that already exists. +for attempt in $(seq 1 30); do + if docker compose exec -T -e KC_ADMIN_PASSWORD="${KEYCLOAK_ADMIN_PASSWORD}" keycloak sh -c ' + /opt/keycloak/bin/kcadm.sh config credentials --server http://localhost:8080 \ + --realm master --user admin --password "$KC_ADMIN_PASSWORD" >/dev/null && + for user in alice bob carol dave; do + /opt/keycloak/bin/kcadm.sh set-password -r docs --username "$user" --new-password "$user" + done'; then + break + fi + [ "${attempt}" = 30 ] && { echo "Keycloak is not ready" >&2; exit 1; } + sleep 4 +done + +./scripts/seed.sh + docker image prune -f >/dev/null docker compose ps --format 'table {{.Service}}\t{{.Image}}\t{{.Status}}' diff --git a/deploy/dev-instance/scripts/init-env.sh b/deploy/dev-instance/scripts/init-env.sh index d4eb604d8d..8d3ea406ad 100755 --- a/deploy/dev-instance/scripts/init-env.sh +++ b/deploy/dev-instance/scripts/init-env.sh @@ -32,12 +32,9 @@ S3_ACCESS_KEY=docs S3_SECRET_KEY=$(random) KEYCLOAK_ADMIN_PASSWORD=$(random 24) -# Dev users of the "docs" realm, imported on the first start of Keycloak. +# The "admin" user of the "docs" realm, also the password of the Django admin +# (admin@example.com). The demo users' password is their user name. SEED_PASSWORD_ADMIN=$(random 16) -SEED_PASSWORD_ALICE=$(random 16) -SEED_PASSWORD_BOB=$(random 16) -SEED_PASSWORD_CAROL=$(random 16) -SEED_PASSWORD_DAVE=$(random 16) ENV echo "Created .env for ${domain}" fi