Replace Solr with Elasticsearch 8 and add candidate-based deployment

.dockerignore

@@ -1,5 +1,9 @@
 .dockerignore
 **/*.sql
+**/*.sql.*
+**/*.dump
+**/*.dump.*
+private/
 base.yml
 dev.yml
 prod.yml

.github/workflows/docker-build.yml

@@ -2,63 +2,125 @@ name: CoMSES Docker CI
 
 on:
   push:
-    branches: [ main ]
+    branches: [main]
   pull_request:
-    branches: [ main ]
+    branches: [main]
+  workflow_dispatch:
 
-jobs:
-
-  build:
+permissions:
+  contents: read
+  packages: write
 
+jobs:
+  build-test-publish:
     runs-on: ubuntu-latest
+    env:
+      CATALOG_IMAGE: ghcr.io/${{ github.repository }}:sha-${{ github.sha }}
+      CATALOG_SOURCE: ${{ github.server_url }}/${{ github.repository }}
 
     steps:
-    - name: Checkout repo + submodules
-      # actions/checkout v7.0.1, pinned to a full commit SHA.
-      uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
-      with:
-        submodules: recursive
-    - name: Prepare development stack
-      # Renders the dev Compose config (validating base.yml + dev.yml)
-      # and generates the local credentials the staging/prod renderings
-      # reference as file-based secrets.
-      run: make bootstrap
-    - name: Validate staging/prod Compose rendering
-      # The deploy contract requires an immutable CATALOG_IMAGE and an
-      # explicit CATALOG_ES_HOST for every staging/prod rendering
-      # (staging.yml uses ${CATALOG_ES_HOST:?...}); provide both, assert
-      # the rendered stack, and assert that rendering fails without the
-      # required ES host (there must be no silent default).
-      env:
-        CATALOG_IMAGE: comses/catalog/ci:ci-${{ github.sha }}
-        CATALOG_ES_HOST: elasticsearch8
-      run: |
-        bash scripts/compose.sh staging /tmp/catalog-ci-staging.yml
-        grep -q "image: ${CATALOG_IMAGE}" /tmp/catalog-ci-staging.yml
-        grep -q "ELASTICSEARCH_HOST: ${CATALOG_ES_HOST}" /tmp/catalog-ci-staging.yml
-        grep -q "DOMAIN_NAME: staging-catalog.comses.net" /tmp/catalog-ci-staging.yml
-        bash scripts/compose.sh prod /tmp/catalog-ci-prod.yml
-        grep -q "image: ${CATALOG_IMAGE}" /tmp/catalog-ci-prod.yml
-        grep -q "ELASTICSEARCH_HOST: ${CATALOG_ES_HOST}" /tmp/catalog-ci-prod.yml
-        grep -q "DOMAIN_NAME: catalog.comses.net" /tmp/catalog-ci-prod.yml
-        if env -u CATALOG_ES_HOST bash scripts/compose.sh staging; then
-          echo "ERROR: staging rendering must fail without CATALOG_ES_HOST" >&2
-          exit 1
-        fi
-    - name: Build production image
-      # Exercises the release build path end to end: immutable reference
-      # validation, release-version metadata generation, and a direct
-      # docker build of the django image with the production RUN_SCRIPT
-      # (Gunicorn runtime) — independent of any Compose rendering — then
-      # verifies the Gunicorn runtime is in the built image. Building
-      # needs only a plain Docker daemon.
-      env:
-        CATALOG_IMAGE: comses/catalog/ci:ci-${{ github.sha }}
-        CATALOG_ES_HOST: elasticsearch8
-      run: |
-        bash scripts/deploy.sh build
-        docker run --rm "${CATALOG_IMAGE}" gunicorn --version
-    - name: Run Django tests
-      # test.sh runs the migration gate (invoke check-migrations) before
-      # the suite, so there is no separate makemigrations --check step.
-      run: make test
+      - name: Checkout Catalog
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
+        with:
+          fetch-depth: 0
+          submodules: false
+
+      - name: Checkout pinned Citation revision
+        env:
+          CITATION_OWNER: ${{ github.event.pull_request.head.repo.owner.login || github.repository_owner }}
+        run: bash scripts/checkout-citation.sh "${CITATION_OWNER}"
+
+      - name: Prepare development stack
+        run: make bootstrap cite-config
+
+      - name: Validate scripts and deployment controller
+        run: |
+          scripts=(
+            scripts/compose.sh
+            scripts/deploy.sh
+            scripts/database.sh
+            scripts/dev-restore.sh
+            scripts/checkout-citation.sh
+            deploy/docker/prod.sh
+            deploy/docker/scheduler.sh
+            deploy/cron/daily_catalog_tasks
+            deploy/cron/monthly_catalog_tasks
+          )
+          bash -n "${scripts[@]}"
+          shellcheck "${scripts[@]}"
+          python3 -m py_compile scripts/catalogctl.py scripts/tests/test_catalogctl.py
+          make deploy-controller-test
+
+      - name: Validate deployment Compose files
+        env:
+          CATALOG_BUNDLE_REVISION: ${{ github.sha }}
+          CATALOG_CONFIG_DIR: ${{ github.workspace }}/deploy/conf
+          CATALOG_SHARED_DIR: /var/lib/comses-catalog/shared
+          CATALOG_BUNDLE_DIR: ${{ github.workspace }}
+          CATALOG_HTTP_BIND: 127.0.0.1:80
+        run: |
+          CATALOG_ENV=staging CATALOG_DOMAIN=staging-catalog.comses.net \
+            bash scripts/compose.sh staging /tmp/catalog-staging.yml
+          CATALOG_ENV=prod CATALOG_DOMAIN=catalog.comses.net \
+            bash scripts/compose.sh prod /tmp/catalog-prod.yml
+          docker compose -f /tmp/catalog-staging.yml config --quiet
+          docker compose -f /tmp/catalog-prod.yml config --quiet
+          grep -q 'catalog.settings.staging' /tmp/catalog-staging.yml
+          grep -q 'catalog.settings.prod' /tmp/catalog-prod.yml
+
+      - name: Run application checks and tests
+        run: make check migrations-check cite-check cite-migrations-check test-all
+
+      - name: Build and inspect production image
+        run: |
+          make image-build
+          docker run --rm "${CATALOG_IMAGE}" gunicorn --version
+          test "$(docker image inspect "${CATALOG_IMAGE}" --format '{{index .Config.Labels "org.opencontainers.image.revision"}}')" = "${GITHUB_SHA}"
+          test "$(docker image inspect "${CATALOG_IMAGE}" --format '{{index .Config.Labels "org.comses.catalog.citation-revision"}}')" = "$(git rev-parse HEAD:citation)"
+
+      - name: Log in to GHCR
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+        env:
+          GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: echo "${GHCR_TOKEN}" | docker login ghcr.io --username "${GITHUB_ACTOR}" --password-stdin
+
+      - name: Publish immutable candidate and handoff
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+        run: |
+          make image-push | tee /tmp/catalog-image-push.log
+          digest="$(sed -n 's/.*digest: \(sha256:[0-9a-f]\{64\}\).*/\1/p' /tmp/catalog-image-push.log | tail -n 1)"
+          test -n "${digest}"
+          image_name="${CATALOG_IMAGE%:*}"
+          image_digest="${image_name}@${digest}"
+          docker pull "${image_digest}"
+          test "$(docker image inspect "${image_digest}" --format '{{index .Config.Labels "org.opencontainers.image.revision"}}')" = "${GITHUB_SHA}"
+          python3 - "${image_digest}" "${GITHUB_SHA}" "$(git rev-parse HEAD:citation)" <<'PY'
+          import json
+          import sys
+
+          image, catalog_revision, citation_revision = sys.argv[1:]
+          with open("release-handoff.json", "w", encoding="utf-8") as output:
+              json.dump(
+                  {
+                      "schema_version": 1,
+                      "image": image,
+                      "bundle_revision": catalog_revision,
+                      "citation_revision": citation_revision,
+                      "workflow_run": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}",
+                  },
+                  output,
+                  indent=2,
+                  sort_keys=True,
+              )
+              output.write("\n")
+          PY
+          cat release-handoff.json >> "${GITHUB_STEP_SUMMARY}"
+
+      - name: Upload release handoff
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
+        with:
+          name: catalog-candidate-${{ github.sha }}
+          path: release-handoff.json
+          if-no-files-found: error
+          retention-days: 30

.github/workflows/release-tag.yml

@@ -0,0 +1,116 @@
+name: Publish final release
+
+on:
+  push:
+    tags:
+      - 'v[0-9][0-9][0-9][0-9].[0-9][0-9]'
+      - 'v[0-9][0-9][0-9][0-9].[0-9][0-9].[0-9]'
+      - 'v[0-9][0-9][0-9][0-9].[0-9][0-9].[0-9][0-9]'
+      - 'v[0-9][0-9][0-9][0-9].[0-9][0-9].[0-9][0-9][0-9]'
+
+permissions:
+  contents: write
+  packages: write
+
+jobs:
+  publish-final-release:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout tag
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
+        with:
+          fetch-depth: 0
+          submodules: false
+
+      - name: Verify annotated release tag on main
+        id: release
+        run: |
+          tag="${GITHUB_REF_NAME}"
+          [[ "${tag}" =~ ^v[0-9]{4}\.[0-9]{2}(\.[0-9]{1,3})?$ ]]
+          test "$(git cat-file -t "${GITHUB_REF}")" = tag
+          revision="$(git rev-list -n 1 "${GITHUB_REF}")"
+          git fetch origin main
+          git merge-base --is-ancestor "${revision}" origin/main
+          echo "tag=${tag}" >> "${GITHUB_OUTPUT}"
+          echo "revision=${revision}" >> "${GITHUB_OUTPUT}"
+
+      - name: Log in to GHCR
+        env:
+          GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: echo "${GHCR_TOKEN}" | docker login ghcr.io --username "${GITHUB_ACTOR}" --password-stdin
+
+      - name: Alias the tested image without rebuilding
+        env:
+          TAG: ${{ steps.release.outputs.tag }}
+          REVISION: ${{ steps.release.outputs.revision }}
+        run: |
+          image_name="ghcr.io/${GITHUB_REPOSITORY}"
+          source_image="${image_name}:sha-${REVISION}"
+          release_image="${image_name}:${TAG}"
+          docker pull "${source_image}"
+          source_ref="$(docker image inspect "${source_image}" --format '{{index .RepoDigests 0}}')"
+          source_digest="${source_ref##*@}"
+          citation_revision="$(git rev-parse "${REVISION}:citation")"
+          test "$(docker image inspect "${source_image}" --format '{{index .Config.Labels "org.opencontainers.image.revision"}}')" = "${REVISION}"
+          test "$(docker image inspect "${source_image}" --format '{{index .Config.Labels "org.comses.catalog.citation-revision"}}')" = "${citation_revision}"
+          docker tag "${source_image}" "${release_image}"
+          docker push "${release_image}" | tee /tmp/catalog-release-push.log
+          release_digest="$(sed -n 's/.*digest: \(sha256:[0-9a-f]\{64\}\).*/\1/p' /tmp/catalog-release-push.log | tail -n 1)"
+          test "${release_digest}" = "${source_digest}"
+          printf '{\n  "bundle_revision": "%s",\n  "citation_revision": "%s",\n  "image": "%s@%s",\n  "release_tag": "%s",\n  "schema_version": 1\n}\n' \
+            "${REVISION}" "${citation_revision}" "${image_name}" "${release_digest}" "${TAG}" > release-handoff.json
+          cat release-handoff.json >> "${GITHUB_STEP_SUMMARY}"
+
+      - name: Upload release handoff
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
+        with:
+          name: catalog-release-${{ steps.release.outputs.tag }}
+          path: release-handoff.json
+          if-no-files-found: error
+          retention-days: 90
+
+      - name: Publish GitHub Release
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          TAG: ${{ steps.release.outputs.tag }}
+        run: |
+          if ! gh release view "${TAG}" --repo "${GITHUB_REPOSITORY}" >/dev/null 2>&1; then
+            gh release create "${TAG}" \
+              'release-handoff.json#Deployment handoff' \
+              --repo "${GITHUB_REPOSITORY}" \
+              --verify-tag \
+              --title "Catalog ${TAG#v}" \
+              --generate-notes
+          else
+            test "$(gh release view "${TAG}" --repo "${GITHUB_REPOSITORY}" --json tagName --jq .tagName)" = "${TAG}"
+            test "$(gh release view "${TAG}" --repo "${GITHUB_REPOSITORY}" --json isPrerelease --jq .isPrerelease)" = false
+
+            asset_count="$(gh release view "${TAG}" --repo "${GITHUB_REPOSITORY}" --json assets \
+              --jq '[.assets[] | select(.name == "release-handoff.json")] | length')"
+            case "${asset_count}" in
+              0)
+                gh release upload "${TAG}" \
+                  'release-handoff.json#Deployment handoff' \
+                  --repo "${GITHUB_REPOSITORY}"
+                ;;
+              1)
+                existing_release_dir="$(mktemp -d)"
+                gh release download "${TAG}" \
+                  --repo "${GITHUB_REPOSITORY}" \
+                  --pattern release-handoff.json \
+                  --dir "${existing_release_dir}"
+                cmp release-handoff.json "${existing_release_dir}/release-handoff.json"
+                ;;
+              *)
+                echo "ERROR: release ${TAG} has duplicate release-handoff.json assets" >&2
+                exit 1
+                ;;
+            esac
+
+            if [[ "$(gh release view "${TAG}" --repo "${GITHUB_REPOSITORY}" --json isDraft --jq .isDraft)" == true ]]; then
+              gh release edit "${TAG}" --repo "${GITHUB_REPOSITORY}" --draft=false
+            fi
+          fi
+
+          test "$(gh release view "${TAG}" --repo "${GITHUB_REPOSITORY}" --json isDraft --jq .isDraft)" = false

.gitignore

@@ -18,11 +18,14 @@ zotero_data.json
 
 # Dumps
 catalog.sql
+*.sql.*
 *.dump
+*.dump.*
+private/
 
 ### Docker
 docker-compose.yml
-docker
+/docker/
 deploy/state/
 
 ### Django ###
@@ -83,3 +86,5 @@ jspm_packages
 
 *.sql
 .slim/deepwork/
+
+thoughts/