Migrate CI/CD pipeline from Docker Hub to JFrog for using server images

.github/actions/run-ee-server/action.yml

@@ -2,77 +2,57 @@ name: 'Run EE Server in a Docker container'
 description: 'Run EE server. Returns once server is ready. Only tested on Linux and macOS'
 # NOTE: do not share this server container with others
 # since it's using the default admin / admin credentials
+outputs:
+  container-network:
+    description: Forwards the container's network name from setup-as-server
+    value: ${{ steps.setup-as-server.outputs.network-name }}
+  container-name:
+    description: Forwards the container name from setup-as-server
+    value: ${{ steps.setup-as-server.outputs.container-names }}
 inputs:
   # All inputs in composite actions are strings
-  registry-name:
-    description: Registry name
-    required: false
-    default: docker.io
-  registry-username:
-    description: Required for using release candidates
-    required: false
-  # Github Composite Actions can't access secrets
-  # so we need to pass them in as inputs
-  registry-password:
-    description: Required for using release candidates
-    required: false
-  image-name:
-    required: false
-    description: aerospike/aerospike-server-enterprise
-    default: 'aerospike/aerospike-server-enterprise'
+  oidc-provider-name:
+    description: For pulling server images from JFrog
+    required: true
+  oidc-audience:
+    description: For pulling server images from JFrog
+    required: true
+  features-content:
+    description: For enabling strong consistency
+    required: true
   server-tag:
     required: true
     description: Specify Docker tag
-    default: 'latest'
   where-is-client-connecting-from:
     required: false
     description: 'docker-host, separate-docker-container, "remote-connection" via DOCKER_HOST'
     default: 'docker-host'
-  env-vars:
-    required: false
-    description: Used to disable server features
-    default: 'STRONG_CONSISTENCY=1 SECURITY=1 MUTUAL_TLS=1'
 
 runs:
   using: "composite"
   steps:
   # Start up server
-
-  - name: Log into registry to get non-public server RCs
-    # We can still pull public images while logged in, so just do this all the time to make things simple
-    uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
+  - uses: aerospike/shared-workflows/.github/actions/setup-aerospike-server@48edf9ff59ab4da5f15915b22d8895f19ce7fd55
+    id: setup-as-server
     with:
-      registry: ${{ inputs.registry-name }}
-      username: ${{ inputs.registry-username }}
-      password: ${{ inputs.registry-password }}
-
-  - run: echo IMAGE_FULL_NAME=${{ inputs.registry-name }}/${{ inputs.image-name }}:${{ inputs.server-tag }} >> $GITHUB_ENV
-    shell: bash
-
-  - run: echo CA_CERT_FILE_NAME="ca.cer" >> $GITHUB_ENV
-    shell: bash
-
-  - run: echo TLS_PORT="4333" >> $GITHUB_ENV
-    shell: bash
+      enable-security: "true"
+      enable-tls: "true"
+      enable-strong-consistency: "true"
+      features-content: ${{ inputs.features-content }}
+      num-nodes: 1
+      oidc-provider: ${{ inputs.oidc-provider-name }}
+      oidc-audience: ${{ inputs.oidc-audience }}
+      server-tag: ${{ inputs.server-tag }}
+      env-vars: DEFAULT_TTL=2592000\;
 
-  - name: 'macOS: install timeout command'
-    if: ${{ runner.os == 'macOS' }}
-    run: brew install coreutils
-    shell: bash
-
-  # Github composite actions don't support env variables for the whole composite action,
-  # so this is a workaround
-  - id: get-container-name
-    run: echo container-name=aerospike >> $GITHUB_OUTPUT
+  - run: echo SUPERUSER_NAME_AND_PASSWORD="superuser" >> $GITHUB_ENV
     shell: bash
 
-  - run: ${{ inputs.env-vars }} bash ./run-ee-server.bash
-    working-directory: .github/workflows/docker-setup
+  - run: |
+      docker run --rm --network host aerospike/aerospike-tools asadm -U admin -P admin --enable --execute "manage acl \
+          create user $SUPERUSER_NAME_AND_PASSWORD password $SUPERUSER_NAME_AND_PASSWORD \
+          roles read-write-udf, sys-admin, user-admin, data-admin"
     shell: bash
-    env:
-      CONTAINER_NAME: ${{ steps.get-container-name.outputs.container-name }}
-
-  # Configure tests
 
   - name: Install crudini to manipulate config.conf
     run: pipx install crudini --pip-args "-c ${{ github.workspace }}/.github/workflows/requirements.txt"
@@ -89,19 +69,21 @@ runs:
     working-directory: test
     shell: bash
 
-  - run: echo SUPERUSER_NAME_AND_PASSWORD="superuser" >> $GITHUB_ENV
-    shell: bash
-
   - name: Set credentials in config file
     run: |
       crudini --existing=param --set config.conf enterprise-edition user ${{ env.SUPERUSER_NAME_AND_PASSWORD }}
       crudini --existing=param --set config.conf enterprise-edition password ${{ env.SUPERUSER_NAME_AND_PASSWORD }}
       crudini --set config.conf tls enable true
       # Cannot use abs path because config.conf is copied into Docker container during cibuildwheel tests
-      crudini --set config.conf tls cafile ../.github/workflows/docker-setup/${{ env.CA_CERT_FILE_NAME }}
-      crudini --set config.conf tls keyfile ../.github/workflows/docker-setup/client.pem
-      crudini --set config.conf tls certfile ../.github/workflows/docker-setup/client.cer
 
+      tls_cert_dir=${{ steps.setup-as-server.outputs.tls-cert-dir }}
+      if [[ "${{ inputs.where-is-client-connecting-from }}" == "separate-docker-container" ]]; then
+        tls_cert_dir="/host${tls_cert_dir}"
+      fi
+
+      crudini --set config.conf tls cafile $tls_cert_dir/ca.crt
+      crudini --set config.conf tls keyfile $tls_cert_dir/client.key
+      crudini --set config.conf tls certfile $tls_cert_dir/client.crt
     working-directory: test
     shell: bash
 
@@ -121,7 +103,10 @@ runs:
 
   - name: Set IP address to Docker container for the server
     if: ${{ inputs.where-is-client-connecting-from == 'separate-docker-container' }}
-    run: echo SERVER_IP=$(docker container inspect -f '{{ .NetworkSettings.Networks.bridge.IPAddress }}' ${{ steps.get-container-name.outputs.container-name }}) >> $GITHUB_ENV
+    run: |
+      server_container_network_name=${{ steps.setup-as-server.outputs.network-name }}
+      # Go templating doesn't support selecting names with hyphens
+      echo SERVER_IP=$(docker container inspect -f json ${{ steps.setup-as-server.outputs.container-names }} | jq -r ".[].NetworkSettings.Networks.\"${server_container_network_name}\".IPAddress") >> $GITHUB_ENV
     shell: bash
 
   - name: Invalid input
@@ -134,6 +119,6 @@ runs:
   - name: Set EE server's IP address and TLS name for test config
     run: |
       cluster_name=$(docker run --rm --network host aerospike/aerospike-tools asinfo -U admin -P admin -v "get-config:context=service" -l | grep -i cluster-name | cut -d = -f 2)
-      crudini --existing=param --set config.conf enterprise-edition hosts "${{ env.SERVER_IP }}:${{ env.TLS_PORT }}|${cluster_name}"
+      crudini --existing=param --set config.conf enterprise-edition hosts "${{ env.SERVER_IP }}:${{ steps.setup-as-server.outputs.tls-service-ports }}|${cluster_name}"
     working-directory: test
     shell: bash

.github/workflows/build-and-run-stage-tests.yml

@@ -1,6 +1,12 @@
 name: Build artifacts and run stage tests
 run-name: Build artifacts and run stage tests (registry-name=${{ inputs.registry-name }}, server-tag=${{ inputs.server-tag }}, test-macos-x86=${{ inputs.test-macos-x86 }})
 
+permissions:
+  contents: read
+  id-token: write
+  statuses: write
+  packages: read
+
 on:
   workflow_dispatch:
     inputs:
@@ -17,7 +23,6 @@ on:
       server-tag:
         type: string
         required: true
-        default: 'latest'
         description: 'Server docker image tag'
       test-macos-x86:
         required: true

.github/workflows/build-artifacts.yml

@@ -5,6 +5,12 @@ run-name: Build artifacts (run_tests=${{ inputs.run_tests }}, registry-name=${{
 # Optionally run tests on manylinux wheels
 # Then upload artifacts to Github
 
+permissions:
+  id-token: write
+  statuses: write
+  packages: read
+  contents: read
+
 on:
   workflow_dispatch:
     inputs:
@@ -36,8 +42,7 @@ on:
         default: 'aerospike/aerospike-server-enterprise'
       server-tag:
         type: string
-        required: true
-        default: 'latest'
+        required: false
         description: 'Server docker image tag (e.g to test a client backport version)'
       test-file:
         required: false
@@ -84,16 +89,11 @@ on:
       server-tag:
         type: string
         required: false
-        default: 'latest'
       test-file:
         required: false
         type: string
         default: ''
     secrets:
-      DOCKER_HUB_BOT_USERNAME:
-        required: true
-      DOCKER_HUB_BOT_PW:
-        required: true
       MAC_M1_SELF_HOSTED_RUNNER_PW:
         required: true
 

.github/workflows/build-wheels.yml

@@ -3,6 +3,12 @@ run-name: 'Build wheels (python-tags=${{ inputs.python-tags }}, platform-tag=${{
 
 # Build wheels on all (or select) Python versions supported by the Python client for a specific platform
 
+permissions:
+  id-token: write
+  statuses: write
+  packages: read
+  contents: read
+
 on:
   workflow_dispatch:
     inputs:
@@ -67,8 +73,7 @@ on:
         description: Image name
         default: 'aerospike/aerospike-server-enterprise'
       server-tag:
-        required: true
-        default: 'latest'
+        required: false
         description: 'Server docker image tag'
       test-file:
         required: false
@@ -116,18 +121,12 @@ on:
       server-tag:
         required: false
         type: string
-        default: 'latest'
         description: 'Server docker image tag'
       test-file:
         required: false
         type: string
         default: ''
     secrets:
-      # Just make all the secrets required to make things simpler...
-      DOCKER_HUB_BOT_USERNAME:
-        required: true
-      DOCKER_HUB_BOT_PW:
-        required: true
       QE_DOCKER_REGISTRY_USERNAME:
         required: true
       QE_DOCKER_REGISTRY_PASSWORD:
@@ -141,8 +140,6 @@ env:
   # Github mac m1 and windows runners don't support Docker / nested virtualization
   # so we need to use self-hosted runners to test wheels for these platforms
   RUN_INTEGRATION_TESTS_IN_CIBW: ${{ inputs.run_tests && (startsWith(inputs.platform-tag, 'manylinux') || inputs.platform-tag == 'macosx_x86_64') }}
-  REGISTRY_USERNAME: ${{ inputs.registry-name == 'docker.io' && secrets.DOCKER_HUB_BOT_USERNAME || secrets.QE_DOCKER_REGISTRY_USERNAME }}
-  REGISTRY_PASSWORD: ${{ inputs.registry-name == 'docker.io' && secrets.DOCKER_HUB_BOT_PW || secrets.QE_DOCKER_REGISTRY_PASSWORD }}
 
 jobs:
   # Maps don't exist in Github Actions, so we have to store the map using a script and fetch it in a job
@@ -224,11 +221,11 @@ jobs:
     - name: 'Run Aerospike server in Docker container and configure tests accordingly'
       if: ${{ env.RUN_INTEGRATION_TESTS_IN_CIBW == 'true' }}
       uses: ./.github/actions/run-ee-server
+      id: run-ee-server
       with:
-        registry-name: ${{ inputs.registry-name }}
-        registry-username: ${{ env.REGISTRY_USERNAME }}
-        registry-password: ${{ env.REGISTRY_PASSWORD }}
-        image-name: ${{ inputs.image-name }}
+        oidc-provider-name: ${{ vars.OIDC_PROVIDER_NAME }}
+        oidc-audience: ${{ vars.OIDC_AUDIENCE }}
+        features-content: ${{ secrets.FEATURES_CONTENT }}
         server-tag: ${{ inputs.server-tag }}
         where-is-client-connecting-from: ${{ inputs.platform-tag == 'macosx_x86_64' && 'docker-host' || 'separate-docker-container' }}
 
@@ -303,6 +300,11 @@ jobs:
       # Just do it for all Python versions (even those that don't require more room) for futureproofing
       run: echo CIBW_ENVIRONMENT_MACOS="LDFLAGS='-headerpad_max_install_names'" >> $GITHUB_ENV
 
+    - if: ${{ env.RUN_INTEGRATION_TESTS_IN_CIBW == 'true' }}
+      run: |
+        echo CIBW_CONTAINER_ENGINE="docker;create_args: --network ${{ steps.run-ee-server.outputs.container-network }}" >> $GITHUB_ENV
+      shell: bash
+
     - name: Build wheel
       uses: pypa/cibuildwheel@298ed2fb2c105540f5ed055e8a6ad78d82dd3a7e # v3.3.1
       id: cibuildwheel
@@ -342,10 +344,9 @@ jobs:
     #   uses: mxschmitt/action-tmate@v3
 
     # For debugging
-    - run: |
-        # Checks that server started up properly
-        docker logs aerospike
-      if: ${{ always() && env.RUN_INTEGRATION_TESTS_IN_CIBW == 'true' }}
+    # Checks that server started up properly
+    - if: ${{ !cancelled() && env.RUN_INTEGRATION_TESTS_IN_CIBW == 'true' }}
+      run: docker logs ${{ steps.run-ee-server.outputs.container-name }}
       shell: bash
 
     - name: Upload wheels to GitHub
@@ -403,10 +404,9 @@ jobs:
 
       - uses: ./.github/actions/run-ee-server
         with:
-          registry-name: ${{ inputs.registry-name }}
-          registry-username: ${{ env.REGISTRY_USERNAME }}
-          registry-password: ${{ env.REGISTRY_PASSWORD }}
-          image-name: ${{ inputs.image-name }}
+          oidc-provider-name: ${{ vars.OIDC_PROVIDER_NAME }}
+          oidc-audience: ${{ vars.OIDC_AUDIENCE }}
+          features-content: ${{ secrets.FEATURES_CONTENT }}
           server-tag: ${{ inputs.server-tag }}
           where-is-client-connecting-from: ${{ inputs.platform-tag == 'win_amd64' && 'remote-connection' || 'docker-host' }}
 

.github/workflows/bump-stage-and-upload-to-jfrog.yml

@@ -1,6 +1,11 @@
 on:
   workflow_call:
     inputs:
+      dry-run:
+        required: false
+        default: false
+        type: boolean
+        description: "Don't tag or upload anything to JFrog"
       passed-dev-tag:
         type: string
         description: Dev tag to fast forward the stage branch to
@@ -31,6 +36,7 @@ jobs:
     with:
       change: 'promote-dev-build-to-rc'
       ref: ${{ vars.STAGE_BRANCH_NAME }}
+      dry-run: ${{ inputs.dry-run }}
     secrets: inherit
 
   rebuild-artifacts-with-rc-version:
@@ -50,6 +56,7 @@ jobs:
     uses: ./.github/workflows/upload-to-jfrog.yml
     with:
       version: ${{ needs.promote-dev-build-to-rc.outputs.new_version }}
+      dry-run: ${{ inputs.dry-run }}
     secrets: inherit
 
   # See reason for deleting artifacts in dev-workflow-p2.yml

.github/workflows/bump-version.yml

@@ -5,6 +5,11 @@ name: Bump version
 on:
   workflow_dispatch:
     inputs:
+      dry-run:
+        required: false
+        default: false
+        type: boolean
+        description: "Don't tag"
       change:
         type: choice
         description: Python script name to update the version
@@ -15,6 +20,11 @@ on:
         - promote-rc-build-to-release
   workflow_call:
     inputs:
+      dry-run:
+        required: false
+        default: false
+        type: boolean
+        description: "Don't tag"
       change:
         # Since workflow_call doesn't support 'options' input type,
         # we take in a string instead that must be a valid Python script name (excluding the .py part)
@@ -93,4 +103,5 @@ jobs:
     with:
       new_version: ${{ needs.get-new-version.outputs.new_version }}
       ref: ${{ inputs.is_workflow_call && inputs.ref || github.ref }}
+      dry-run: ${{ inputs.dry-run }}
     secrets: inherit