From 86d532fd2c833936b9e19ce9fe4bcc9b3a1b64a9 Mon Sep 17 00:00:00 2001 From: Patrik Suba Date: Fri, 19 Jun 2026 11:13:31 +0200 Subject: [PATCH] Initial draft of detection of node rollout prow job --- .../Azure/ARO-HCP/Azure-ARO-HCP-main.yaml | 11 +++ .../Azure-ARO-HCP-main-presubmits.yaml | 87 +++++++++++++++++++ .../aro-hcp/git/checkout-head/OWNERS | 8 ++ .../aro-hcp-git-checkout-head-commands.sh | 21 +++++ ...ro-hcp-git-checkout-head-ref.metadata.json | 15 ++++ .../aro-hcp-git-checkout-head-ref.yaml | 16 ++++ .../step-registry/aro-hcp/git/checkout/OWNERS | 8 ++ .../checkout/aro-hcp-git-checkout-commands.sh | 20 +++++ .../aro-hcp-git-checkout-ref.metadata.json | 15 ++++ .../checkout/aro-hcp-git-checkout-ref.yaml | 19 ++++ .../aro-hcp/local-e2e-upgrade/OWNERS | 12 +++ ...p-local-e2e-upgrade-workflow.metadata.json | 19 ++++ .../aro-hcp-local-e2e-upgrade-workflow.yaml | 33 +++++++ .../aro-hcp/test/local-post-upgrade/OWNERS | 12 +++ ...ro-hcp-test-local-post-upgrade-commands.sh | 58 +++++++++++++ ...-test-local-post-upgrade-ref.metadata.json | 19 ++++ .../aro-hcp-test-local-post-upgrade-ref.yaml | 52 +++++++++++ .../aro-hcp/test/local-pre-upgrade/OWNERS | 12 +++ ...aro-hcp-test-local-pre-upgrade-commands.sh | 54 ++++++++++++ ...p-test-local-pre-upgrade-ref.metadata.json | 19 ++++ .../aro-hcp-test-local-pre-upgrade-ref.yaml | 57 ++++++++++++ .../aro-hcp/upgrade/infra/OWNERS | 8 ++ .../infra/aro-hcp-upgrade-infra-commands.sh | 76 ++++++++++++++++ .../aro-hcp-upgrade-infra-ref.metadata.json | 15 ++++ .../infra/aro-hcp-upgrade-infra-ref.yaml | 26 ++++++ core-services/prow/02_config/_boskos.yaml | 4 + .../prow/02_config/generate-boskos.py | 3 + 27 files changed, 699 insertions(+) create mode 100644 ci-operator/step-registry/aro-hcp/git/checkout-head/OWNERS create mode 100644 ci-operator/step-registry/aro-hcp/git/checkout-head/aro-hcp-git-checkout-head-commands.sh create mode 100644 ci-operator/step-registry/aro-hcp/git/checkout-head/aro-hcp-git-checkout-head-ref.metadata.json create mode 100644 ci-operator/step-registry/aro-hcp/git/checkout-head/aro-hcp-git-checkout-head-ref.yaml create mode 100644 ci-operator/step-registry/aro-hcp/git/checkout/OWNERS create mode 100644 ci-operator/step-registry/aro-hcp/git/checkout/aro-hcp-git-checkout-commands.sh create mode 100644 ci-operator/step-registry/aro-hcp/git/checkout/aro-hcp-git-checkout-ref.metadata.json create mode 100644 ci-operator/step-registry/aro-hcp/git/checkout/aro-hcp-git-checkout-ref.yaml create mode 100644 ci-operator/step-registry/aro-hcp/local-e2e-upgrade/OWNERS create mode 100644 ci-operator/step-registry/aro-hcp/local-e2e-upgrade/aro-hcp-local-e2e-upgrade-workflow.metadata.json create mode 100644 ci-operator/step-registry/aro-hcp/local-e2e-upgrade/aro-hcp-local-e2e-upgrade-workflow.yaml create mode 100644 ci-operator/step-registry/aro-hcp/test/local-post-upgrade/OWNERS create mode 100755 ci-operator/step-registry/aro-hcp/test/local-post-upgrade/aro-hcp-test-local-post-upgrade-commands.sh create mode 100644 ci-operator/step-registry/aro-hcp/test/local-post-upgrade/aro-hcp-test-local-post-upgrade-ref.metadata.json create mode 100644 ci-operator/step-registry/aro-hcp/test/local-post-upgrade/aro-hcp-test-local-post-upgrade-ref.yaml create mode 100644 ci-operator/step-registry/aro-hcp/test/local-pre-upgrade/OWNERS create mode 100755 ci-operator/step-registry/aro-hcp/test/local-pre-upgrade/aro-hcp-test-local-pre-upgrade-commands.sh create mode 100644 ci-operator/step-registry/aro-hcp/test/local-pre-upgrade/aro-hcp-test-local-pre-upgrade-ref.metadata.json create mode 100644 ci-operator/step-registry/aro-hcp/test/local-pre-upgrade/aro-hcp-test-local-pre-upgrade-ref.yaml create mode 100644 ci-operator/step-registry/aro-hcp/upgrade/infra/OWNERS create mode 100644 ci-operator/step-registry/aro-hcp/upgrade/infra/aro-hcp-upgrade-infra-commands.sh create mode 100644 ci-operator/step-registry/aro-hcp/upgrade/infra/aro-hcp-upgrade-infra-ref.metadata.json create mode 100644 ci-operator/step-registry/aro-hcp/upgrade/infra/aro-hcp-upgrade-infra-ref.yaml diff --git a/ci-operator/config/Azure/ARO-HCP/Azure-ARO-HCP-main.yaml b/ci-operator/config/Azure/ARO-HCP/Azure-ARO-HCP-main.yaml index 1b08f0e134930..a11905d1d4ea4 100644 --- a/ci-operator/config/Azure/ARO-HCP/Azure-ARO-HCP-main.yaml +++ b/ci-operator/config/Azure/ARO-HCP/Azure-ARO-HCP-main.yaml @@ -502,6 +502,17 @@ tests: ARO_HCP_SUITE_PARALLELISM: "55" MULTISTAGE_PARAM_OVERRIDE_LOCATION: westus3 workflow: aro-hcp-local-e2e +- as: e2e-parallel-hypershift-upgrade + optional: true + run_if_changed: ^(?:dev-infrastructure/|config/|.*hypershift.*|\.ci-operator\.yaml$) + steps: + env: + ALLOWED_SUBSCRIPTIONS: ARO HCP E2E Hosted Clusters - Dev - 03 + ARO_HCP_CLOUD: dev + ARO_HCP_DEPLOY_ENV: ci01 + MULTISTAGE_PARAM_OVERRIDE_LOCATION: westus3 + workflow: aro-hcp-local-e2e-upgrade + timeout: 12h0m0s - as: global-pipeline-postsubmit max_concurrency: 4 postsubmit: true diff --git a/ci-operator/jobs/Azure/ARO-HCP/Azure-ARO-HCP-main-presubmits.yaml b/ci-operator/jobs/Azure/ARO-HCP/Azure-ARO-HCP-main-presubmits.yaml index 67af4c17b95b2..445aee6b79491 100644 --- a/ci-operator/jobs/Azure/ARO-HCP/Azure-ARO-HCP-main-presubmits.yaml +++ b/ci-operator/jobs/Azure/ARO-HCP/Azure-ARO-HCP-main-presubmits.yaml @@ -525,6 +525,93 @@ presubmits: secret: secretName: result-aggregator trigger: (?m)^/test( | .* )e2e-parallel,?($|\s.*) + - agent: kubernetes + always_run: false + branches: + - ^main$ + - ^main- + cluster: build05 + context: ci/prow/e2e-parallel-hypershift-upgrade + decorate: true + decoration_config: + sparse_checkout_files: + - admin/Dockerfile + - backend/Dockerfile + - fleet/Dockerfile + - frontend/Dockerfile + - hcp-recovery/Dockerfile + - image-sync/oc-mirror/Dockerfile + - kube-applier/Dockerfile + - mgmt-agent/Dockerfile + - sessiongate/Dockerfile + - test/Containerfile.e2e + - tooling/aro-hcp-exporter/Dockerfile + timeout: 12h0m0s + labels: + ci.openshift.io/generator: prowgen + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: pull-ci-Azure-ARO-HCP-main-e2e-parallel-hypershift-upgrade + optional: true + rerun_command: /test e2e-parallel-hypershift-upgrade + run_if_changed: ^(?:dev-infrastructure/|config/|.*hypershift.*|\.ci-operator\.yaml$) + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --target=e2e-parallel-hypershift-upgrade + command: + - ci-operator + env: + - name: HTTP_SERVER_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator + trigger: (?m)^/test( | .* )e2e-parallel-hypershift-upgrade,?($|\s.*) - agent: kubernetes always_run: true branches: diff --git a/ci-operator/step-registry/aro-hcp/git/checkout-head/OWNERS b/ci-operator/step-registry/aro-hcp/git/checkout-head/OWNERS new file mode 100644 index 0000000000000..8b5c8784266a3 --- /dev/null +++ b/ci-operator/step-registry/aro-hcp/git/checkout-head/OWNERS @@ -0,0 +1,8 @@ +approvers: +- aro-hcp-sl-approvers +- geoberle +- deads2k +reviewers: +- aro-hcp-sl-reviewers +- geoberle +- deads2k diff --git a/ci-operator/step-registry/aro-hcp/git/checkout-head/aro-hcp-git-checkout-head-commands.sh b/ci-operator/step-registry/aro-hcp/git/checkout-head/aro-hcp-git-checkout-head-commands.sh new file mode 100644 index 0000000000000..992ad19d4efc2 --- /dev/null +++ b/ci-operator/step-registry/aro-hcp/git/checkout-head/aro-hcp-git-checkout-head-commands.sh @@ -0,0 +1,21 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +ref="${GIT_REF:-${PULL_PULL_SHA:-}}" +if [[ -z "${ref}" ]]; then + echo "ERROR: PR head ref unknown; set GIT_REF or run on a presubmit with PULL_PULL_SHA" + exit 1 +fi + +echo "Checking out PR head ${ref}" +git fetch --tags origin "${ref}" 2>/dev/null || git fetch origin "${ref}" +git fetch --unshallow origin 2>/dev/null || true +git checkout "${ref}" || { + echo "ERROR: failed to checkout ${ref}" + exit 1 +} +git rev-parse HEAD +echo "${ref}" > "${SHARED_DIR}/git-checkout-ref" +git rev-parse HEAD > "${SHARED_DIR}/git-checkout-sha" diff --git a/ci-operator/step-registry/aro-hcp/git/checkout-head/aro-hcp-git-checkout-head-ref.metadata.json b/ci-operator/step-registry/aro-hcp/git/checkout-head/aro-hcp-git-checkout-head-ref.metadata.json new file mode 100644 index 0000000000000..ed31afc6706d1 --- /dev/null +++ b/ci-operator/step-registry/aro-hcp/git/checkout-head/aro-hcp-git-checkout-head-ref.metadata.json @@ -0,0 +1,15 @@ +{ + "path": "aro-hcp/git/checkout-head/aro-hcp-git-checkout-head-ref.yaml", + "owners": { + "approvers": [ + "aro-hcp-sl-approvers", + "geoberle", + "deads2k" + ], + "reviewers": [ + "aro-hcp-sl-reviewers", + "geoberle", + "deads2k" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/aro-hcp/git/checkout-head/aro-hcp-git-checkout-head-ref.yaml b/ci-operator/step-registry/aro-hcp/git/checkout-head/aro-hcp-git-checkout-head-ref.yaml new file mode 100644 index 0000000000000..0608db20bc365 --- /dev/null +++ b/ci-operator/step-registry/aro-hcp/git/checkout-head/aro-hcp-git-checkout-head-ref.yaml @@ -0,0 +1,16 @@ +ref: + as: aro-hcp-git-checkout-head + from: aro-hcp-e2e-tools + commands: aro-hcp-git-checkout-head-commands.sh + grace_period: 15s + resources: + requests: + cpu: 100m + memory: 256Mi + env: + - name: GIT_REF + default: "" + documentation: |- + Override for the PR head. When empty, uses PULL_PULL_SHA from the Prow job. + documentation: |- + Checkout the PR head revision after baseline tests and before in-place operator upgrade. diff --git a/ci-operator/step-registry/aro-hcp/git/checkout/OWNERS b/ci-operator/step-registry/aro-hcp/git/checkout/OWNERS new file mode 100644 index 0000000000000..8b5c8784266a3 --- /dev/null +++ b/ci-operator/step-registry/aro-hcp/git/checkout/OWNERS @@ -0,0 +1,8 @@ +approvers: +- aro-hcp-sl-approvers +- geoberle +- deads2k +reviewers: +- aro-hcp-sl-reviewers +- geoberle +- deads2k diff --git a/ci-operator/step-registry/aro-hcp/git/checkout/aro-hcp-git-checkout-commands.sh b/ci-operator/step-registry/aro-hcp/git/checkout/aro-hcp-git-checkout-commands.sh new file mode 100644 index 0000000000000..93339e4e513a9 --- /dev/null +++ b/ci-operator/step-registry/aro-hcp/git/checkout/aro-hcp-git-checkout-commands.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +ref="${GIT_REF:-main}" +if [[ -z "${GIT_REF:-}" ]]; then + echo "GIT_REF unset; using default ref=${ref}" +fi + +echo "Checking out ${ref}" +git fetch --tags origin "${ref}" 2>/dev/null || git fetch origin "${ref}" +git fetch --unshallow origin 2>/dev/null || true +git checkout "${ref}" || { + echo "ERROR: failed to checkout ${ref}" + exit 1 +} +git rev-parse HEAD +echo "${ref}" > "${SHARED_DIR}/git-checkout-ref" +git rev-parse HEAD > "${SHARED_DIR}/git-checkout-sha" diff --git a/ci-operator/step-registry/aro-hcp/git/checkout/aro-hcp-git-checkout-ref.metadata.json b/ci-operator/step-registry/aro-hcp/git/checkout/aro-hcp-git-checkout-ref.metadata.json new file mode 100644 index 0000000000000..f82f672536c8a --- /dev/null +++ b/ci-operator/step-registry/aro-hcp/git/checkout/aro-hcp-git-checkout-ref.metadata.json @@ -0,0 +1,15 @@ +{ + "path": "aro-hcp/git/checkout/aro-hcp-git-checkout-ref.yaml", + "owners": { + "approvers": [ + "aro-hcp-sl-approvers", + "geoberle", + "deads2k" + ], + "reviewers": [ + "aro-hcp-sl-reviewers", + "geoberle", + "deads2k" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/aro-hcp/git/checkout/aro-hcp-git-checkout-ref.yaml b/ci-operator/step-registry/aro-hcp/git/checkout/aro-hcp-git-checkout-ref.yaml new file mode 100644 index 0000000000000..2b954beaf8125 --- /dev/null +++ b/ci-operator/step-registry/aro-hcp/git/checkout/aro-hcp-git-checkout-ref.yaml @@ -0,0 +1,19 @@ +ref: + as: aro-hcp-git-checkout + from: aro-hcp-e2e-tools + commands: aro-hcp-git-checkout-commands.sh + grace_period: 15s + resources: + requests: + cpu: 100m + memory: 256Mi + env: + - name: GIT_REF + default: "main" + documentation: |- + Git ref to checkout (branch, tag, or SHA). Defaults to main for the base + provision phase. Override via job env when a different base ref is needed. + documentation: |- + Fetch and checkout the base git ref before regional provision. Defaults to main + so infrastructure is bootstrapped from the stable branch; later steps checkout + the PR head for upgrade validation. diff --git a/ci-operator/step-registry/aro-hcp/local-e2e-upgrade/OWNERS b/ci-operator/step-registry/aro-hcp/local-e2e-upgrade/OWNERS new file mode 100644 index 0000000000000..3ff2089d3c778 --- /dev/null +++ b/ci-operator/step-registry/aro-hcp/local-e2e-upgrade/OWNERS @@ -0,0 +1,12 @@ +approvers: +- geoberle +- mmazur +- roivaz +- venkateshsredhat +- deads2k +reviewers: +- geoberle +- mmazur +- roivaz +- venkateshsredhat +- deads2k diff --git a/ci-operator/step-registry/aro-hcp/local-e2e-upgrade/aro-hcp-local-e2e-upgrade-workflow.metadata.json b/ci-operator/step-registry/aro-hcp/local-e2e-upgrade/aro-hcp-local-e2e-upgrade-workflow.metadata.json new file mode 100644 index 0000000000000..b02830fefd6bf --- /dev/null +++ b/ci-operator/step-registry/aro-hcp/local-e2e-upgrade/aro-hcp-local-e2e-upgrade-workflow.metadata.json @@ -0,0 +1,19 @@ +{ + "path": "aro-hcp/local-e2e-upgrade/aro-hcp-local-e2e-upgrade-workflow.yaml", + "owners": { + "approvers": [ + "geoberle", + "mmazur", + "roivaz", + "venkateshsredhat", + "deads2k" + ], + "reviewers": [ + "geoberle", + "mmazur", + "roivaz", + "venkateshsredhat", + "deads2k" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/aro-hcp/local-e2e-upgrade/aro-hcp-local-e2e-upgrade-workflow.yaml b/ci-operator/step-registry/aro-hcp/local-e2e-upgrade/aro-hcp-local-e2e-upgrade-workflow.yaml new file mode 100644 index 0000000000000..dfff9c2fc07db --- /dev/null +++ b/ci-operator/step-registry/aro-hcp/local-e2e-upgrade/aro-hcp-local-e2e-upgrade-workflow.yaml @@ -0,0 +1,33 @@ +workflow: + as: aro-hcp-local-e2e-upgrade + steps: + allow_best_effort_post_steps: true + leases: + - env: LEASED_MSI_MOCK_SP + resource_type: aro-hcp-msi-mock-cs-sp-dev + pre: + - ref: aro-hcp-lease-acquire + - ref: aro-hcp-git-checkout + - ref: aro-hcp-write-config + - ref: aro-hcp-provision-environment + test: + - ref: aro-hcp-test-local-pre-upgrade + - ref: aro-hcp-git-checkout-head + - ref: aro-hcp-upgrade-infra + - ref: aro-hcp-test-local-post-upgrade + post: + - ref: aro-hcp-gather-provision-failure + - ref: aro-hcp-gather-visualization + - ref: aro-hcp-gather-test-visualization + - ref: aro-hcp-gather-custom-link-tools + - ref: aro-hcp-gather-observability + - ref: aro-hcp-gather-snapshot + - ref: aro-hcp-deprovision-environment + - ref: aro-hcp-lease-release + documentation: |- + Acquire runtime leases, checkout main (or GIT_REF override), provision regional infrastructure, run upgrade/create tests, checkout the PR head, + rerun pipeline/RP.HypershiftOperator using hypershift.image and hypershift.sharedIngressImage + digests from PR-head config/config.yaml, then run upgrade/post-infra tests. + Uses the dedicated aro-hcp-dev-upgrade-westus3-slot pool (Dev - 03, slot_count: 1) + so the provisioned cluster persists isolated across both upgrade suite phases. + Override GIT_REF on git-checkout steps via job env when needed. diff --git a/ci-operator/step-registry/aro-hcp/test/local-post-upgrade/OWNERS b/ci-operator/step-registry/aro-hcp/test/local-post-upgrade/OWNERS new file mode 100644 index 0000000000000..3ff2089d3c778 --- /dev/null +++ b/ci-operator/step-registry/aro-hcp/test/local-post-upgrade/OWNERS @@ -0,0 +1,12 @@ +approvers: +- geoberle +- mmazur +- roivaz +- venkateshsredhat +- deads2k +reviewers: +- geoberle +- mmazur +- roivaz +- venkateshsredhat +- deads2k diff --git a/ci-operator/step-registry/aro-hcp/test/local-post-upgrade/aro-hcp-test-local-post-upgrade-commands.sh b/ci-operator/step-registry/aro-hcp/test/local-post-upgrade/aro-hcp-test-local-post-upgrade-commands.sh new file mode 100755 index 0000000000000..f82b92c80d50c --- /dev/null +++ b/ci-operator/step-registry/aro-hcp/test/local-post-upgrade/aro-hcp-test-local-post-upgrade-commands.sh @@ -0,0 +1,58 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +: "${ARO_HCP_SUITE_NAME:?ARO_HCP_SUITE_NAME must be set}" + +# Must match aro-hcp-test-local-pre-upgrade: post-infra loads cluster state via setup.go. +export SETUP_FILEPATH="${SETUP_FILEPATH:-${SHARED_DIR}/e2e-setup.json}" +if [[ ! -f "${SETUP_FILEPATH}" ]]; then + printf 'Missing e2e setup file: %s (upgrade/create must run in a prior step)\n' "${SETUP_FILEPATH}" >&2 + exit 1 +fi + +env_file="${SHARED_DIR}/aro-hcp-slot.env" +if [[ ! -f "${env_file}" ]]; then + printf 'Missing runtime lease export file: %s\n' "${env_file}" >&2 + exit 1 +fi + +# shellcheck disable=SC1090 +source "${env_file}" + +export LOCATION="${SELECTED_LOCATION:-${LOCATION:-}}" +: "${LOCATION:?LOCATION must be provided by SELECTED_LOCATION or the legacy runtime slot export file}" + +export CLUSTER_PROFILE_DIR="/var/run/aro-hcp-${VAULT_SECRET_PROFILE}" + +export AZURE_CLIENT_ID; AZURE_CLIENT_ID=$(cat "${CLUSTER_PROFILE_DIR}/client-id") +export AZURE_TENANT_ID; AZURE_TENANT_ID=$(cat "${CLUSTER_PROFILE_DIR}/tenant") +export AZURE_CLIENT_SECRET; AZURE_CLIENT_SECRET=$(cat "${CLUSTER_PROFILE_DIR}/client-secret") +export INFRA_SUBSCRIPTION_ID; INFRA_SUBSCRIPTION_ID=$(cat "${CLUSTER_PROFILE_DIR}/infra-${ARO_HCP_DEPLOY_ENV}-subscription-id") +export DEPLOY_ENV="${ARO_HCP_DEPLOY_ENV}" + +az login --service-principal -u "${AZURE_CLIENT_ID}" -p "${AZURE_CLIENT_SECRET}" --tenant "${AZURE_TENANT_ID}" --output none + +unset GOFLAGS + +# This block prepares the environment to run the tests in. +# It runs against INFRA_SUBSCRIPTION. +az account set --subscription "${INFRA_SUBSCRIPTION_ID}" +make -C dev-infrastructure/ svc.aks.kubeconfig.pipeline SVC_KUBECONFIG_FILE=../kubeconfig DEPLOY_ENV="${DEPLOY_ENV}" +export KUBECONFIG=kubeconfig +export AZURE_TOKEN_CREDENTIALS=prod +FRONTEND_ADDRESS="https://$(kubectl get virtualservice -n aro-hcp aro-hcp-vs-frontend -o jsonpath='{.spec.hosts[0]}')" +make frontend-grant-ingress DEPLOY_ENV="${DEPLOY_ENV}" + +# This block runs the tests against CUSTOMER_SUBSCRIPTION. +az account set --subscription "${CUSTOMER_SUBSCRIPTION}" +make e2e-local/setup FRONTEND_ADDRESS="${FRONTEND_ADDRESS}" + +./test/aro-hcp-tests run-suite "${ARO_HCP_SUITE_NAME}" \ + --junit-path="${ARTIFACT_DIR}/junit.xml" \ + --html-path="${ARTIFACT_DIR}/extension-test-result-summary.html" \ + --max-concurrency 100 + +junit_shared_name="${E2E_JUNIT_SHARED_NAME:-junit-e2e-suite.xml.gz}" +gzip -c "${ARTIFACT_DIR}/junit.xml" > "${SHARED_DIR}/${junit_shared_name}" diff --git a/ci-operator/step-registry/aro-hcp/test/local-post-upgrade/aro-hcp-test-local-post-upgrade-ref.metadata.json b/ci-operator/step-registry/aro-hcp/test/local-post-upgrade/aro-hcp-test-local-post-upgrade-ref.metadata.json new file mode 100644 index 0000000000000..3aa745358ec50 --- /dev/null +++ b/ci-operator/step-registry/aro-hcp/test/local-post-upgrade/aro-hcp-test-local-post-upgrade-ref.metadata.json @@ -0,0 +1,19 @@ +{ + "path": "aro-hcp/test/local-post-upgrade/aro-hcp-test-local-post-upgrade-ref.yaml", + "owners": { + "approvers": [ + "geoberle", + "mmazur", + "roivaz", + "venkateshsredhat", + "deads2k" + ], + "reviewers": [ + "geoberle", + "mmazur", + "roivaz", + "venkateshsredhat", + "deads2k" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/aro-hcp/test/local-post-upgrade/aro-hcp-test-local-post-upgrade-ref.yaml b/ci-operator/step-registry/aro-hcp/test/local-post-upgrade/aro-hcp-test-local-post-upgrade-ref.yaml new file mode 100644 index 0000000000000..91696240bbf15 --- /dev/null +++ b/ci-operator/step-registry/aro-hcp/test/local-post-upgrade/aro-hcp-test-local-post-upgrade-ref.yaml @@ -0,0 +1,52 @@ +ref: + as: aro-hcp-test-local-post-upgrade + from: aro-hcp-e2e-tests + commands: aro-hcp-test-local-post-upgrade-commands.sh + credentials: + - namespace: test-credentials + name: aro-hcp-qe-pull-secret + mount_path: /var/run/aro-hcp-qe-pull-secret + - namespace: test-credentials + name: cluster-secrets-aro-hcp-dev + mount_path: /var/run/aro-hcp-dev + grace_period: 30s + timeout: 4h0m0s + resources: + requests: + cpu: 1000m + memory: 1Gi + env: + - name: VAULT_SECRET_PROFILE + default: "dev" + documentation: Selects which environment's cluster secrets to use. + - name: AROHCP_ENV + default: "development" + - name: LOCATION + default: "westus3" + documentation: Azure region for the test environment. + - name: POOLED_IDENTITIES + default: "true" + documentation: Whether to use pooled identities for the test. + - name: ARO_HCP_DEPLOY_ENV + default: "prow" + documentation: Config environment name. + - name: ARO_HCP_SUITE_NAME + default: "upgrade/post-infra" + documentation: |- + Suite to run via aro-hcp-tests run-suite after the infra upgrade step. + - name: SETUP_FILEPATH + default: "" + documentation: |- + Path to e2e-setup.json written by upgrade/create in the pre-upgrade step. When + empty, the step uses ${SHARED_DIR}/e2e-setup.json (same as pre-upgrade). + - name: E2E_JUNIT_SHARED_NAME + default: "junit-e2e-post-upgrade.xml.gz" + documentation: |- + Gzip junit artifact filename under SHARED_DIR (avoids clobbering baseline junit). + - name: COMPRESS_TIMING_METADATA + default: "true" + documentation: Whether to compress timing metadata files with gzip. + documentation: |- + Same local e2e harness as aro-hcp-test-local, run after infra upgrade. + Defaults to the upgrade/post-infra suite, loading e2e-setup.json from + ${SHARED_DIR} written by the pre-upgrade step. diff --git a/ci-operator/step-registry/aro-hcp/test/local-pre-upgrade/OWNERS b/ci-operator/step-registry/aro-hcp/test/local-pre-upgrade/OWNERS new file mode 100644 index 0000000000000..3ff2089d3c778 --- /dev/null +++ b/ci-operator/step-registry/aro-hcp/test/local-pre-upgrade/OWNERS @@ -0,0 +1,12 @@ +approvers: +- geoberle +- mmazur +- roivaz +- venkateshsredhat +- deads2k +reviewers: +- geoberle +- mmazur +- roivaz +- venkateshsredhat +- deads2k diff --git a/ci-operator/step-registry/aro-hcp/test/local-pre-upgrade/aro-hcp-test-local-pre-upgrade-commands.sh b/ci-operator/step-registry/aro-hcp/test/local-pre-upgrade/aro-hcp-test-local-pre-upgrade-commands.sh new file mode 100755 index 0000000000000..f579a0c57e2ef --- /dev/null +++ b/ci-operator/step-registry/aro-hcp/test/local-pre-upgrade/aro-hcp-test-local-pre-upgrade-commands.sh @@ -0,0 +1,54 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +: "${ARO_HCP_SUITE_NAME:?ARO_HCP_SUITE_NAME must be set}" + +# upgrade/create writes; upgrade/post-infra reads. SHARED_DIR persists across workflow steps. +export SETUP_FILEPATH="${SETUP_FILEPATH:-${SHARED_DIR}/e2e-setup.json}" + +env_file="${SHARED_DIR}/aro-hcp-slot.env" +if [[ ! -f "${env_file}" ]]; then + printf 'Missing runtime lease export file: %s\n' "${env_file}" >&2 + exit 1 +fi + +# shellcheck disable=SC1090 +source "${env_file}" + +export LOCATION="${SELECTED_LOCATION:-${LOCATION:-}}" +: "${LOCATION:?LOCATION must be provided by SELECTED_LOCATION or the legacy runtime slot export file}" + +export CLUSTER_PROFILE_DIR="/var/run/aro-hcp-${VAULT_SECRET_PROFILE}" + +export AZURE_CLIENT_ID; AZURE_CLIENT_ID=$(cat "${CLUSTER_PROFILE_DIR}/client-id") +export AZURE_TENANT_ID; AZURE_TENANT_ID=$(cat "${CLUSTER_PROFILE_DIR}/tenant") +export AZURE_CLIENT_SECRET; AZURE_CLIENT_SECRET=$(cat "${CLUSTER_PROFILE_DIR}/client-secret") +export INFRA_SUBSCRIPTION_ID; INFRA_SUBSCRIPTION_ID=$(cat "${CLUSTER_PROFILE_DIR}/infra-${ARO_HCP_DEPLOY_ENV}-subscription-id") +export DEPLOY_ENV="${ARO_HCP_DEPLOY_ENV}" + +az login --service-principal -u "${AZURE_CLIENT_ID}" -p "${AZURE_CLIENT_SECRET}" --tenant "${AZURE_TENANT_ID}" --output none + +unset GOFLAGS + +# This block prepares the environment to run the tests in. +# It runs against INFRA_SUBSCRIPTION. +az account set --subscription "${INFRA_SUBSCRIPTION_ID}" +make -C dev-infrastructure/ svc.aks.kubeconfig.pipeline SVC_KUBECONFIG_FILE=../kubeconfig DEPLOY_ENV="${DEPLOY_ENV}" +export KUBECONFIG=kubeconfig +export AZURE_TOKEN_CREDENTIALS=prod +FRONTEND_ADDRESS="https://$(kubectl get virtualservice -n aro-hcp aro-hcp-vs-frontend -o jsonpath='{.spec.hosts[0]}')" +make frontend-grant-ingress DEPLOY_ENV="${DEPLOY_ENV}" + +# This block runs the tests against CUSTOMER_SUBSCRIPTION. +az account set --subscription "${CUSTOMER_SUBSCRIPTION}" +make e2e-local/setup FRONTEND_ADDRESS="${FRONTEND_ADDRESS}" + +./test/aro-hcp-tests run-suite "${ARO_HCP_SUITE_NAME}" \ + --junit-path="${ARTIFACT_DIR}/junit.xml" \ + --html-path="${ARTIFACT_DIR}/extension-test-result-summary.html" \ + --max-concurrency 100 + +junit_shared_name="${E2E_JUNIT_SHARED_NAME:-junit-e2e-suite.xml.gz}" +gzip -c "${ARTIFACT_DIR}/junit.xml" > "${SHARED_DIR}/${junit_shared_name}" diff --git a/ci-operator/step-registry/aro-hcp/test/local-pre-upgrade/aro-hcp-test-local-pre-upgrade-ref.metadata.json b/ci-operator/step-registry/aro-hcp/test/local-pre-upgrade/aro-hcp-test-local-pre-upgrade-ref.metadata.json new file mode 100644 index 0000000000000..bea0e79ce8e6a --- /dev/null +++ b/ci-operator/step-registry/aro-hcp/test/local-pre-upgrade/aro-hcp-test-local-pre-upgrade-ref.metadata.json @@ -0,0 +1,19 @@ +{ + "path": "aro-hcp/test/local-pre-upgrade/aro-hcp-test-local-pre-upgrade-ref.yaml", + "owners": { + "approvers": [ + "geoberle", + "mmazur", + "roivaz", + "venkateshsredhat", + "deads2k" + ], + "reviewers": [ + "geoberle", + "mmazur", + "roivaz", + "venkateshsredhat", + "deads2k" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/aro-hcp/test/local-pre-upgrade/aro-hcp-test-local-pre-upgrade-ref.yaml b/ci-operator/step-registry/aro-hcp/test/local-pre-upgrade/aro-hcp-test-local-pre-upgrade-ref.yaml new file mode 100644 index 0000000000000..39997ff29b120 --- /dev/null +++ b/ci-operator/step-registry/aro-hcp/test/local-pre-upgrade/aro-hcp-test-local-pre-upgrade-ref.yaml @@ -0,0 +1,57 @@ +ref: + as: aro-hcp-test-local-pre-upgrade + from: aro-hcp-e2e-tests + commands: aro-hcp-test-local-pre-upgrade-commands.sh + credentials: + - namespace: test-credentials + name: aro-hcp-qe-pull-secret + mount_path: /var/run/aro-hcp-qe-pull-secret + - namespace: test-credentials + name: cluster-secrets-aro-hcp-dev + mount_path: /var/run/aro-hcp-dev + grace_period: 30s + timeout: 4h0m0s + resources: + requests: + cpu: 1000m + memory: 1Gi + env: + - name: VAULT_SECRET_PROFILE + default: "dev" + documentation: Selects which environment's cluster secrets to use. + - name: AROHCP_ENV + default: "development" + - name: LOCATION + default: "westus3" + documentation: Azure region for the test environment. + - name: POOLED_IDENTITIES + default: "true" + documentation: Whether to use pooled identities for the test. + - name: ARO_HCP_DEPLOY_ENV + default: "ci00" + documentation: Config environment name. + - name: ARO_HCP_SUITE_NAME + default: "upgrade/create" + documentation: |- + Suite to run via aro-hcp-tests run-suite before the infra upgrade step. + - name: SETUP_FILEPATH + default: "" + documentation: |- + Path for upgrade/create to write e2e-setup.json. When empty, the step sets + ${SHARED_DIR}/e2e-setup.json so upgrade/post-infra can load it in a later step. + - name: ARO_E2E_SKIP_CLEANUP + default: "true" + documentation: |- + Skip cluster cleanup after the create suite so the cluster remains for + infra upgrade and post-upgrade validation. + - name: E2E_JUNIT_SHARED_NAME + default: "junit-e2e-pre-upgrade.xml.gz" + documentation: |- + Gzip junit artifact filename under SHARED_DIR (avoids clobbering post-upgrade junit). + - name: COMPRESS_TIMING_METADATA + default: "true" + documentation: Whether to compress timing metadata files with gzip. + documentation: |- + Same local e2e harness as aro-hcp-test-local for the pre-upgrade phase of the + upgrade workflow. Defaults to the upgrade/create suite, which writes + ${SHARED_DIR}/e2e-setup.json for upgrade/post-infra. diff --git a/ci-operator/step-registry/aro-hcp/upgrade/infra/OWNERS b/ci-operator/step-registry/aro-hcp/upgrade/infra/OWNERS new file mode 100644 index 0000000000000..8b5c8784266a3 --- /dev/null +++ b/ci-operator/step-registry/aro-hcp/upgrade/infra/OWNERS @@ -0,0 +1,8 @@ +approvers: +- aro-hcp-sl-approvers +- geoberle +- deads2k +reviewers: +- aro-hcp-sl-reviewers +- geoberle +- deads2k diff --git a/ci-operator/step-registry/aro-hcp/upgrade/infra/aro-hcp-upgrade-infra-commands.sh b/ci-operator/step-registry/aro-hcp/upgrade/infra/aro-hcp-upgrade-infra-commands.sh new file mode 100644 index 0000000000000..a903544ea4d07 --- /dev/null +++ b/ci-operator/step-registry/aro-hcp/upgrade/infra/aro-hcp-upgrade-infra-commands.sh @@ -0,0 +1,76 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +if [[ ! -f "${SHARED_DIR}/config.yaml" ]]; then + echo "ERROR: ${SHARED_DIR}/config.yaml missing; run aro-hcp-provision-environment first" + exit 1 +fi + +env_file="${SHARED_DIR}/aro-hcp-slot.env" +if [[ ! -f "${env_file}" ]]; then + printf 'Missing runtime lease export file: %s\n' "${env_file}" >&2 + exit 1 +fi + +# shellcheck disable=SC1090 +source "${env_file}" + +export LOCATION="${SELECTED_LOCATION:-${LOCATION:-}}" +: "${LOCATION:?LOCATION must be provided by SELECTED_LOCATION or the legacy runtime slot export file}" + +export CLUSTER_PROFILE_DIR="/var/run/aro-hcp-${VAULT_SECRET_PROFILE}" + +export AZURE_CLIENT_ID; AZURE_CLIENT_ID=$(cat "${CLUSTER_PROFILE_DIR}/client-id") +export AZURE_TENANT_ID; AZURE_TENANT_ID=$(cat "${CLUSTER_PROFILE_DIR}/tenant") +export AZURE_CLIENT_SECRET; AZURE_CLIENT_SECRET=$(cat "${CLUSTER_PROFILE_DIR}/client-secret") +INFRA_SUBSCRIPTION_ID=$(cat "${CLUSTER_PROFILE_DIR}/infra-${ARO_HCP_DEPLOY_ENV}-subscription-id") +export INFRA_SUBSCRIPTION_ID +export DEPLOY_ENV="${ARO_HCP_DEPLOY_ENV}" +export AZURE_TOKEN_CREDENTIALS=prod +export SKIP_CONFIRM=true +export PERSIST=true +export DETECT_DIRTY_GIT_WORKTREE=0 + +az login --service-principal -u "${AZURE_CLIENT_ID}" -p "${AZURE_CLIENT_SECRET}" --tenant "${AZURE_TENANT_ID}" --output none +az account set --subscription "${INFRA_SUBSCRIPTION_ID}" + +if ! yq -e ".clouds.dev.environments.${DEPLOY_ENV}.defaults.hypershift" config/config.yaml >/dev/null; then + echo "ERROR: hypershift defaults missing in config/config.yaml for DEPLOY_ENV=${DEPLOY_ENV}" >&2 + exit 1 +fi + +OVERRIDE_CONFIG_FILE="${SHARED_DIR}/config-override-upgrade.yaml" + +yq eval -n " + .clouds.dev.environments.${DEPLOY_ENV}.defaults.hypershift = ( + load(\"config/config.yaml\") | .clouds.dev.environments.${DEPLOY_ENV}.defaults.hypershift + ) +" > "${OVERRIDE_CONFIG_FILE}" + +echo "Created upgrade override at: ${OVERRIDE_CONFIG_FILE}" +cat "${OVERRIDE_CONFIG_FILE}" + +cp "${OVERRIDE_CONFIG_FILE}" "${SHARED_DIR}/config-override.yaml" + +echo "Hypershift operator image (in override, sourced from PR-head config/config.yaml):" +yq ".clouds.dev.environments.${DEPLOY_ENV}.defaults.hypershift.image" "${OVERRIDE_CONFIG_FILE}" +echo "Hypershift shared ingress image (in override, sourced from PR-head config/config.yaml):" +yq ".clouds.dev.environments.${DEPLOY_ENV}.defaults.hypershift.sharedIngressImage" "${OVERRIDE_CONFIG_FILE}" + +unset GOFLAGS + +run_pipeline() { + local target="$1" + echo "Running pipeline/${target}" + make "pipeline/${target}" \ + DEPLOY_ENV="${DEPLOY_ENV}" \ + OVERRIDE_CONFIG_FILE="${OVERRIDE_CONFIG_FILE}" +} + +cd dev-infrastructure && make mgmt.aks.kubeconfig DEPLOY_ENV="${DEPLOY_ENV}" && cd .. +run_pipeline RP.HypershiftOperator + +echo "upgrade" > "${SHARED_DIR}/provision-phase" +date -u +"%Y-%m-%dT%H:%M:%SZ" > "${SHARED_DIR}/infra-upgrade-timestamp-rfc3339" diff --git a/ci-operator/step-registry/aro-hcp/upgrade/infra/aro-hcp-upgrade-infra-ref.metadata.json b/ci-operator/step-registry/aro-hcp/upgrade/infra/aro-hcp-upgrade-infra-ref.metadata.json new file mode 100644 index 0000000000000..6c1ce1eff558b --- /dev/null +++ b/ci-operator/step-registry/aro-hcp/upgrade/infra/aro-hcp-upgrade-infra-ref.metadata.json @@ -0,0 +1,15 @@ +{ + "path": "aro-hcp/upgrade/infra/aro-hcp-upgrade-infra-ref.yaml", + "owners": { + "approvers": [ + "aro-hcp-sl-approvers", + "geoberle", + "deads2k" + ], + "reviewers": [ + "aro-hcp-sl-reviewers", + "geoberle", + "deads2k" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/aro-hcp/upgrade/infra/aro-hcp-upgrade-infra-ref.yaml b/ci-operator/step-registry/aro-hcp/upgrade/infra/aro-hcp-upgrade-infra-ref.yaml new file mode 100644 index 0000000000000..f10b6271cb93a --- /dev/null +++ b/ci-operator/step-registry/aro-hcp/upgrade/infra/aro-hcp-upgrade-infra-ref.yaml @@ -0,0 +1,26 @@ +ref: + as: aro-hcp-upgrade-infra + from: aro-hcp-e2e-tools + commands: aro-hcp-upgrade-infra-commands.sh + grace_period: 15s + timeout: 2h0m0s + resources: + requests: + cpu: 1000m + memory: 1Gi + credentials: + - namespace: test-credentials + name: cluster-secrets-aro-hcp-dev + mount_path: /var/run/aro-hcp-dev + env: + - name: VAULT_SECRET_PROFILE + default: "dev" + documentation: |- + Selects which environment's cluster secrets to use (dev, int, stg, prod). + - name: ARO_HCP_DEPLOY_ENV + default: "ci00" + documentation: Config environment name. + documentation: |- + In-place Hypershift upgrade on the existing regional environment after git-checkout-head. + Builds OVERRIDE_CONFIG_FILE from the hypershift block (operator + sharedIngressImage + digests) in PR-head config/config.yaml, then reruns pipeline/RP.HypershiftOperator. diff --git a/core-services/prow/02_config/_boskos.yaml b/core-services/prow/02_config/_boskos.yaml index d4ce926606daa..8c71cf160d81b 100644 --- a/core-services/prow/02_config/_boskos.yaml +++ b/core-services/prow/02_config/_boskos.yaml @@ -98,6 +98,10 @@ resources: - aro-hcp-dev-shard3-slot-02 state: free type: aro-hcp-dev-shard3-slot +- names: + - aro-hcp-dev-upgrade-westus3-slot-00 + state: free + type: aro-hcp-dev-upgrade-westus3-slot - max-count: 1 min-count: 1 state: free diff --git a/core-services/prow/02_config/generate-boskos.py b/core-services/prow/02_config/generate-boskos.py index ef3f01b3a44d0..8900408439ea2 100755 --- a/core-services/prow/02_config/generate-boskos.py +++ b/core-services/prow/02_config/generate-boskos.py @@ -324,6 +324,7 @@ 'aro-hcp-dev-shard1-slot': {}, 'aro-hcp-dev-shard2-slot': {}, 'aro-hcp-dev-shard3-slot': {}, + 'aro-hcp-dev-upgrade-westus3-slot': {}, 'aro-hcp-int-shard0-slot': {}, 'aro-hcp-stg-shard0-slot': {}, # END ARO-HCP E2E SLOT TYPES @@ -795,6 +796,8 @@ CONFIG['aro-hcp-dev-shard2-slot']['aro-hcp-dev-shard2-slot-{i:0>2}'.format(i=i)] = 1 for i in range(3): CONFIG['aro-hcp-dev-shard3-slot']['aro-hcp-dev-shard3-slot-{i:0>2}'.format(i=i)] = 1 +for i in range(1): + CONFIG['aro-hcp-dev-upgrade-westus3-slot']['aro-hcp-dev-upgrade-westus3-slot-{i:0>2}'.format(i=i)] = 1 for i in range(1): CONFIG['aro-hcp-int-shard0-slot']['aro-hcp-int-shard0-slot-{i:0>2}'.format(i=i)] = 1 for i in range(1):