[OCPERT-210][hold]Add OTE (OpenShift Tests Extension) migration testing for monorepo mode

Makefile

@@ -133,11 +133,8 @@ clean:
 # Build OpenShift test extension following OTE requirements:
 # - Static linking (CGO_ENABLED=0)
 # - ART compliance exemption (GO_COMPLIANCE_POLICY=exempt_all)
-cloud-credential-tests-ext:
-	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO_COMPLIANCE_POLICY=exempt_all \
-		go build -mod=vendor \
-		-ldflags "-X $(GO_PACKAGE)/pkg/version.versionFromGit=$$(git describe --long --tags --abbrev=7 --match 'v[0-9]*' )" \
-		./cmd/cloud-credential-tests-ext
+cloud-credential-tests-ext: tests-ext-build
+	cp bin/cloud-credential-operator-tests-ext ./cloud-credential-tests-ext
 .PHONY: cloud-credential-tests-ext
 
 # Run against the configured cluster in ~/.kube/config
@@ -208,3 +205,42 @@ update-go-modules-k8s:
 	done
 	go mod tidy
 	go mod vendor
+
+# OTE test extension binary configuration (Variant B: Subdirectory mode)
+TESTS_EXT_DIR := test/e2e/extension/cmd
+TESTS_EXT_BINARY := bin/cloud-credential-operator-tests-ext
+
+# Build OTE extension binary (builds from test module, outputs to bin/)
+.PHONY: tests-ext-build
+tests-ext-build:
+	@echo "Building OTE test extension binary..."
+	@cd test/e2e/extension && $(MAKE) -f bindata.mk update-bindata
+	@mkdir -p bin
+	cd test/e2e/extension && GOTOOLCHAIN=auto GOSUMDB=sum.golang.org go build -o ../../../$(TESTS_EXT_BINARY) ./cmd
+	@echo "✅ Extension binary built: $(TESTS_EXT_BINARY)"
+
+# Compress OTE extension binary (for CI/CD and container builds)
+.PHONY: tests-ext-compress
+tests-ext-compress: tests-ext-build
+	@echo "Compressing OTE extension binary..."
+	@cd bin && tar -czvf cloud-credential-operator-tests-ext.tar.gz cloud-credential-operator-tests-ext && rm -f cloud-credential-operator-tests-ext
+	@echo "Compressed binary created at bin/cloud-credential-operator-tests-ext.tar.gz"
+
+# Copy compressed binary to _output directory (for CI/CD)
+.PHONY: tests-ext-copy
+tests-ext-copy: tests-ext-compress
+	@echo "Copying compressed binary to _output..."
+	@mkdir -p _output
+	@cp bin/cloud-credential-operator-tests-ext.tar.gz _output/
+	@echo "Binary copied to _output/cloud-credential-operator-tests-ext.tar.gz"
+
+# Alias for backward compatibility
+.PHONY: extension
+extension: tests-ext-build
+
+# Clean extension binary
+.PHONY: clean-extension
+clean-extension:
+	@echo "Cleaning extension binary..."
+	@rm -f $(TESTS_EXT_BINARY) bin/cloud-credential-operator-tests-ext.tar.gz _output/cloud-credential-operator-tests-ext.tar.gz
+	@cd test/e2e/extension && $(MAKE) -f bindata.mk clean-bindata 2>/dev/null || true

go.mod

@@ -242,3 +242,5 @@ replace github.com/golang/glog => github.com/golang/glog v1.2.5
 
 // Required for openshift-tests-extension compatibility (uses OpenShift's ginkgo fork)
 replace github.com/onsi/ginkgo/v2 => github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20241205171354-8006f302fd12
+
+replace github.com/openshift/cloud-credential-operator/test/e2e/extension => ./test/e2e/extension

test/e2e/extension/bindata.mk

@@ -0,0 +1,31 @@
+# bindata.mk for embedding testdata files
+
+BINDATA_PKG := testdata
+BINDATA_OUT := testdata/bindata.go
+
+.PHONY: update-bindata
+update-bindata:
+	@echo "Generating bindata for testdata files..."
+	go-bindata \
+		-nocompress \
+		-nometadata \
+		-prefix "testdata" \
+		-pkg $(BINDATA_PKG) \
+		-o testdata/bindata.go \
+		testdata/...
+	@echo "✅ Bindata generated successfully"
+
+.PHONY: verify-bindata
+verify-bindata: update-bindata
+	@echo "Verifying bindata is up to date..."
+	git diff --exit-code $(BINDATA_OUT) || (echo "❌ Bindata is out of date. Run 'make update-bindata'" && exit 1)
+	@echo "✅ Bindata is up to date"
+
+# Legacy alias for backward compatibility
+.PHONY: bindata
+bindata: update-bindata
+
+.PHONY: clean-bindata
+clean-bindata:
+	@echo "Cleaning bindata..."
+	@rm -f $(BINDATA_OUT)

test/e2e/extension/cloudcredential_util.go

@@ -0,0 +1,279 @@
+package cloudcredential
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"sort"
+	"strings"
+	"time"
+
+	"k8s.io/apimachinery/pkg/util/wait"
+
+	o "github.com/onsi/gomega"
+	exutil "github.com/openshift/origin/test/extended/util"
+	compat_otp "github.com/openshift/origin/test/extended/util/compat_otp"
+	e2e "k8s.io/kubernetes/test/e2e/framework"
+)
+
+const (
+	ccoNs                    = "openshift-cloud-credential-operator"
+	ccoCap                   = "CloudCredential"
+	ccoRepo                  = "cloud-credential-operator"
+	ccoManifestPath          = "manifests"
+	defaultSTSCloudTokenPath = "/var/run/secrets/kubernetes.io/serviceaccount/token"
+	DefaultTimeout           = 120
+)
+
+type prometheusQueryResult struct {
+	Data struct {
+		Result []struct {
+			Metric struct {
+				Name      string `json:"__name__"`
+				Container string `json:"container"`
+				Endpoint  string `json:"endpoint"`
+				Instance  string `json:"instance"`
+				Job       string `json:"job"`
+				Mode      string `json:"mode"`
+				Namespace string `json:"namespace"`
+				Pod       string `json:"pod"`
+				Service   string `json:"service"`
+			} `json:"metric"`
+			Value []interface{} `json:"value"`
+		} `json:"result"`
+		ResultType string `json:"resultType"`
+	} `json:"data"`
+	Status string `json:"status"`
+}
+
+type credentialsRequest struct {
+	name      string
+	namespace string
+	provider  string
+	template  string
+}
+
+type azureCredential struct {
+	key   string
+	value string
+}
+
+type gcpCredential struct {
+	key   string
+	value string
+}
+
+type OcpClientVerb = string
+
+func doOcpReq(oc *exutil.CLI, verb OcpClientVerb, notEmpty bool, args ...string) string {
+	e2e.Logf("running command : oc %s %s", string(verb), strings.Join(args, " "))
+	res, err := oc.AsAdmin().WithoutNamespace().Run(string(verb)).Args(args...).Output()
+	o.Expect(err).ShouldNot(o.HaveOccurred())
+	if notEmpty {
+		o.Expect(res).ShouldNot(o.BeEmpty())
+	}
+	return res
+}
+
+func getCloudCredentialMode(oc *exutil.CLI) (string, error) {
+	var (
+		mode           string
+		iaasPlatform   string
+		rootSecretName string
+		err            error
+	)
+	iaasPlatform, err = getIaasPlatform(oc)
+	if err != nil {
+		return "", err
+	}
+	if iaasPlatform == "none" || iaasPlatform == "baremetal" {
+		mode = "none" //mode none is for baremetal
+		return mode, nil
+	}
+	//Check if the cloud providers which support Manual mode only
+	if iaasPlatform == "ibmcloud" || iaasPlatform == "alibabacloud" || iaasPlatform == "nutanix" {
+		mode = "manual"
+		return mode, nil
+	}
+	modeInCloudCredential, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("cloudcredential", "cluster", "-o=jsonpath={.spec.credentialsMode}").Output()
+	if err != nil {
+		return "", err
+	}
+	if modeInCloudCredential != "Manual" {
+		rootSecretName, err = getRootSecretName(oc)
+		if err != nil {
+			return "", err
+		}
+		modeInSecretAnnotation, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("secret", rootSecretName, "-n=kube-system", "-o=jsonpath={.metadata.annotations.cloudcredential\\.openshift\\.io/mode}").Output()
+		if err != nil {
+			if strings.Contains(modeInSecretAnnotation, "NotFound") {
+				if iaasPlatform != "aws" && iaasPlatform != "azure" && iaasPlatform != "gcp" {
+					mode = "passthrough"
+					return mode, nil
+				}
+				mode = "credsremoved"
+				return mode, nil
+			}
+			return "", err
+		}
+		if modeInSecretAnnotation == "insufficient" {
+			mode = "degraded"
+			return mode, nil
+		}
+		mode = modeInSecretAnnotation
+		return mode, nil
+	}
+	if iaasPlatform == "aws" {
+		if compat_otp.IsSTSCluster(oc) {
+			mode = "manualpodidentity"
+			return mode, nil
+		}
+	}
+	if iaasPlatform == "azure" {
+		if compat_otp.IsWorkloadIdentityCluster(oc) {
+			mode = "manualpodidentity"
+			return mode, nil
+		}
+	}
+	mode = "manual"
+	return mode, nil
+}
+
+func getRootSecretName(oc *exutil.CLI) (string, error) {
+	var rootSecretName string
+
+	iaasPlatform, err := getIaasPlatform(oc)
+	if err != nil {
+		return "", err
+	}
+	switch iaasPlatform {
+	case "aws":
+		rootSecretName = "aws-creds"
+	case "gcp":
+		rootSecretName = "gcp-credentials"
+	case "azure":
+		rootSecretName = "azure-credentials"
+	case "vsphere":
+		rootSecretName = "vsphere-creds"
+	case "openstack":
+		rootSecretName = "openstack-credentials"
+	case "ovirt":
+		rootSecretName = "ovirt-credentials"
+	default:
+		e2e.Logf("Unsupported platform: %v", iaasPlatform)
+		return "", nil
+
+	}
+	return rootSecretName, nil
+}
+
+func getIaasPlatform(oc *exutil.CLI) (string, error) {
+	output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("infrastructure", "cluster", "-o=jsonpath={.status.platformStatus.type}").Output()
+	if err != nil {
+		return "", err
+	}
+	iaasPlatform := strings.ToLower(output)
+
+	if iaasPlatform == "external" {
+		externalPlatformNameOutput, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("infrastructure", "cluster", "-o=jsonpath={.spec.platformSpec.external.platformName}").Output()
+		o.Expect(err).NotTo(o.HaveOccurred())
+		iaasPlatform = strings.ToLower(externalPlatformNameOutput)
+	}
+
+	return iaasPlatform, nil
+}
+
+func checkModeInMetric(oc *exutil.CLI, token string, mode string) error {
+	var (
+		data         prometheusQueryResult
+		modeInMetric string
+	)
+	return wait.Poll(10*time.Second, 3*time.Minute, func() (bool, error) {
+		msg, _, err := oc.AsAdmin().WithoutNamespace().Run("exec").Args("-n", "openshift-monitoring", "prometheus-k8s-0", "-c", "prometheus", "--", "curl", "-k", "-H", fmt.Sprintf("Authorization: Bearer %v", token), "https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?query=cco_credentials_mode").Outputs()
+		o.Expect(err).NotTo(o.HaveOccurred())
+		o.Expect(msg).NotTo(o.BeEmpty())
+		json.Unmarshal([]byte(msg), &data)
+		modeInMetric = data.Data.Result[0].Metric.Mode
+		e2e.Logf("cco mode in metric is %v", modeInMetric)
+		if modeInMetric != mode {
+			e2e.Logf("cco mode should be %v, but is %v in metric", mode, modeInMetric)
+			return false, nil
+		}
+		return true, nil
+	})
+}
+
+func checkSTSStyle(oc *exutil.CLI, mode string) bool {
+	output, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("secret", "cloud-credentials", "-n", "openshift-ingress-operator", "-o=jsonpath={.data.credentials}").Output()
+	o.Expect(err).NotTo(o.HaveOccurred())
+	o.Expect(output).NotTo(o.BeEmpty())
+	credentials, _ := base64.StdEncoding.DecodeString(output)
+	credConfig := strings.Split(string(credentials), "\n")
+	//Credentials items are in different order for self-managed OCP and ROSA, so sort firstly
+	sort.SliceStable(credConfig, func(i, j int) bool {
+		return strings.Compare(credConfig[i], credConfig[j]) < 0
+	})
+	if mode == "manualpodidentity" {
+		return strings.Contains(credConfig[0], "[default]") && strings.Contains(credConfig[1], "role_arn") && strings.Contains(credConfig[2], "sts_regional_endpoints") && strings.Contains(credConfig[3], "web_identity_token_file")
+	}
+	return strings.Contains(credConfig[0], "[default]") && strings.Contains(credConfig[1], "aws_access_key_id") && strings.Contains(credConfig[2], "aws_secret_access_key")
+}
+
+func patchResourceAsAdmin(oc *exutil.CLI, ns, resource, rsname, patch string) {
+	err := oc.AsAdmin().WithoutNamespace().Run("patch").Args(resource, rsname, "--type=json", "-p", patch, "-n", ns).Execute()
+	o.Expect(err).NotTo(o.HaveOccurred())
+}
+
+func (cr *credentialsRequest) create(oc *exutil.CLI) {
+	compat_otp.ApplyNsResourceFromTemplate(oc, "openshift-cloud-credential-operator", "--ignore-unknown-parameters=true", "-f", cr.template, "-p", "NAME="+cr.name, "NAMESPACE="+cr.namespace, "PROVIDER="+cr.provider)
+}
+
+// Check if CCO conditions are health
+func checkCCOHealth(oc *exutil.CLI, mode string) {
+	availableStatus, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("co", "cloud-credential", `-o=jsonpath={.status.conditions[?(@.type=="Available")].status}`).Output()
+	o.Expect(err).NotTo(o.HaveOccurred())
+	o.Expect(availableStatus).To(o.Equal("True"))
+	degradedStatus, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("co", "cloud-credential", `-o=jsonpath={.status.conditions[?(@.type=="Degraded")].status}`).Output()
+	o.Expect(err).NotTo(o.HaveOccurred())
+	o.Expect(degradedStatus).To(o.Equal("False"))
+	progressingStatus, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("co", "cloud-credential", `-o=jsonpath={.status.conditions[?(@.type=="Progressing")].status}`).Output()
+	o.Expect(err).NotTo(o.HaveOccurred())
+	o.Expect(progressingStatus).To(o.Equal("False"))
+	upgradeableStatus, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("co", "cloud-credential", `-o=jsonpath={.status.conditions[?(@.type=="Upgradeable")].status}`).Output()
+	o.Expect(err).NotTo(o.HaveOccurred())
+	//when cco mode is manual or manual+sts, upgradeableStatus is "False" due to MissingUpgradeableAnnotation
+	if mode == "manual" || mode == "manualpodidentity" {
+		o.Expect(upgradeableStatus).To(o.Equal("False"))
+		upgradeableReason, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("co", "cloud-credential", `-o=jsonpath={.status.conditions[?(@.type=="Upgradeable")].reason}`).Output()
+		o.Expect(err).NotTo(o.HaveOccurred())
+		o.Expect(upgradeableReason).To(o.Equal("MissingUpgradeableAnnotation"))
+	} else {
+		o.Expect(upgradeableStatus).To(o.Equal("True"))
+	}
+}
+
+// check webhook pod securityContext
+func checkWebhookSecurityContext(oc *exutil.CLI, podnum int) {
+	webHookPodName := make([]string, podnum)
+	for i := 0; i < len(webHookPodName); i++ {
+		var err error
+		webHookPod, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("pod", "-l", "app=pod-identity-webhook", "-n", "openshift-cloud-credential-operator", "-o=jsonpath={.items[*].metadata.name}").Output()
+		o.Expect(err).NotTo(o.HaveOccurred())
+		webHookPodName = strings.Split(strings.TrimSpace(webHookPod), " ")
+		o.Expect(len(webHookPodName)).To(o.BeNumerically(">", 0))
+		e2e.Logf("webHookPodName is %s ", webHookPodName[i])
+		allowPrivilegeEscalation, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("pod", webHookPodName[i], "-n", "openshift-cloud-credential-operator", "-o=jsonpath={.spec.containers[*].securityContext.allowPrivilegeEscalation}").Output()
+		o.Expect(err).NotTo(o.HaveOccurred())
+		o.Expect(allowPrivilegeEscalation).To(o.Equal("false"))
+		drop, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("pod", webHookPodName[i], "-n", "openshift-cloud-credential-operator", "-o=jsonpath={.spec.containers[*].securityContext.capabilities.drop}").Output()
+		o.Expect(err).NotTo(o.HaveOccurred())
+		dropAllCount := strings.Count(drop, "ALL")
+		o.Expect(dropAllCount).To(o.Equal(1))
+		runAsNonRoot, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("pod", webHookPodName[i], "-n", "openshift-cloud-credential-operator", "-o=jsonpath={.spec.securityContext.runAsNonRoot}").Output()
+		o.Expect(err).NotTo(o.HaveOccurred())
+		o.Expect(runAsNonRoot).To(o.Equal("true"))
+		seccompProfileType, err := oc.AsAdmin().WithoutNamespace().Run("get").Args("pod", webHookPodName[i], "-n", "openshift-cloud-credential-operator", "-o=jsonpath={.spec.securityContext.seccompProfile.type}").Output()
+		o.Expect(err).NotTo(o.HaveOccurred())
+		o.Expect(seccompProfileType).To(o.Equal("RuntimeDefault"))
+	}
+}