diff --git a/internal/controller/assets/postgres.conf b/internal/controller/assets/postgres.conf index 67446efe..2fb04d23 100644 --- a/internal/controller/assets/postgres.conf +++ b/internal/controller/assets/postgres.conf @@ -2,4 +2,8 @@ huge_pages = off ssl = on ssl_cert_file = '/etc/certs/tls.crt' ssl_key_file = '/etc/certs/tls.key' -ssl_ca_file = '/etc/certs/cm-olspostgresca/service-ca.crt' +# mTLS is not supported by lightspeed-stack or OGX (llama-stack) for the +# database connection. Neither application supports presenting client certificates +# (sslcert/sslkey) to PostgreSQL, so ssl_ca_file has no effect (even when pg_hba.conf +# is correctly configured for mTLS) +# ssl_ca_file = '' diff --git a/internal/controller/ca_bundle.go b/internal/controller/ca_bundle.go new file mode 100644 index 00000000..7c43a1d8 --- /dev/null +++ b/internal/controller/ca_bundle.go @@ -0,0 +1,190 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "bytes" + "context" + "crypto/sha256" + "crypto/x509" + "encoding/pem" + "fmt" + "os" + "time" + + common_helper "github.com/openstack-k8s-operators/lib-common/modules/common/helper" + apiv1beta1 "github.com/openstack-lightspeed/operator/api/v1beta1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +type caCert struct { + hash [sha256.Size]byte + cert *x509.Certificate + expire time.Time +} + +type caBundle struct { + certs []caCert +} + +// getOperatorCABundle reads the system CA bundle from the operator pod's filesystem. +var getOperatorCABundle = func() ([]byte, error) { + contents, err := os.ReadFile(SystemTLSCABundlePath) + if err != nil { + return nil, fmt.Errorf("failed to read system CA bundle: %w", err) + } + return contents, nil +} + +// getCertsFromPEM parses PEM data and adds valid certificates to the bundle. +// Rejects non-CERTIFICATE blocks and invalid X.509 data. Skips expired certs. +// Deduplicates by SHA256 hash of the raw DER bytes. +func (cab *caBundle) getCertsFromPEM(pemData []byte) error { + if pemData == nil { + return fmt.Errorf("certificate data is nil") + } + + rest := pemData + for { + var block *pem.Block + block, rest = pem.Decode(rest) + if block == nil { + break + } + + if block.Type != "CERTIFICATE" { + return fmt.Errorf("invalid PEM block type %q: only CERTIFICATE blocks are permitted", block.Type) + } + + certificate, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return fmt.Errorf("invalid certificate: %w", err) + } + + if time.Now().After(certificate.NotAfter) { + continue + } + + blockHash := sha256.Sum256(block.Bytes) + isDuplicate := false + for _, existing := range cab.certs { + if existing.hash == blockHash { + isDuplicate = true + break + } + } + if !isDuplicate { + cab.certs = append(cab.certs, caCert{ + hash: blockHash, + cert: certificate, + expire: certificate.NotAfter, + }) + } + } + + if len(bytes.TrimSpace(rest)) > 0 { + return fmt.Errorf("trailing non-PEM data (%d bytes)", len(bytes.TrimSpace(rest))) + } + + return nil +} + +// encodePEM encodes all certificates in the bundle back to PEM format. +func (cab *caBundle) encodePEM() []byte { + var result []byte + for _, c := range cab.certs { + block := &pem.Block{ + Type: "CERTIFICATE", + Bytes: c.cert.Raw, + } + result = append(result, pem.EncodeToMemory(block)...) + } + return result +} + +// mergeCertsFromConfigMap reads the Data section of the given ConfigMap +// and adds any valid certificate entries to the bundle. +func (cab *caBundle) mergeCertsFromConfigMap(h *common_helper.Helper, ctx context.Context, cmName string) error { + cm := &corev1.ConfigMap{} + if err := h.GetClient().Get(ctx, client.ObjectKey{ + Name: cmName, + Namespace: h.GetBeforeObject().GetNamespace(), + }, cm); err != nil { + return err + } + for key, certData := range cm.Data { + if err := cab.getCertsFromPEM([]byte(certData)); err != nil { + return fmt.Errorf("%w: key %q in ConfigMap %q: %v", ErrParseUserCA, key, cmName, err) + } + } + return nil +} + +// reconcileCABundleConfigMap builds a CA bundle containing the operator's +// system CA certificates, a user-provided CA ConfigMap (if specified), as well as +// the "kube-root-ca.crt" and "openshift-service-ca.crt" ConfigMaps. It then creates +// or updates the managed ConfigMap, which is mounted into application pods. +func reconcileCABundleConfigMap(h *common_helper.Helper, ctx context.Context, instance *apiv1beta1.OpenStackLightspeed) error { + logger := h.GetLogger() + bundle := &caBundle{} + + systemCAs, err := getOperatorCABundle() + if err != nil { + return fmt.Errorf("%w: %v", ErrReadSystemCABundle, err) + } + + if err := bundle.getCertsFromPEM(systemCAs); err != nil { + return fmt.Errorf("%w: %v", ErrParseSystemCABundle, err) + } + + certsCMs := []string{OpenShiftServiceCAConfigMap, KubeRootCAConfigMap} + if instance.Spec.TLSCACertBundle != "" { + certsCMs = append(certsCMs, instance.Spec.TLSCACertBundle) + } + + for _, certCM := range certsCMs { + if err := bundle.mergeCertsFromConfigMap(h, ctx, certCM); err != nil { + return fmt.Errorf("%w %q: %v", ErrGetCAConfigMap, certCM, err) + } + logger.Info("CA certificates merged", "configmap", certCM) + } + + bundlePEM := bundle.encodePEM() + + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: CABundleConfigMapName, + Namespace: h.GetBeforeObject().GetNamespace(), + }, + } + + result, err := controllerutil.CreateOrPatch(ctx, h.GetClient(), cm, func() error { + cm.Data = map[string]string{ + CABundleKey: string(bundlePEM), + } + return controllerutil.SetControllerReference(h.GetBeforeObject(), cm, h.GetScheme()) + }) + if err != nil { + return fmt.Errorf("%w: %v", ErrCreateCABundle, err) + } + + logger.Info("CA bundle ConfigMap reconciled", "name", cm.Name, "result", result, "certCount", len(bundle.certs)) + return nil +} diff --git a/internal/controller/ca_bundle_test.go b/internal/controller/ca_bundle_test.go new file mode 100644 index 00000000..5105ed37 --- /dev/null +++ b/internal/controller/ca_bundle_test.go @@ -0,0 +1,638 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "bytes" + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/sha256" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "log" + "math/big" + "os" + "strings" + "testing" + "time" + + common_helper "github.com/openstack-k8s-operators/lib-common/modules/common/helper" + apiv1beta1 "github.com/openstack-lightspeed/operator/api/v1beta1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/log/zap" +) + +// validPEM is a self-signed test certificate generated once in TestMain. +var validPEM []byte + +func TestMain(m *testing.M) { + now := time.Now() + data, err := makeSelfSignedPEM(now.Add(-1*time.Hour), now.Add(10*365*24*time.Hour)) + if err != nil { + log.Fatalf("failed to generate test certificate: %v", err) + } + validPEM = data + + os.Exit(m.Run()) +} + +func makeSelfSignedPEM(notBefore, notAfter time.Time) ([]byte, error) { + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return nil, err + } + + template := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "test-cert"}, + NotBefore: notBefore, + NotAfter: notAfter, + KeyUsage: x509.KeyUsageCertSign, + IsCA: true, + BasicConstraintsValid: true, + } + + derBytes, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key) + if err != nil { + return nil, err + } + + return pem.EncodeToMemory(&pem.Block{ + Type: "CERTIFICATE", + Bytes: derBytes, + }), nil +} + +// generateSelfSignedPEM creates a self-signed certificate PEM with the given +// validity window. Useful for producing expired or far-future test certs. +func generateSelfSignedPEM(t *testing.T, notBefore, notAfter time.Time) []byte { + t.Helper() + + data, err := makeSelfSignedPEM(notBefore, notAfter) + if err != nil { + t.Fatalf("failed to generate certificate: %v", err) + } + + return data +} + +// --------------------------------------------------------------------------- +// getCertsFromPEM tests +// --------------------------------------------------------------------------- + +func TestGetCertsFromPEM_ValidSingleCert(t *testing.T) { + cab := &caBundle{} + err := cab.getCertsFromPEM(validPEM) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(cab.certs) != 1 { + t.Fatalf("expected 1 cert, got %d", len(cab.certs)) + } + if cab.certs[0].cert.Subject.CommonName != "test-cert" { + t.Errorf("unexpected CN: %s", cab.certs[0].cert.Subject.CommonName) + } +} + +func TestGetCertsFromPEM_ValidMultipleCerts(t *testing.T) { + now := time.Now() + cert1 := generateSelfSignedPEM(t, now.Add(-1*time.Hour), now.Add(10*365*24*time.Hour)) + cert2 := generateSelfSignedPEM(t, now.Add(-2*time.Hour), now.Add(10*365*24*time.Hour)) + + combined := append(cert1, cert2...) + + cab := &caBundle{} + err := cab.getCertsFromPEM(combined) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(cab.certs) != 2 { + t.Fatalf("expected 2 certs, got %d", len(cab.certs)) + } +} + +func TestGetCertsFromPEM_InvalidBlockType(t *testing.T) { + privKeyPEM := []byte(`-----BEGIN PRIVATE KEY----- +MC4CAQAwBQYDK2VwBCIEIOlahWCjbH4UqSYP3tqOmP6MuEnMxOPHsYulrfWZ/3aV +-----END PRIVATE KEY----- +`) + cab := &caBundle{} + err := cab.getCertsFromPEM(privKeyPEM) + if err == nil { + t.Fatal("expected error for non-CERTIFICATE block, got nil") + } + if !strings.Contains(err.Error(), "PRIVATE KEY") { + t.Errorf("error should mention the invalid block type, got: %v", err) + } +} + +func TestGetCertsFromPEM_InvalidX509Data(t *testing.T) { + // A CERTIFICATE block with garbage bytes inside. + garbage := pem.EncodeToMemory(&pem.Block{ + Type: "CERTIFICATE", + Bytes: []byte("this is not valid DER data"), + }) + + cab := &caBundle{} + err := cab.getCertsFromPEM(garbage) + if err == nil { + t.Fatal("expected error for invalid X.509 data, got nil") + } + if !strings.Contains(err.Error(), "invalid certificate") { + t.Errorf("error should mention invalid certificate, got: %v", err) + } +} + +func TestGetCertsFromPEM_NilInput(t *testing.T) { + cab := &caBundle{} + err := cab.getCertsFromPEM(nil) + if err == nil { + t.Fatal("expected error for nil input, got nil") + } + if !strings.Contains(err.Error(), "nil") { + t.Errorf("error should mention nil, got: %v", err) + } +} + +func TestGetCertsFromPEM_EmptyInput(t *testing.T) { + cab := &caBundle{} + err := cab.getCertsFromPEM([]byte("")) + if err != nil { + t.Fatalf("unexpected error for empty input: %v", err) + } + if len(cab.certs) != 0 { + t.Errorf("expected 0 certs for empty input, got %d", len(cab.certs)) + } +} + +func TestGetCertsFromPEM_NoPEMBlocks(t *testing.T) { + // Non-empty but contains no PEM blocks (just plain text). + cab := &caBundle{} + err := cab.getCertsFromPEM([]byte("hello world, no PEM here")) + if err == nil { + t.Fatal("expected error for input with no PEM blocks, got nil") + } + if !strings.Contains(err.Error(), "trailing non-PEM data") { + t.Errorf("expected trailing non-PEM data error, got: %v", err) + } +} + +func TestGetCertsFromPEM_Deduplication(t *testing.T) { + // The same valid PEM certificate concatenated twice. + doubled := append(validPEM, validPEM...) + + cab := &caBundle{} + err := cab.getCertsFromPEM(doubled) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(cab.certs) != 1 { + t.Fatalf("expected 1 cert after dedup, got %d", len(cab.certs)) + } +} + +func TestGetCertsFromPEM_DeduplicationAcrossCalls(t *testing.T) { + // Calling getCertsFromPEM twice with the same data should still deduplicate. + cab := &caBundle{} + if err := cab.getCertsFromPEM(validPEM); err != nil { + t.Fatalf("first call: unexpected error: %v", err) + } + if err := cab.getCertsFromPEM(validPEM); err != nil { + t.Fatalf("second call: unexpected error: %v", err) + } + if len(cab.certs) != 1 { + t.Fatalf("expected 1 cert after dedup across calls, got %d", len(cab.certs)) + } +} + +func TestGetCertsFromPEM_ExpiredCertSkipped(t *testing.T) { + // Generate a certificate that expired yesterday. + now := time.Now() + expiredPEM := generateSelfSignedPEM(t, now.Add(-48*time.Hour), now.Add(-24*time.Hour)) + + cab := &caBundle{} + err := cab.getCertsFromPEM(expiredPEM) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(cab.certs) != 0 { + t.Fatalf("expected 0 certs (expired should be skipped), got %d", len(cab.certs)) + } +} + +func TestGetCertsFromPEM_MixedExpiredAndValid(t *testing.T) { + now := time.Now() + expiredPEM := generateSelfSignedPEM(t, now.Add(-48*time.Hour), now.Add(-24*time.Hour)) + combined := append(expiredPEM, validPEM...) + + cab := &caBundle{} + err := cab.getCertsFromPEM(combined) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(cab.certs) != 1 { + t.Fatalf("expected 1 cert (only the valid one), got %d", len(cab.certs)) + } + if cab.certs[0].cert.Subject.CommonName != "test-cert" { + t.Errorf("expected the valid cert to remain, got CN=%s", cab.certs[0].cert.Subject.CommonName) + } +} + +// --------------------------------------------------------------------------- +// encodePEM tests +// --------------------------------------------------------------------------- + +func TestEncodePEM_RoundTrip(t *testing.T) { + // Parse the valid cert, encode it, parse again, compare. + cab := &caBundle{} + if err := cab.getCertsFromPEM(validPEM); err != nil { + t.Fatalf("initial parse failed: %v", err) + } + + encoded := cab.encodePEM() + if len(encoded) == 0 { + t.Fatal("encodePEM returned empty output") + } + + cab2 := &caBundle{} + if err := cab2.getCertsFromPEM(encoded); err != nil { + t.Fatalf("round-trip parse failed: %v", err) + } + + if len(cab2.certs) != len(cab.certs) { + t.Fatalf("cert count mismatch after round-trip: original=%d, decoded=%d", + len(cab.certs), len(cab2.certs)) + } + + if cab.certs[0].hash != cab2.certs[0].hash { + t.Error("certificate hash mismatch after round-trip") + } +} + +func TestEncodePEM_MultipleCerts(t *testing.T) { + now := time.Now() + cert1 := generateSelfSignedPEM(t, now.Add(-1*time.Hour), now.Add(10*365*24*time.Hour)) + cert2 := generateSelfSignedPEM(t, now.Add(-2*time.Hour), now.Add(10*365*24*time.Hour)) + combined := append(cert1, cert2...) + + cab := &caBundle{} + if err := cab.getCertsFromPEM(combined); err != nil { + t.Fatalf("initial parse failed: %v", err) + } + if len(cab.certs) != 2 { + t.Fatalf("expected 2 certs, got %d", len(cab.certs)) + } + + encoded := cab.encodePEM() + + cab2 := &caBundle{} + if err := cab2.getCertsFromPEM(encoded); err != nil { + t.Fatalf("round-trip parse failed: %v", err) + } + if len(cab2.certs) != 2 { + t.Fatalf("expected 2 certs after round-trip, got %d", len(cab2.certs)) + } +} + +func TestEncodePEM_EmptyBundle(t *testing.T) { + cab := &caBundle{} + encoded := cab.encodePEM() + if encoded != nil { + t.Errorf("expected nil for empty bundle, got %d bytes", len(encoded)) + } +} + +func TestEncodePEM_ProducesValidPEMBlocks(t *testing.T) { + cab := &caBundle{} + if err := cab.getCertsFromPEM(validPEM); err != nil { + t.Fatalf("parse failed: %v", err) + } + + encoded := cab.encodePEM() + + // Verify each PEM block can be decoded and is of type CERTIFICATE. + rest := encoded + blockCount := 0 + for { + var block *pem.Block + block, rest = pem.Decode(rest) + if block == nil { + break + } + blockCount++ + if block.Type != "CERTIFICATE" { + t.Errorf("expected CERTIFICATE block type, got %q", block.Type) + } + } + if blockCount != 1 { + t.Errorf("expected 1 PEM block, got %d", blockCount) + } +} + +// --------------------------------------------------------------------------- +// encodePEM / getCertsFromPEM: raw bytes preservation +// --------------------------------------------------------------------------- + +func TestEncodePEM_PreservesRawCertBytes(t *testing.T) { + cab := &caBundle{} + if err := cab.getCertsFromPEM(validPEM); err != nil { + t.Fatalf("parse failed: %v", err) + } + + // Grab the raw DER bytes from the parsed cert. + originalRaw := cab.certs[0].cert.Raw + + encoded := cab.encodePEM() + block, _ := pem.Decode(encoded) + if block == nil { + t.Fatal("failed to decode PEM from encodePEM output") + } + + if !bytes.Equal(block.Bytes, originalRaw) { + t.Error("encoded DER bytes do not match original raw bytes") + } +} + +// --------------------------------------------------------------------------- +// caBundle hash field +// --------------------------------------------------------------------------- + +func TestCaBundleCert_HashMatchesDER(t *testing.T) { + cab := &caBundle{} + if err := cab.getCertsFromPEM(validPEM); err != nil { + t.Fatalf("parse failed: %v", err) + } + + block, _ := pem.Decode(validPEM) + if block == nil { + t.Fatal("failed to decode validPEM") + } + expectedHash := sha256.Sum256(block.Bytes) + + if cab.certs[0].hash != expectedHash { + t.Error("stored hash does not match SHA256 of DER bytes") + } +} + +// --------------------------------------------------------------------------- +// reconcileCABundleConfigMap tests +// --------------------------------------------------------------------------- + +func newTestHelper(t *testing.T, objs ...client.Object) *common_helper.Helper { + t.Helper() + + scheme := runtime.NewScheme() + if err := corev1.AddToScheme(scheme); err != nil { + t.Fatalf("failed to add corev1 to scheme: %v", err) + } + if err := apiv1beta1.AddToScheme(scheme); err != nil { + t.Fatalf("failed to add apiv1beta1 to scheme: %v", err) + } + + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(objs...). + Build() + + instance := &apiv1beta1.OpenStackLightspeed{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-instance", + Namespace: "test-ns", + }, + } + + logger := zap.New(zap.WriteTo(bytes.NewBuffer(nil))) + h, err := common_helper.NewHelper(instance, fakeClient, nil, scheme, logger) + if err != nil { + t.Fatalf("failed to create helper: %v", err) + } + return h +} + +func TestReconcileCABundle_MergesServiceCA(t *testing.T) { + original := getOperatorCABundle + t.Cleanup(func() { getOperatorCABundle = original }) + + now := time.Now() + systemCert := generateSelfSignedPEM(t, now.Add(-1*time.Hour), now.Add(10*365*24*time.Hour)) + serviceCACert := generateSelfSignedPEM(t, now.Add(-2*time.Hour), now.Add(10*365*24*time.Hour)) + kubeRootCACert := generateSelfSignedPEM(t, now.Add(-3*time.Hour), now.Add(10*365*24*time.Hour)) + + getOperatorCABundle = func() ([]byte, error) { + return systemCert, nil + } + + serviceCAConfigMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: OpenShiftServiceCAConfigMap, + Namespace: "test-ns", + }, + Data: map[string]string{ + "service-ca.crt": string(serviceCACert), + }, + } + kubeRootCAConfigMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: KubeRootCAConfigMap, + Namespace: "test-ns", + }, + Data: map[string]string{ + "ca.crt": string(kubeRootCACert), + }, + } + + h := newTestHelper(t, serviceCAConfigMap, kubeRootCAConfigMap) + + instance := &apiv1beta1.OpenStackLightspeed{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-instance", + Namespace: "test-ns", + }, + } + + ctx := context.Background() + err := reconcileCABundleConfigMap(h, ctx, instance) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + resultCM := &corev1.ConfigMap{} + err = h.GetClient().Get(ctx, types.NamespacedName{ + Name: CABundleConfigMapName, + Namespace: "test-ns", + }, resultCM) + if err != nil { + t.Fatalf("failed to get CA bundle ConfigMap: %v", err) + } + + bundleData := resultCM.Data[CABundleKey] + cab := &caBundle{} + if err := cab.getCertsFromPEM([]byte(bundleData)); err != nil { + t.Fatalf("failed to parse CA bundle: %v", err) + } + if len(cab.certs) != 3 { + t.Fatalf("expected 3 certs (system + service CA + kube root CA), got %d", len(cab.certs)) + } +} + +func TestReconcileCABundle_FailsWithoutServiceCA(t *testing.T) { + original := getOperatorCABundle + t.Cleanup(func() { getOperatorCABundle = original }) + + now := time.Now() + systemCert := generateSelfSignedPEM(t, now.Add(-1*time.Hour), now.Add(10*365*24*time.Hour)) + getOperatorCABundle = func() ([]byte, error) { + return systemCert, nil + } + + kubeRootCAConfigMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: KubeRootCAConfigMap, + Namespace: "test-ns", + }, + Data: map[string]string{ + "ca.crt": string(systemCert), + }, + } + + h := newTestHelper(t, kubeRootCAConfigMap) + + instance := &apiv1beta1.OpenStackLightspeed{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-instance", + Namespace: "test-ns", + }, + } + + err := reconcileCABundleConfigMap(h, context.Background(), instance) + if err == nil { + t.Fatal("expected error when service CA ConfigMap is missing, got nil") + } + if !strings.Contains(err.Error(), OpenShiftServiceCAConfigMap) { + t.Errorf("expected error to mention %s, got: %v", OpenShiftServiceCAConfigMap, err) + } +} + +func TestReconcileCABundle_FailsWithoutKubeRootCA(t *testing.T) { + original := getOperatorCABundle + t.Cleanup(func() { getOperatorCABundle = original }) + + now := time.Now() + systemCert := generateSelfSignedPEM(t, now.Add(-1*time.Hour), now.Add(10*365*24*time.Hour)) + getOperatorCABundle = func() ([]byte, error) { + return systemCert, nil + } + + serviceCAConfigMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: OpenShiftServiceCAConfigMap, + Namespace: "test-ns", + }, + Data: map[string]string{ + "service-ca.crt": string(systemCert), + }, + } + + h := newTestHelper(t, serviceCAConfigMap) + + instance := &apiv1beta1.OpenStackLightspeed{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-instance", + Namespace: "test-ns", + }, + } + + err := reconcileCABundleConfigMap(h, context.Background(), instance) + if err == nil { + t.Fatal("expected error when kube root CA ConfigMap is missing, got nil") + } + if !strings.Contains(err.Error(), KubeRootCAConfigMap) { + t.Errorf("expected error to mention %s, got: %v", KubeRootCAConfigMap, err) + } +} + +func TestReconcileCABundle_DeduplicatesAcrossSources(t *testing.T) { + original := getOperatorCABundle + t.Cleanup(func() { getOperatorCABundle = original }) + + now := time.Now() + sharedCert := generateSelfSignedPEM(t, now.Add(-1*time.Hour), now.Add(10*365*24*time.Hour)) + + getOperatorCABundle = func() ([]byte, error) { + return sharedCert, nil + } + + serviceCAConfigMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: OpenShiftServiceCAConfigMap, + Namespace: "test-ns", + }, + Data: map[string]string{ + "service-ca.crt": string(sharedCert), + }, + } + kubeRootCAConfigMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: KubeRootCAConfigMap, + Namespace: "test-ns", + }, + Data: map[string]string{ + "ca.crt": string(sharedCert), + }, + } + + h := newTestHelper(t, serviceCAConfigMap, kubeRootCAConfigMap) + + instance := &apiv1beta1.OpenStackLightspeed{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-instance", + Namespace: "test-ns", + }, + } + + ctx := context.Background() + err := reconcileCABundleConfigMap(h, ctx, instance) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + resultCM := &corev1.ConfigMap{} + err = h.GetClient().Get(ctx, types.NamespacedName{ + Name: CABundleConfigMapName, + Namespace: "test-ns", + }, resultCM) + if err != nil { + t.Fatalf("failed to get CA bundle ConfigMap: %v", err) + } + + bundleData := resultCM.Data[CABundleKey] + cab := &caBundle{} + if err := cab.getCertsFromPEM([]byte(bundleData)); err != nil { + t.Fatalf("failed to parse CA bundle: %v", err) + } + if len(cab.certs) != 1 { + t.Fatalf("expected 1 cert (same cert deduplicated across all sources), got %d", len(cab.certs)) + } +} diff --git a/internal/controller/common.go b/internal/controller/common.go index 183af712..b83e5cb4 100644 --- a/internal/controller/common.go +++ b/internal/controller/common.go @@ -89,41 +89,6 @@ func providerNameToEnvVarName(providerName string) string { return name } -// getPostgresCAConfigVolume returns a Volume for the Postgres CA certificate ConfigMap. -func getPostgresCAConfigVolume() corev1.Volume { - defaultMode := VolumeDefaultMode - return corev1.Volume{ - Name: PostgresCAVolume, - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: OpenStackLightspeedCAConfigMap, - }, - DefaultMode: &defaultMode, - }, - }, - } -} - -// getPostgresCAVolumeMount returns a VolumeMount for the Postgres CA certificate. -func getPostgresCAVolumeMount() corev1.VolumeMount { - return corev1.VolumeMount{ - Name: PostgresCAVolume, - MountPath: OpenStackLightspeedAppCertsMountRoot + "/postgres-ca", - ReadOnly: true, - } -} - -// getPostgresCAVolumeMountWithPath returns a VolumeMount for the Postgres CA certificate -// at the specified mount path. Used by the postgres container itself. -func getPostgresCAVolumeMountWithPath(mountPath string) corev1.VolumeMount { - return corev1.VolumeMount{ - Name: PostgresCAVolume, - MountPath: mountPath, - ReadOnly: true, - } -} - // generatePostgresSelectorLabels returns selector labels for Postgres components. func generatePostgresSelectorLabels() map[string]string { return map[string]string{ diff --git a/internal/controller/constants.go b/internal/controller/constants.go index 3ce9dd5f..6ef0a5a1 100644 --- a/internal/controller/constants.go +++ b/internal/controller/constants.go @@ -33,7 +33,6 @@ const ( OpenStackLightspeedAppServerServicePort = 8443 OpenStackLightspeedAppServerServiceName = "lightspeed-app-server" OpenStackLightspeedAppServerNetworkPolicyName = "lightspeed-app-server" - OpenStackLightspeedCertsSecretName = "lightspeed-tls" OpenStackLightspeedDefaultProvider = "openstack-lightspeed-provider" OpenStackLightspeedVectorDBPath = "/rag/vector_db/os_product_docs" @@ -43,26 +42,16 @@ const ( MetricsReaderServiceAccountTokenSecretName = "metrics-reader-token" MetricsReaderServiceAccountName = "lightspeed-operator-metrics-reader" - // Cert / CA - OpenStackLightspeedAppCertsMountRoot = "/etc/certs" - OpenStackLightspeedCAConfigMap = "openshift-service-ca.crt" - OpenShiftCAVolumeName = "openshift-ca" - AdditionalCAVolumeName = "additional-ca" - AdditionalCACertFile = "cert.crt" - // Postgres - PostgresCAVolume = "cm-olspostgresca" PostgresDeploymentName = "lightspeed-postgres-server" PostgresServiceName = "lightspeed-postgres-server" PostgresSecretName = "lightspeed-postgres-secret" - PostgresCertsSecretName = "lightspeed-postgres-certs" PostgresBootstrapSecretName = "lightspeed-postgres-bootstrap" PostgresConfigMapName = "lightspeed-postgres-conf" PostgresNetworkPolicyName = "lightspeed-postgres-server" PostgresServicePort = int32(5432) PostgresDefaultUser = "postgres" PostgresDefaultDbName = "postgres" - PostgresDefaultSSLMode = "require" PostgresSharedBuffers = "256MB" PostgresMaxConnections = 100 OpenStackLightspeedComponentPasswordFileName = "password" @@ -190,12 +179,79 @@ const ( VectorDBScriptsConfigMapVersionAnnotation = "ols.openshift.io/vector-db-scripts-configmap-version" LlamaStackConfigMapResourceVersionAnnotation = "ols.openshift.io/llamastack-configmap-version" LCoreConfigMapResourceVersionAnnotation = "ols.openshift.io/lcore-configmap-version" + CABundleConfigMapVersionAnnotation = "ols.openshift.io/ca-bundle-configmap-version" // Volume Permissions // These constants define file permissions for volumes mounted in containers. VolumeDefaultMode = int32(420) VolumeRestrictedMode = int32(0600) VolumeExecutableMode = int32(0755) + + // CABundleConfigMapName is the name of the ConfigMap that stores the + // CA certificate bundle. It aggregates certificates from three sources — + // operator system CAs, the OpenShift service serving CA (for in-cluster + // service-to-service TLS), and the OpenShift API server CA — along with + // any user-provided additional CAs. + CABundleConfigMapName = "openstack-lightspeed-ca-bundle" + + // CABundleKey is the key within the CA bundle ConfigMap under which + // the PEM-encoded certificate data is stored. + CABundleKey = "tls-ca-bundle.pem" + + // CABundleVolumeName is the name of the volume used to mount the + // CA bundle ConfigMap into containers. + CABundleVolumeName = "ca-bundle" + + // CABundleMountPath is the filesystem path where the CA bundle is + // mounted inside application containers. + CABundleMountPath = "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem" + + // SystemTLSCABundlePath is the path to the system-wide CA certificate bundle + // on the operator pod's filesystem. Used to read trusted root certificates + // when building the CA bundle. + SystemTLSCABundlePath = "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem" + + // KubeRootCAConfigMap is the name of the ConfigMap auto-created by + // kube-controller-manager in every namespace, containing the CA certificate + // that signs the API server's serving certificate. Read during CA + // bundle reconciliation and merged into the bundle. + KubeRootCAConfigMap = "kube-root-ca.crt" + + // OpenStackLightspeedTLSCertPath is the path to the TLS certificate file + // inside the lightspeed-service-api container, used to serve HTTPS. + OpenStackLightspeedTLSCertPath = OpenStackLightspeedAppCertsMountRoot + "/lightspeed-tls/tls.crt" + + // OpenStackLightspeedTLSKeyPath is the path to the TLS private key file + // inside the lightspeed-service-api container, used to serve HTTPS. + OpenStackLightspeedTLSKeyPath = OpenStackLightspeedAppCertsMountRoot + "/lightspeed-tls/tls.key" + + // OpenStackLightspeedCertsSecretName is the name of the Secret auto-provisioned + // by the OpenShift service-ca operator when the lightspeed-app-server Service is + // annotated with service.beta.openshift.io/serving-cert-secret-name. Contains + // tls.crt and tls.key used by the lightspeed-service-api container to serve HTTPS. + OpenStackLightspeedCertsSecretName = "lightspeed-tls" + + // PostgresCertsSecretName is the name of the Secret auto-provisioned by the + // OpenShift service-ca operator when the lightspeed-postgres-server Service is + // annotated with service.beta.openshift.io/serving-cert-secret-name. Contains + // tls.crt and tls.key used by the postgres container to serve TLS connections. + PostgresCertsSecretName = "lightspeed-postgres-certs" + + // PostgresDefaultSSLMode is the sslmode used when connecting to PostgreSQL. + // "verify-full" requires a valid server certificate and checks + // that the server hostname matches the certificate CN/SAN, ensuring both + // encryption and authentication of the database connection. + PostgresDefaultSSLMode = "verify-full" + + // OpenStackLightspeedAppCertsMountRoot is the base directory under which + // all application certificate volumes are mounted inside application containers. + OpenStackLightspeedAppCertsMountRoot = "/etc/certs" + + // OpenShiftServiceCAConfigMap is the name of the ConfigMap containing the + // OpenShift service serving CA certificate (public part only). This is the CA + // that signs TLS certificates auto-provisioned for Services via the + // service.beta.openshift.io/serving-cert-secret-name annotation. + OpenShiftServiceCAConfigMap = "openshift-service-ca.crt" ) // PostgreSQL Bootstrap Script - creates database, extensions, and schemas diff --git a/internal/controller/errors.go b/internal/controller/errors.go index 630468c5..c3415ab7 100644 --- a/internal/controller/errors.go +++ b/internal/controller/errors.go @@ -30,12 +30,15 @@ var ( ErrDeleteSARClusterRole = errors.New("failed to delete SAR cluster role") ErrDeleteSARClusterRoleBinding = errors.New("failed to delete SAR cluster role binding") ErrGenerateAPIConfigmap = errors.New("failed to generate OpenStack Lightspeed configmap") - ErrGetAdditionalCACM = errors.New("failed to get additional CA configmap") - ErrGetProxyCACM = errors.New("failed to get proxy CA configmap") ErrGetTLSSecret = errors.New("failed to get TLS secret") ErrCreateLlamaStackConfigMap = errors.New("failed to create Llama Stack configmap") ErrGenerateLlamaStackConfigMap = errors.New("failed to generate Llama Stack configmap") ErrCreateExporterConfigMap = errors.New("failed to create exporter configmap") + ErrReadSystemCABundle = errors.New("failed to read system CA bundle") + ErrParseSystemCABundle = errors.New("failed to parse system CA bundle") + ErrParseUserCA = errors.New("failed to parse user CA certificate") + ErrCreateCABundle = errors.New("failed to create CA bundle configmap") + ErrGetCAConfigMap = errors.New("failed to get CA configmap") // Console Plugin Errors ErrReconcileConsolePlugin = errors.New("failed to reconcile console plugin") diff --git a/internal/controller/lcore_config.go b/internal/controller/lcore_config.go index dd0238d9..fb63b99d 100644 --- a/internal/controller/lcore_config.go +++ b/internal/controller/lcore_config.go @@ -83,8 +83,8 @@ func buildLCoreServiceConfig(_ *common_helper.Helper, _ *apiv1beta1.OpenStackLig "color_log": false, "access_log": true, "tls_config": map[string]interface{}{ - "tls_certificate_path": "/etc/certs/lightspeed-tls/tls.crt", - "tls_key_path": "/etc/certs/lightspeed-tls/tls.key", + "tls_certificate_path": OpenStackLightspeedTLSCertPath, + "tls_key_path": OpenStackLightspeedTLSKeyPath, }, } } @@ -133,7 +133,7 @@ func buildLCoreDatabaseConfig(h *common_helper.Helper, _ *apiv1beta1.OpenStackLi "user": PostgresDefaultUser, "ssl_mode": PostgresDefaultSSLMode, "gss_encmode": "disable", - "ca_cert_path": "/etc/certs/postgres-ca/service-ca.crt", + "ca_cert_path": CABundleMountPath, // Environment variable substitution via llama_stack.core.stack.replace_env_vars "password": "${env.POSTGRES_PASSWORD}", @@ -166,7 +166,7 @@ func buildLCoreConversationCacheConfig(h *common_helper.Helper, _ *apiv1beta1.Op "password": "${env.POSTGRES_PASSWORD}", "ssl_mode": PostgresDefaultSSLMode, "gss_encmode": "disable", - "ca_cert_path": "/etc/certs/postgres-ca/service-ca.crt", + "ca_cert_path": CABundleMountPath, "namespace": "conversation_cache", }, } diff --git a/internal/controller/lcore_deployment.go b/internal/controller/lcore_deployment.go index d5b7773f..3720f736 100644 --- a/internal/controller/lcore_deployment.go +++ b/internal/controller/lcore_deployment.go @@ -44,12 +44,9 @@ func buildLCorePodTemplateSpec(h *common_helper.Helper, ctx context.Context, ins buildVectorDBScriptsVolume(), } - // Shared volumes - CA, postgres + // Shared volumes - CA bundle covers all cluster CAs sharedMounts := []corev1.VolumeMount{} - addOpenShiftCAVolumesAndMounts(&volumes, &sharedMounts, VolumeDefaultMode) - addOpenShiftRootCAVolumesAndMounts(&volumes, &sharedMounts, VolumeDefaultMode) - addPostgresCAVolumesAndMounts(&volumes, &sharedMounts) - addUserCAVolumesAndMounts(&volumes, &sharedMounts, instance, VolumeDefaultMode) + addCABundleVolumesAndMounts(&volumes, &sharedMounts) addVectorDBDataVolumesAndMounts(&volumes, &sharedMounts) // Llama cache emptydir @@ -146,6 +143,12 @@ func buildLCorePodTemplateSpec(h *common_helper.Helper, ctx context.Context, ins MountPath: ExporterConfigMountPath, ReadOnly: true, }, + { + Name: CABundleVolumeName, + MountPath: CABundleMountPath, + SubPath: CABundleKey, + ReadOnly: true, + }, }, Resources: corev1.ResourceRequirements{ Requests: corev1.ResourceList{ @@ -359,52 +362,6 @@ func addTLSVolumesAndMounts(volumes *[]corev1.Volume, mounts *[]corev1.VolumeMou }) } -// addOpenShiftCAVolumesAndMounts adds the OpenShift service-ca CA bundle volume and mount. -func addOpenShiftCAVolumesAndMounts(volumes *[]corev1.Volume, mounts *[]corev1.VolumeMount, volumeDefaultMode int32) { - *volumes = append(*volumes, corev1.Volume{ - Name: OpenShiftCAVolumeName, - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: OpenStackLightspeedCAConfigMap, - }, - DefaultMode: toPtr(volumeDefaultMode), - }, - }, - }) - *mounts = append(*mounts, corev1.VolumeMount{ - Name: OpenShiftCAVolumeName, - MountPath: OpenStackLightspeedAppCertsMountRoot + "/openshift-ca", - ReadOnly: true, - }) -} - -// addOpenShiftRootCAVolumesAndMounts adds the OpenShift cluster-wide root CA bundle. -func addOpenShiftRootCAVolumesAndMounts(volumes *[]corev1.Volume, mounts *[]corev1.VolumeMount, volumeDefaultMode int32) { - *volumes = append(*volumes, corev1.Volume{ - Name: "openshift-root-ca", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "kube-root-ca.crt", - }, - DefaultMode: toPtr(volumeDefaultMode), - }, - }, - }) - *mounts = append(*mounts, corev1.VolumeMount{ - Name: "openshift-root-ca", - MountPath: OpenStackLightspeedAppCertsMountRoot + "/openshift-root-ca", - ReadOnly: true, - }) -} - -// addPostgresCAVolumesAndMounts adds the Postgres CA certificate volume and mount. -func addPostgresCAVolumesAndMounts(volumes *[]corev1.Volume, mounts *[]corev1.VolumeMount) { - *volumes = append(*volumes, getPostgresCAConfigVolume()) - *mounts = append(*mounts, getPostgresCAVolumeMount()) -} - // addLlamaCacheVolumesAndMounts adds an emptydir volume for llama-stack cache. func addLlamaCacheVolumesAndMounts(volumes *[]corev1.Volume, mounts *[]corev1.VolumeMount) { *volumes = append(*volumes, corev1.Volume{ @@ -441,49 +398,29 @@ func addDataCollectorVolumes(volumes *[]corev1.Volume, volumeDefaultMode int32) }) } -// addUserCAVolumesAndMounts adds user-provided additional CA certificate volume and mount -// if instance.Spec.TLSCACertBundle is set. -func addUserCAVolumesAndMounts(volumes *[]corev1.Volume, mounts *[]corev1.VolumeMount, instance *apiv1beta1.OpenStackLightspeed, volumeDefaultMode int32) { - if instance.Spec.TLSCACertBundle == "" { - return - } +// addCABundleVolumesAndMounts adds the CA bundle volume and mount. +// The CA bundle is always present (created by reconcileCABundleConfigMap) +// and mounted at the RHEL system CA path so applications find it automatically. +func addCABundleVolumesAndMounts(volumes *[]corev1.Volume, mounts *[]corev1.VolumeMount) { *volumes = append(*volumes, corev1.Volume{ - Name: AdditionalCAVolumeName, + Name: CABundleVolumeName, VolumeSource: corev1.VolumeSource{ ConfigMap: &corev1.ConfigMapVolumeSource{ LocalObjectReference: corev1.LocalObjectReference{ - Name: instance.Spec.TLSCACertBundle, + Name: CABundleConfigMapName, }, - DefaultMode: toPtr(volumeDefaultMode), + DefaultMode: toPtr(VolumeDefaultMode), }, }, }) *mounts = append(*mounts, corev1.VolumeMount{ - Name: AdditionalCAVolumeName, - MountPath: OpenStackLightspeedAppCertsMountRoot + "/additional-ca", + Name: CABundleVolumeName, + MountPath: CABundleMountPath, + SubPath: CABundleKey, ReadOnly: true, }) } -// buildAdditionalCAEnvVars returns REQUESTS_CA_BUNDLE and SSL_CERT_FILE env vars -// pointing to the additional CA cert file, if an additional CA configmap is configured. -func buildAdditionalCAEnvVars(instance *apiv1beta1.OpenStackLightspeed) []corev1.EnvVar { - if instance.Spec.TLSCACertBundle == "" { - return nil - } - certPath := OpenStackLightspeedAppCertsMountRoot + "/additional-ca/" + AdditionalCACertFile - return []corev1.EnvVar{ - { - Name: "REQUESTS_CA_BUNDLE", - Value: certPath, - }, - { - Name: "SSL_CERT_FILE", - Value: certPath, - }, - } -} - // buildLlamaStackEnvVars builds environment variables for llama-stack, // primarily provider API keys read from Kubernetes secrets. func buildLlamaStackEnvVars(h *common_helper.Helper, ctx context.Context, instance *apiv1beta1.OpenStackLightspeed) ([]corev1.EnvVar, error) { @@ -604,9 +541,6 @@ func buildLlamaStackEnvVars(h *common_helper.Helper, ctx context.Context, instan Value: VectorDBVolumeMountPath, }) - // Additional CA env vars - envVars = append(envVars, buildAdditionalCAEnvVars(instance)...) - return envVars, nil } @@ -721,5 +655,14 @@ func buildConfigMapAnnotations(h *common_helper.Helper, ctx context.Context) (ma annotations[VectorDBScriptsConfigMapVersionAnnotation] = vectorDBScriptsVersion } + caBundleVersion, err := getConfigMapResourceVersion(ctx, h, CABundleConfigMapName, h.GetBeforeObject().GetNamespace()) + if err != nil { + if !errors.IsNotFound(err) { + return nil, fmt.Errorf("failed to get CA bundle configmap resource version: %w", err) + } + } else { + annotations[CABundleConfigMapVersionAnnotation] = caBundleVersion + } + return annotations, nil } diff --git a/internal/controller/lcore_reconciler.go b/internal/controller/lcore_reconciler.go index ed3c6fa5..71cc8197 100644 --- a/internal/controller/lcore_reconciler.go +++ b/internal/controller/lcore_reconciler.go @@ -48,7 +48,7 @@ func ReconcileLCoreResources(h *common_helper.Helper, ctx context.Context, insta {Name: "LcoreConfigMap", Task: reconcileLcoreConfigMap}, {Name: "ExporterConfigMap", Task: reconcileExporterConfigMap}, {Name: "VectorDBScriptsConfigMap", Task: reconcileVectorDBScriptsConfigMap}, - {Name: "OpenStackLightspeedAdditionalCAConfigMap", Task: reconcileOpenStackLightspeedAdditionalCAConfigMap}, + {Name: "CABundle", Task: reconcileCABundleConfigMap}, {Name: "ProxyCAConfigMap", Task: reconcileProxyCAConfigMap}, {Name: "NetworkPolicy", Task: reconcileNetworkPolicy}, } @@ -310,29 +310,6 @@ func reconcileVectorDBScriptsConfigMap(h *common_helper.Helper, ctx context.Cont return nil } -// reconcileOpenStackLightspeedAdditionalCAConfigMap verifies that the additional CA config map -// exists if one is specified in the configuration. -func reconcileOpenStackLightspeedAdditionalCAConfigMap(h *common_helper.Helper, ctx context.Context, instance *apiv1beta1.OpenStackLightspeed) error { - logger := h.GetLogger() - - if instance.Spec.TLSCACertBundle == "" { - logger.Info("no additional CA configmap configured, skipping") - return nil - } - - existing := &corev1.ConfigMap{} - err := h.GetClient().Get(ctx, client.ObjectKey{ - Name: instance.Spec.TLSCACertBundle, - Namespace: h.GetBeforeObject().GetNamespace(), - }, existing) - if err != nil { - return fmt.Errorf("%w %q: %v", ErrGetAdditionalCACM, instance.Spec.TLSCACertBundle, err) - } - - logger.Info("additional CA configmap found", "name", instance.Spec.TLSCACertBundle) - return nil -} - // reconcileProxyCAConfigMap is a no-op for the minimal mapping (no proxy config). func reconcileProxyCAConfigMap(h *common_helper.Helper, _ context.Context, _ *apiv1beta1.OpenStackLightspeed) error { logger := h.GetLogger() diff --git a/internal/controller/llama_stack_config.go b/internal/controller/llama_stack_config.go index d2642a8a..d353ddc1 100644 --- a/internal/controller/llama_stack_config.go +++ b/internal/controller/llama_stack_config.go @@ -265,9 +265,6 @@ func buildLlamaStackStorage(_ *common_helper.Helper, instance *apiv1beta1.OpenSt "password": "${env.POSTGRES_PASSWORD}", // Note: Database name is HARDCODED to "llamastack" in llama-stack's postgres adapter // Not configurable - llama-stack ignores image_name for database selection - "ssl_mode": "require", - "ca_cert_path": "/etc/certs/postgres-ca/service-ca.crt", - "gss_encmode": "disable", }, } diff --git a/internal/controller/openstacklightspeed_controller.go b/internal/controller/openstacklightspeed_controller.go index b369c3a0..95516745 100644 --- a/internal/controller/openstacklightspeed_controller.go +++ b/internal/controller/openstacklightspeed_controller.go @@ -295,9 +295,36 @@ func (r *OpenStackLightspeedReconciler) SetupWithManager(mgr ctrl.Manager) error handler.EnqueueRequestsFromMapFunc(r.NotifyAllOpenStackLightspeeds), builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}), ). + Watches( + &corev1.ConfigMap{}, + handler.EnqueueRequestsFromMapFunc(r.NotifyOpenStackLightspeedsByCAConfigMap), + builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}), + ). Complete(r) } +// NotifyOpenStackLightspeedsByCAConfigMap watches ConfigMaps and triggers reconciliation when +// a user-provided CA ConfigMap (referenced by an OpenStackLightspeed CR) changes. +func (r *OpenStackLightspeedReconciler) NotifyOpenStackLightspeedsByCAConfigMap(ctx context.Context, obj client.Object) []ctrl.Request { + var lightspeedList apiv1beta1.OpenStackLightspeedList + if err := r.List(ctx, &lightspeedList, client.InNamespace(obj.GetNamespace())); err != nil { + return nil + } + + var requests []ctrl.Request + for _, item := range lightspeedList.Items { + if item.Spec.TLSCACertBundle == obj.GetName() { + requests = append(requests, ctrl.Request{ + NamespacedName: client.ObjectKey{ + Namespace: item.GetNamespace(), + Name: item.GetName(), + }, + }) + } + } + return requests +} + // NotifyAllOpenStackLightspeeds returns a list of reconcile requests for all OpenStackLightspeed objects. // For namespace-scoped resources (like InstallPlan), it lists in the same namespace as the triggering object. // For cluster-scoped resources (like ClusterVersion), it lists in all namespaces the operator can access. diff --git a/internal/controller/postgres_deployment.go b/internal/controller/postgres_deployment.go index 3988d0e9..4d365207 100644 --- a/internal/controller/postgres_deployment.go +++ b/internal/controller/postgres_deployment.go @@ -17,7 +17,6 @@ limitations under the License. package controller import ( - "path" "strconv" apiv1beta1 "github.com/openstack-lightspeed/operator/api/v1beta1" @@ -97,10 +96,6 @@ func buildPostgresPodTemplateSpec() corev1.PodTemplateSpec { MountPath: PostgresDataVolumeMountPath, }) - // Postgres CA volume - volumes = append(volumes, getPostgresCAConfigVolume()) - volumeMounts = append(volumeMounts, getPostgresCAVolumeMountWithPath(path.Join(OpenStackLightspeedAppCertsMountRoot, PostgresCAVolume))) - // Var run volume (writable runtime directory) volumes = append(volumes, corev1.Volume{ Name: PostgresVarRunVolumeName, diff --git a/test/kuttl/common/expected-configs/lightspeed-stack-update.yaml b/test/kuttl/common/expected-configs/lightspeed-stack-update.yaml index a509126d..89d6e040 100644 --- a/test/kuttl/common/expected-configs/lightspeed-stack-update.yaml +++ b/test/kuttl/common/expected-configs/lightspeed-stack-update.yaml @@ -7,14 +7,14 @@ byok_rag: db_path: NONE conversation_cache: postgres: - ca_cert_path: /etc/certs/postgres-ca/service-ca.crt + ca_cert_path: /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem db: postgres gss_encmode: disable host: lightspeed-postgres-server.openstack-lightspeed.svc namespace: conversation_cache password: ${env.POSTGRES_PASSWORD} port: 5432 - ssl_mode: require + ssl_mode: verify-full user: postgres type: postgres customization: @@ -55,14 +55,14 @@ customization: \ (which run RHEL).\n" database: postgres: - ca_cert_path: /etc/certs/postgres-ca/service-ca.crt + ca_cert_path: /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem db: postgres gss_encmode: disable host: lightspeed-postgres-server.openstack-lightspeed.svc namespace: lcore password: ${env.POSTGRES_PASSWORD} port: 5432 - ssl_mode: require + ssl_mode: verify-full user: postgres inference: default_model: ibm-granite/granite-3.1-8b-instruct-UPDATE diff --git a/test/kuttl/common/expected-configs/lightspeed-stack.yaml b/test/kuttl/common/expected-configs/lightspeed-stack.yaml index e9c35b72..f0ebae5d 100644 --- a/test/kuttl/common/expected-configs/lightspeed-stack.yaml +++ b/test/kuttl/common/expected-configs/lightspeed-stack.yaml @@ -7,14 +7,14 @@ byok_rag: db_path: NONE conversation_cache: postgres: - ca_cert_path: /etc/certs/postgres-ca/service-ca.crt + ca_cert_path: /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem db: postgres gss_encmode: disable host: lightspeed-postgres-server.openstack-lightspeed.svc namespace: conversation_cache password: ${env.POSTGRES_PASSWORD} port: 5432 - ssl_mode: require + ssl_mode: verify-full user: postgres type: postgres customization: @@ -55,14 +55,14 @@ customization: \ (which run RHEL).\n" database: postgres: - ca_cert_path: /etc/certs/postgres-ca/service-ca.crt + ca_cert_path: /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem db: postgres gss_encmode: disable host: lightspeed-postgres-server.openstack-lightspeed.svc namespace: lcore password: ${env.POSTGRES_PASSWORD} port: 5432 - ssl_mode: require + ssl_mode: verify-full user: postgres inference: default_model: ibm-granite/granite-3.1-8b-instruct diff --git a/test/kuttl/common/expected-configs/ogx_config-update.yaml b/test/kuttl/common/expected-configs/ogx_config-update.yaml index 68f4a8a4..6e9b0616 100644 --- a/test/kuttl/common/expected-configs/ogx_config-update.yaml +++ b/test/kuttl/common/expected-configs/ogx_config-update.yaml @@ -115,12 +115,9 @@ storage: db_path: /tmp/llama-stack/kv_store.db type: kv_sqlite postgres_backend: - ca_cert_path: /etc/certs/postgres-ca/service-ca.crt - gss_encmode: disable host: lightspeed-postgres-server.openstack-lightspeed.svc password: ${env.POSTGRES_PASSWORD} port: 5432 - ssl_mode: require type: sql_postgres user: postgres sql_default: diff --git a/test/kuttl/common/expected-configs/ogx_config.yaml b/test/kuttl/common/expected-configs/ogx_config.yaml index 9520aa0e..de0f460a 100644 --- a/test/kuttl/common/expected-configs/ogx_config.yaml +++ b/test/kuttl/common/expected-configs/ogx_config.yaml @@ -115,12 +115,9 @@ storage: db_path: /tmp/llama-stack/kv_store.db type: kv_sqlite postgres_backend: - ca_cert_path: /etc/certs/postgres-ca/service-ca.crt - gss_encmode: disable host: lightspeed-postgres-server.openstack-lightspeed.svc password: ${env.POSTGRES_PASSWORD} port: 5432 - ssl_mode: require type: sql_postgres user: postgres sql_default: diff --git a/test/kuttl/common/openstack-lightspeed-instance/assert-openstack-lightspeed-instance.yaml b/test/kuttl/common/openstack-lightspeed-instance/assert-openstack-lightspeed-instance.yaml index 4356761a..d0fd3ff8 100644 --- a/test/kuttl/common/openstack-lightspeed-instance/assert-openstack-lightspeed-instance.yaml +++ b/test/kuttl/common/openstack-lightspeed-instance/assert-openstack-lightspeed-instance.yaml @@ -135,6 +135,12 @@ metadata: namespace: openstack-lightspeed --- apiVersion: v1 +kind: ConfigMap +metadata: + name: openstack-lightspeed-ca-bundle + namespace: openstack-lightspeed +--- +apiVersion: v1 kind: Service metadata: name: lightspeed-app-server @@ -148,48 +154,103 @@ metadata: spec: template: spec: + initContainers: + - name: vector-database-collect + - name: vector-database-config-build containers: - - name: llama-stack - env: - - name: OPENSTACK_LIGHTSPEED_PROVIDER_API_KEY - valueFrom: - secretKeyRef: - key: apitoken - name: openstack-lightspeed-apitoken - - name: POSTGRES_PASSWORD - valueFrom: - secretKeyRef: - key: password - name: lightspeed-postgres-secret - - name: LLAMA_STACK_LOGGING - value: all=debug - - name: OGX_LOGGING - value: all=debug - - name: VECTOR_DB_DATA_PATH - value: /vector-db-discovered-values - - name: REQUESTS_CA_BUNDLE - value: /etc/certs/additional-ca/cert.crt - - name: SSL_CERT_FILE - value: /etc/certs/additional-ca/cert.crt - - name: lightspeed-service-api - env: - - name: LIGHTSPEED_STACK_LOG_LEVEL - value: WARNING - - name: POSTGRES_PASSWORD - valueFrom: - secretKeyRef: - key: password - name: lightspeed-postgres-secret - - name: lightspeed-to-dataverse-exporter - args: - - --mode - - openshift - - --config - - /etc/config/config.yaml - - --log-level - - DEBUG - - --data-dir - - /tmp/data + - name: llama-stack + env: + - name: OPENSTACK_LIGHTSPEED_PROVIDER_API_KEY + valueFrom: + secretKeyRef: + key: apitoken + name: openstack-lightspeed-apitoken + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + key: password + name: lightspeed-postgres-secret + - name: LLAMA_STACK_LOGGING + value: all=debug + - name: OGX_LOGGING + value: all=debug + - name: VECTOR_DB_DATA_PATH + value: /vector-db-discovered-values + volumeMounts: + - name: ca-bundle + mountPath: /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem + subPath: tls-ca-bundle.pem + readOnly: true + - name: vector-db-discovered-values + mountPath: /vector-db-discovered-values + - name: llama-cache + mountPath: /tmp/llama-stack + - name: lightspeed-service-api + env: + - name: LIGHTSPEED_STACK_LOG_LEVEL + value: WARNING + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + key: password + name: lightspeed-postgres-secret + volumeMounts: + - name: ca-bundle + mountPath: /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem + subPath: tls-ca-bundle.pem + readOnly: true + - name: vector-db-discovered-values + mountPath: /vector-db-discovered-values + - name: tls-certs + mountPath: /etc/certs/lightspeed-tls + readOnly: true + - name: ols-user-data + mountPath: /tmp/data + - name: lightspeed-to-dataverse-exporter + args: + - --mode + - openshift + - --config + - /etc/config/config.yaml + - --log-level + - DEBUG + - --data-dir + - /tmp/data + volumeMounts: + - name: ols-user-data + mountPath: /tmp/data + - name: exporter-config + mountPath: /etc/config + readOnly: true + - name: ca-bundle + mountPath: /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem + subPath: tls-ca-bundle.pem + readOnly: true + volumes: + - name: ogx-config + configMap: + name: llama-stack-config + - name: lightspeed-stack-config + configMap: + name: lightspeed-stack-config + - name: vector-db-scripts + configMap: + name: vector-db-scripts + - name: ca-bundle + configMap: + name: openstack-lightspeed-ca-bundle + - name: vector-db-discovered-values + emptyDir: {} + - name: llama-cache + emptyDir: {} + - name: ols-user-data + emptyDir: {} + - name: exporter-config + configMap: + name: lightspeed-exporter-config + - name: tls-certs + secret: + secretName: lightspeed-tls status: replicas: 1 readyReplicas: 1 diff --git a/test/kuttl/common/openstack-lightspeed-instance/errors-openstack-lightspeed-instance.yaml b/test/kuttl/common/openstack-lightspeed-instance/errors-openstack-lightspeed-instance.yaml index a33cf122..e042da71 100644 --- a/test/kuttl/common/openstack-lightspeed-instance/errors-openstack-lightspeed-instance.yaml +++ b/test/kuttl/common/openstack-lightspeed-instance/errors-openstack-lightspeed-instance.yaml @@ -79,6 +79,14 @@ kind: ConsolePlugin metadata: name: lightspeed-console-plugin +# CA bundle ConfigMap should be gone +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: openstack-lightspeed-ca-bundle + namespace: openstack-lightspeed + # Postgres resources should be gone --- apiVersion: apps/v1 diff --git a/test/kuttl/tests/update-openstacklightspeed/08-assert-openstacklightspeed-update.yaml b/test/kuttl/tests/update-openstacklightspeed/08-assert-openstacklightspeed-update.yaml index 1a6add7e..2e7a821a 100644 --- a/test/kuttl/tests/update-openstacklightspeed/08-assert-openstacklightspeed-update.yaml +++ b/test/kuttl/tests/update-openstacklightspeed/08-assert-openstacklightspeed-update.yaml @@ -69,40 +69,78 @@ metadata: spec: template: spec: + initContainers: + - name: vector-database-collect + - name: vector-database-config-build containers: - - name: llama-stack - env: - - name: OPENSTACK_LIGHTSPEED_PROVIDER_API_KEY - valueFrom: - secretKeyRef: - key: apitoken - name: openstack-lightspeed-apitoken-update - - name: VLLM_URL - value: http://mock-llm-api-server-pod-UPDATE:8000/v1 - - name: POSTGRES_PASSWORD - valueFrom: - secretKeyRef: - key: password - name: lightspeed-postgres-secret - - name: LLAMA_STACK_LOGGING - value: core=debug,providers=info - - name: OGX_LOGGING - value: core=debug,providers=info - - name: VECTOR_DB_DATA_PATH - value: /vector-db-discovered-values - - name: REQUESTS_CA_BUNDLE - value: /etc/certs/additional-ca/cert.crt - - name: SSL_CERT_FILE - value: /etc/certs/additional-ca/cert.crt - - name: lightspeed-service-api - env: - - name: LIGHTSPEED_STACK_LOG_LEVEL - value: ERROR - - name: POSTGRES_PASSWORD - valueFrom: - secretKeyRef: - key: password - name: lightspeed-postgres-secret + - name: llama-stack + env: + - name: OPENSTACK_LIGHTSPEED_PROVIDER_API_KEY + valueFrom: + secretKeyRef: + key: apitoken + name: openstack-lightspeed-apitoken-update + - name: VLLM_URL + value: http://mock-llm-api-server-pod-UPDATE:8000/v1 + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + key: password + name: lightspeed-postgres-secret + - name: LLAMA_STACK_LOGGING + value: core=debug,providers=info + - name: OGX_LOGGING + value: core=debug,providers=info + - name: VECTOR_DB_DATA_PATH + value: /vector-db-discovered-values + volumeMounts: + - name: ca-bundle + mountPath: /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem + subPath: tls-ca-bundle.pem + readOnly: true + - name: vector-db-discovered-values + mountPath: /vector-db-discovered-values + - name: llama-cache + mountPath: /tmp/llama-stack + - name: lightspeed-service-api + env: + - name: LIGHTSPEED_STACK_LOG_LEVEL + value: ERROR + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + key: password + name: lightspeed-postgres-secret + volumeMounts: + - name: ca-bundle + mountPath: /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem + subPath: tls-ca-bundle.pem + readOnly: true + - name: vector-db-discovered-values + mountPath: /vector-db-discovered-values + - name: tls-certs + mountPath: /etc/certs/lightspeed-tls + readOnly: true + volumes: + - name: ogx-config + configMap: + name: llama-stack-config + - name: lightspeed-stack-config + configMap: + name: lightspeed-stack-config + - name: vector-db-scripts + configMap: + name: vector-db-scripts + - name: ca-bundle + configMap: + name: openstack-lightspeed-ca-bundle + - name: vector-db-discovered-values + emptyDir: {} + - name: llama-cache + emptyDir: {} + - name: tls-certs + secret: + secretName: lightspeed-tls status: replicas: 1 readyReplicas: 1 @@ -115,6 +153,14 @@ metadata: name: lightspeed-app-server namespace: openstack-lightspeed +# CA bundle ConfigMap +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: openstack-lightspeed-ca-bundle + namespace: openstack-lightspeed + # Vector DB init containers scripts --- apiVersion: v1 diff --git a/test/kuttl/tests/update-openstacklightspeed/11-assert-configmaps-update.yaml b/test/kuttl/tests/update-openstacklightspeed/11-assert-configmaps-update.yaml index 7749f1d0..e028d6b0 100644 --- a/test/kuttl/tests/update-openstacklightspeed/11-assert-configmaps-update.yaml +++ b/test/kuttl/tests/update-openstacklightspeed/11-assert-configmaps-update.yaml @@ -16,6 +16,14 @@ metadata: name: lightspeed-stack-config namespace: openstack-lightspeed +# Verify CA bundle ConfigMap still exists after update +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: openstack-lightspeed-ca-bundle + namespace: openstack-lightspeed + # Verify all operator-managed resources still exist after update --- apiVersion: apps/v1