diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index ecf9372e..a481d72e 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -7,6 +7,7 @@ rules: - apiGroups: - "" resources: + - configmaps - endpoints - namespaces - pods diff --git a/docs/en/latest/concepts/gateway-api.md b/docs/en/latest/concepts/gateway-api.md index dd77dd35..99db95d0 100644 --- a/docs/en/latest/concepts/gateway-api.md +++ b/docs/en/latest/concepts/gateway-api.md @@ -82,4 +82,5 @@ The fields below are specified in the Gateway API specification but are either p | `spec.listeners[].tls.certificateRefs[].group` | Partially supported | Only `""` is supported; other group values cause validation failure. | | `spec.listeners[].tls.certificateRefs[].kind` | Partially supported | Only `Secret` is supported. | | `spec.listeners[].tls.mode` | Partially supported | `Terminate` is implemented; `Passthrough` is effectively unsupported for Gateway listeners. | +| `spec.listeners[].tls.frontendValidation` | Partially supported | Enables downstream (client) mTLS. `caCertificateRefs` may reference a `ConfigMap` (Gateway API Core support) or a `Secret` (implementation-specific) holding the CA certificate under the `ca.crt` key; clients are then required to present a certificate signed by one of the referenced CAs. | | `spec.addresses` | Not supported | Controller does not read or act on `spec.addresses`. | diff --git a/internal/adc/translator/gateway.go b/internal/adc/translator/gateway.go index 53c67144..a5af3738 100644 --- a/internal/adc/translator/gateway.go +++ b/internal/adc/translator/gateway.go @@ -20,8 +20,10 @@ package translator import ( "encoding/json" "fmt" + "strings" "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" @@ -76,6 +78,12 @@ func (t *Translator) translateSecret(tctx *provider.TranslateContext, listener g sslObjs := make([]*adctypes.SSL, 0) switch *listener.TLS.Mode { case gatewayv1.TLSModeTerminate: + // frontendValidation configures downstream mTLS: clients must present a + // certificate signed by one of the referenced CAs during the TLS handshake. + client, err := t.translateFrontendValidation(tctx, listener, obj) + if err != nil { + return nil, err + } for refIndex, ref := range listener.TLS.CertificateRefs { ns := obj.GetNamespace() if ref.Namespace != nil { @@ -118,6 +126,7 @@ func (t *Translator) translateSecret(tctx *provider.TranslateContext, listener g } sslObj.Snis = append(sslObj.Snis, hosts...) } + sslObj.Client = client sslObj.ID = id.GenID(fmt.Sprintf("%s_%s_%d", adctypes.ComposeSSLName(internaltypes.KindGateway, obj.Namespace, obj.Name), listener.Name, refIndex)) t.Log.V(1).Info("generated ssl id", "ssl id", sslObj.ID, "secret", secretNN.String()) sslObj.Labels = label.GenLabel(obj) @@ -135,6 +144,65 @@ func (t *Translator) translateSecret(tctx *provider.TranslateContext, listener g return sslObjs, nil } +// translateFrontendValidation builds the downstream mTLS client configuration from a +// listener's frontendValidation. The referenced CA certificates (ConfigMap, key `ca.crt`) +// are bundled into a single trust anchor used to validate client certificates. +func (t *Translator) translateFrontendValidation(tctx *provider.TranslateContext, listener gatewayv1.Listener, obj *gatewayv1.Gateway) (*adctypes.ClientClass, error) { + if listener.TLS.FrontendValidation == nil || len(listener.TLS.FrontendValidation.CACertificateRefs) == 0 { + return nil, nil + } + + cas := make([]string, 0, len(listener.TLS.FrontendValidation.CACertificateRefs)) + for _, ref := range listener.TLS.FrontendValidation.CACertificateRefs { + // caCertificateRefs must be in the core API group. ConfigMap is the + // Gateway API Core support; Secret is an implementation-specific extension. + if ref.Group != "" && string(ref.Group) != corev1.GroupName { + return nil, fmt.Errorf("unsupported frontendValidation caCertificateRef group %q in listener %s, only the core group is supported", ref.Group, listener.Name) + } + ns := obj.GetNamespace() + if ref.Namespace != nil { + ns = string(*ref.Namespace) + } + nn := types.NamespacedName{Namespace: ns, Name: string(ref.Name)} + + kind := internaltypes.KindConfigMap + if ref.Kind != "" { + kind = string(ref.Kind) + } + var ( + ca []byte + err error + ) + switch kind { + case internaltypes.KindConfigMap: + cm := tctx.ConfigMaps[nn] + if cm == nil { + return nil, fmt.Errorf("frontendValidation CA ConfigMap %s not found", nn.String()) + } + if ca, err = sslutils.ExtractCAFromConfigMap(cm); err != nil { + t.Log.Error(err, "failed to extract CA from configmap", "configmap", nn.String()) + return nil, fmt.Errorf("failed to extract CA from ConfigMap %s: %w", nn.String(), err) + } + case internaltypes.KindSecret: + secret := tctx.Secrets[nn] + if secret == nil { + return nil, fmt.Errorf("frontendValidation CA Secret %s not found", nn.String()) + } + if ca, err = sslutils.ExtractCAFromSecret(secret); err != nil { + t.Log.Error(err, "failed to extract CA from secret", "secret", nn.String()) + return nil, fmt.Errorf("failed to extract CA from Secret %s: %w", nn.String(), err) + } + default: + return nil, fmt.Errorf("unsupported frontendValidation caCertificateRef kind %q in listener %s, only ConfigMap and Secret are supported", ref.Kind, listener.Name) + } + cas = append(cas, strings.TrimSpace(string(ca))) + } + + return &adctypes.ClientClass{ + CA: strings.Join(cas, "\n"), + }, nil +} + // fillPluginsFromGatewayProxy fill plugins from GatewayProxy to given plugins func (t *Translator) fillPluginsFromGatewayProxy(plugins adctypes.GlobalRule, gatewayProxy *v1alpha1.GatewayProxy) { if gatewayProxy == nil { diff --git a/internal/adc/translator/gateway_test.go b/internal/adc/translator/gateway_test.go new file mode 100644 index 00000000..4c2c6cae --- /dev/null +++ b/internal/adc/translator/gateway_test.go @@ -0,0 +1,210 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package translator + +import ( + "context" + "testing" + + "github.com/go-logr/logr" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/ptr" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" + + "github.com/apache/apisix-ingress-controller/internal/provider" +) + +const testCACert = `-----BEGIN CERTIFICATE----- +MIIBQzCB6qADAgECAgEBMAoGCCqGSM49BAMCMBIxEDAOBgNVBAMTB3Rlc3QtY2Ew +HhcNNzAwMTAxMDAwMDAwWhcNMzgwMTE5MDMxNDA4WjASMRAwDgYDVQQDEwd0ZXN0 +LWNhMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEJo4AsM30ZHN+mYeHjqwceGBz +V2bMz1+OyNXuaPYVrSF7HShZhanOYNHb6QLNhjGxMsBDQHVLolPjyTQJp9R5GqMx +MC8wDgYDVR0PAQH/BAQDAgIEMB0GA1UdDgQWBBRzjh0YVmnpN/cFJziO0aYySuti +4DAKBggqhkjOPQQDAgNIADBFAiEA7fEGiQA7wX0LrrkRH4KplAPOgVV5Kvm/1dv1 +3TLq9ssCIHKkv2dhydRvv36KC1WsRDcrl7W+7YmEnCS9PZfb8agM +-----END CERTIFICATE-----` + +func newTLSGateway(frontendValidation *gatewayv1.FrontendTLSValidation) *gatewayv1.Gateway { + return &gatewayv1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "gw", + }, + Spec: gatewayv1.GatewaySpec{ + Listeners: []gatewayv1.Listener{ + { + Name: "https", + Hostname: ptr.To(gatewayv1.Hostname("example.com")), + TLS: &gatewayv1.GatewayTLSConfig{ + Mode: ptr.To(gatewayv1.TLSModeTerminate), + CertificateRefs: []gatewayv1.SecretObjectReference{ + { + Kind: ptr.To(gatewayv1.Kind("Secret")), + Name: gatewayv1.ObjectName("server-cert"), + }, + }, + FrontendValidation: frontendValidation, + }, + }, + }, + }, + } +} + +func newTranslateContextWithTLS() *provider.TranslateContext { + tctx := provider.NewDefaultTranslateContext(context.Background()) + tctx.Secrets[types.NamespacedName{Namespace: "default", Name: "server-cert"}] = &corev1.Secret{ + Data: map[string][]byte{ + "cert": []byte("server-cert-data"), + "key": []byte("server-key-data"), + }, + } + tctx.ConfigMaps[types.NamespacedName{Namespace: "default", Name: "ca-cm"}] = &corev1.ConfigMap{ + Data: map[string]string{ + corev1.ServiceAccountRootCAKey: testCACert, + }, + } + tctx.Secrets[types.NamespacedName{Namespace: "default", Name: "ca-secret"}] = &corev1.Secret{ + Data: map[string][]byte{ + corev1.ServiceAccountRootCAKey: []byte(testCACert), + }, + } + return tctx +} + +func TestTranslateSecret_FrontendValidation(t *testing.T) { + t.Run("with frontendValidation sets downstream mTLS client CA", func(t *testing.T) { + tr := &Translator{Log: logr.Discard()} + gateway := newTLSGateway(&gatewayv1.FrontendTLSValidation{ + CACertificateRefs: []gatewayv1.ObjectReference{ + { + Group: "", + Kind: "ConfigMap", + Name: "ca-cm", + }, + }, + }) + tctx := newTranslateContextWithTLS() + + sslObjs, err := tr.translateSecret(tctx, gateway.Spec.Listeners[0], gateway) + require.NoError(t, err) + require.Len(t, sslObjs, 1) + require.NotNil(t, sslObjs[0].Client, "client mTLS config should be set") + assert.Equal(t, testCACert, sslObjs[0].Client.CA) + assert.Equal(t, []string{"example.com"}, sslObjs[0].Snis) + }) + + t.Run("with Secret CA ref sets downstream mTLS client CA", func(t *testing.T) { + tr := &Translator{Log: logr.Discard()} + gateway := newTLSGateway(&gatewayv1.FrontendTLSValidation{ + CACertificateRefs: []gatewayv1.ObjectReference{ + {Group: "", Kind: "Secret", Name: "ca-secret"}, + }, + }) + tctx := newTranslateContextWithTLS() + + sslObjs, err := tr.translateSecret(tctx, gateway.Spec.Listeners[0], gateway) + require.NoError(t, err) + require.Len(t, sslObjs, 1) + require.NotNil(t, sslObjs[0].Client, "client mTLS config should be set") + assert.Equal(t, testCACert, sslObjs[0].Client.CA) + }) + + t.Run("missing CA Secret returns error", func(t *testing.T) { + tr := &Translator{Log: logr.Discard()} + gateway := newTLSGateway(&gatewayv1.FrontendTLSValidation{ + CACertificateRefs: []gatewayv1.ObjectReference{ + {Kind: "Secret", Name: "missing"}, + }, + }) + tctx := newTranslateContextWithTLS() + + _, err := tr.translateSecret(tctx, gateway.Spec.Listeners[0], gateway) + require.Error(t, err) + }) + + t.Run("without frontendValidation leaves client nil", func(t *testing.T) { + tr := &Translator{Log: logr.Discard()} + gateway := newTLSGateway(nil) + tctx := newTranslateContextWithTLS() + + sslObjs, err := tr.translateSecret(tctx, gateway.Spec.Listeners[0], gateway) + require.NoError(t, err) + require.Len(t, sslObjs, 1) + assert.Nil(t, sslObjs[0].Client) + }) + + t.Run("missing CA ConfigMap returns error", func(t *testing.T) { + tr := &Translator{Log: logr.Discard()} + gateway := newTLSGateway(&gatewayv1.FrontendTLSValidation{ + CACertificateRefs: []gatewayv1.ObjectReference{ + {Kind: "ConfigMap", Name: "missing"}, + }, + }) + tctx := newTranslateContextWithTLS() + + _, err := tr.translateSecret(tctx, gateway.Spec.Listeners[0], gateway) + require.Error(t, err) + }) + + t.Run("unsupported CA ref kind returns error", func(t *testing.T) { + tr := &Translator{Log: logr.Discard()} + gateway := newTLSGateway(&gatewayv1.FrontendTLSValidation{ + CACertificateRefs: []gatewayv1.ObjectReference{ + {Kind: "Pod", Name: "ca-cm"}, + }, + }) + tctx := newTranslateContextWithTLS() + + _, err := tr.translateSecret(tctx, gateway.Spec.Listeners[0], gateway) + require.Error(t, err) + }) + + t.Run("unsupported CA ref group returns error", func(t *testing.T) { + tr := &Translator{Log: logr.Discard()} + gateway := newTLSGateway(&gatewayv1.FrontendTLSValidation{ + CACertificateRefs: []gatewayv1.ObjectReference{ + {Group: "example.com", Kind: "ConfigMap", Name: "ca-cm"}, + }, + }) + tctx := newTranslateContextWithTLS() + + _, err := tr.translateSecret(tctx, gateway.Spec.Listeners[0], gateway) + require.Error(t, err) + }) + + t.Run("malformed CA data returns error", func(t *testing.T) { + tr := &Translator{Log: logr.Discard()} + gateway := newTLSGateway(&gatewayv1.FrontendTLSValidation{ + CACertificateRefs: []gatewayv1.ObjectReference{ + {Kind: "ConfigMap", Name: "ca-cm"}, + }, + }) + tctx := newTranslateContextWithTLS() + tctx.ConfigMaps[types.NamespacedName{Namespace: "default", Name: "ca-cm"}] = &corev1.ConfigMap{ + Data: map[string]string{corev1.ServiceAccountRootCAKey: " not a pem cert "}, + } + + _, err := tr.translateSecret(tctx, gateway.Spec.Listeners[0], gateway) + require.Error(t, err) + }) +} diff --git a/internal/controller/gateway_controller.go b/internal/controller/gateway_controller.go index 147bdd17..13e88c1f 100644 --- a/internal/controller/gateway_controller.go +++ b/internal/controller/gateway_controller.go @@ -91,6 +91,10 @@ func (r *GatewayReconciler) SetupWithManager(mgr ctrl.Manager) error { Watches( &corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(r.listGatewaysForSecret), + ). + Watches( + &corev1.ConfigMap{}, + handler.EnqueueRequestsFromMapFunc(r.listGatewaysForConfigMap), ) if GetEnableReferenceGrant() { @@ -390,6 +394,34 @@ func (r *GatewayReconciler) listGatewaysForSecret(ctx context.Context, obj clien return requests } +func (r *GatewayReconciler) listGatewaysForConfigMap(ctx context.Context, obj client.Object) (requests []reconcile.Request) { + configMap, ok := obj.(*corev1.ConfigMap) + if !ok { + r.Log.Error( + errors.New("unexpected object type"), + "ConfigMap watch predicate received unexpected object type", + "expected", FullTypeName(new(corev1.ConfigMap)), "found", FullTypeName(obj), + ) + return nil + } + var gatewayList gatewayv1.GatewayList + if err := r.List(ctx, &gatewayList, client.MatchingFields{ + indexer.ConfigMapIndexRef: indexer.GenIndexKey(configMap.GetNamespace(), configMap.GetName()), + }); err != nil { + r.Log.Error(err, "failed to list gateways for configmap") + return nil + } + for _, gateway := range gatewayList.Items { + requests = append(requests, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: gateway.GetNamespace(), + Name: gateway.GetName(), + }, + }) + } + return requests +} + func (r *GatewayReconciler) listReferenceGrantsForGateway(ctx context.Context, obj client.Object) (requests []reconcile.Request) { grant, ok := obj.(*v1beta1.ReferenceGrant) if !ok { @@ -434,7 +466,7 @@ func (r *GatewayReconciler) processInfrastructure(tctx *provider.TranslateContex func (r *GatewayReconciler) processListenerConfig(tctx *provider.TranslateContext, gateway *gatewayv1.Gateway) { listeners := gateway.Spec.Listeners for _, listener := range listeners { - if listener.TLS == nil || listener.TLS.CertificateRefs == nil { + if listener.TLS == nil { continue } secret := corev1.Secret{} @@ -457,5 +489,41 @@ func (r *GatewayReconciler) processListenerConfig(tctx *provider.TranslateContex tctx.Secrets[types.NamespacedName{Namespace: ns, Name: string(ref.Name)}] = &secret } } + // frontendValidation references CA ConfigMaps or Secrets used for downstream mTLS. + if listener.TLS.FrontendValidation != nil { + for _, ref := range listener.TLS.FrontendValidation.CACertificateRefs { + ns := gateway.GetNamespace() + if ref.Namespace != nil { + ns = string(*ref.Namespace) + } + nn := types.NamespacedName{Namespace: ns, Name: string(ref.Name)} + kind := KindConfigMap + if ref.Kind != "" { + kind = string(ref.Kind) + } + switch kind { + case KindConfigMap: + configMap := corev1.ConfigMap{} + if err := r.Get(context.Background(), nn, &configMap); err != nil { + r.Log.Error(err, "failed to get CA configmap", "namespace", ns, "name", ref.Name) + SetGatewayListenerConditionProgrammed(gateway, string(listener.Name), false, err.Error()) + SetGatewayListenerConditionResolvedRefs(gateway, string(listener.Name), false, err.Error()) + continue + } + r.Log.Info("Setting CA configmap for listener", "listener", listener.Name, "configmap", configMap.Name, "namespace", ns) + tctx.ConfigMaps[nn] = &configMap + case KindSecret: + caSecret := corev1.Secret{} + if err := r.Get(context.Background(), nn, &caSecret); err != nil { + r.Log.Error(err, "failed to get CA secret", "namespace", ns, "name", ref.Name) + SetGatewayListenerConditionProgrammed(gateway, string(listener.Name), false, err.Error()) + SetGatewayListenerConditionResolvedRefs(gateway, string(listener.Name), false, err.Error()) + continue + } + r.Log.Info("Setting CA secret for listener", "listener", listener.Name, "secret", caSecret.Name, "namespace", ns) + tctx.Secrets[nn] = &caSecret + } + } + } } } diff --git a/internal/controller/indexer/indexer.go b/internal/controller/indexer/indexer.go index 61a64c5d..81b9c0ef 100644 --- a/internal/controller/indexer/indexer.go +++ b/internal/controller/indexer/indexer.go @@ -47,6 +47,7 @@ const ( ParentRefs = "parentRefs" IngressClass = "ingressClass" SecretIndexRef = "secretRefs" + ConfigMapIndexRef = "configMapRefs" IngressClassRef = "ingressClassRef" IngressClassParametersRef = "ingressClassParametersRef" ConsumerGatewayRef = "consumerGatewayRef" @@ -146,6 +147,15 @@ func setupGatewayIndexer(mgr ctrl.Manager) error { return err } + if err := mgr.GetFieldIndexer().IndexField( + context.Background(), + &gatewayv1.Gateway{}, + ConfigMapIndexRef, + GatewayConfigMapIndexFunc, + ); err != nil { + return err + } + if err := mgr.GetFieldIndexer().IndexField( context.Background(), &gatewayv1.Gateway{}, @@ -591,8 +601,15 @@ func IngressSecretIndexFunc(rawObj client.Object) []string { func GatewaySecretIndexFunc(rawObj client.Object) (keys []string) { gateway := rawObj.(*gatewayv1.Gateway) var m = make(map[string]struct{}) + add := func(namespace, name string) { + key := GenIndexKey(namespace, name) + if _, ok := m[key]; !ok { + m[key] = struct{}{} + keys = append(keys, key) + } + } for _, listener := range gateway.Spec.Listeners { - if listener.TLS == nil || len(listener.TLS.CertificateRefs) == 0 { + if listener.TLS == nil { continue } for _, ref := range listener.TLS.CertificateRefs { @@ -603,6 +620,42 @@ func GatewaySecretIndexFunc(rawObj client.Object) (keys []string) { if ref.Namespace != nil { namespace = string(*ref.Namespace) } + add(namespace, string(ref.Name)) + } + // frontendValidation CA references that are Secrets. + if listener.TLS.FrontendValidation != nil { + for _, ref := range listener.TLS.FrontendValidation.CACertificateRefs { + if string(ref.Kind) != internaltypes.KindSecret { + continue + } + namespace := gateway.GetNamespace() + if ref.Namespace != nil { + namespace = string(*ref.Namespace) + } + add(namespace, string(ref.Name)) + } + } + } + return keys +} + +// GatewayConfigMapIndexFunc indexes Gateways by the CA ConfigMaps referenced via +// listener TLS frontendValidation, so that ConfigMap changes can trigger reconciliation. +func GatewayConfigMapIndexFunc(rawObj client.Object) (keys []string) { + gateway := rawObj.(*gatewayv1.Gateway) + var m = make(map[string]struct{}) + for _, listener := range gateway.Spec.Listeners { + if listener.TLS == nil || listener.TLS.FrontendValidation == nil { + continue + } + for _, ref := range listener.TLS.FrontendValidation.CACertificateRefs { + if ref.Kind != "" && string(ref.Kind) != internaltypes.KindConfigMap { + continue + } + namespace := gateway.GetNamespace() + if ref.Namespace != nil { + namespace = string(*ref.Namespace) + } key := GenIndexKey(namespace, string(ref.Name)) if _, ok := m[key]; !ok { m[key] = struct{}{} diff --git a/internal/controller/utils.go b/internal/controller/utils.go index f34f01a4..04400609 100644 --- a/internal/controller/utils.go +++ b/internal/controller/utils.go @@ -54,6 +54,7 @@ import ( "github.com/apache/apisix-ingress-controller/internal/controller/config" "github.com/apache/apisix-ingress-controller/internal/controller/indexer" "github.com/apache/apisix-ingress-controller/internal/provider" + sslutils "github.com/apache/apisix-ingress-controller/internal/ssl" "github.com/apache/apisix-ingress-controller/internal/types" "github.com/apache/apisix-ingress-controller/internal/utils" pkgutils "github.com/apache/apisix-ingress-controller/pkg/utils" @@ -70,6 +71,7 @@ const ( KindIngressClass = "IngressClass" KindGatewayProxy = "GatewayProxy" KindSecret = "Secret" + KindConfigMap = "ConfigMap" KindService = "Service" KindApisixRoute = "ApisixRoute" KindApisixGlobalRule = "ApisixGlobalRule" @@ -938,6 +940,12 @@ func getListenerStatus( break } } + + // frontendValidation (downstream mTLS) only applies to Terminate listeners. + if listener.TLS.FrontendValidation != nil && + (listener.TLS.Mode == nil || *listener.TLS.Mode == gatewayv1.TLSModeTerminate) { + validateListenerFrontendValidation(ctx, mrgc, gateway, listener.TLS.FrontendValidation, &conditionResolvedRefs, &conditionProgrammed) + } } status := gatewayv1.ListenerStatus{ @@ -976,6 +984,86 @@ func getListenerStatus( return statusArray, nil } +// validateListenerFrontendValidation validates a listener's TLS frontendValidation +// (downstream mTLS CA references) and records the outcome on the listener conditions. +func validateListenerFrontendValidation( + ctx context.Context, + mrgc client.Client, + gateway *gatewayv1.Gateway, + frontendValidation *gatewayv1.FrontendTLSValidation, + conditionResolvedRefs, conditionProgrammed *metav1.Condition, +) { + setInvalid := func(reason gatewayv1.ListenerConditionReason, message string) { + conditionResolvedRefs.Status = metav1.ConditionFalse + conditionResolvedRefs.Reason = string(reason) + conditionResolvedRefs.Message = message + conditionProgrammed.Status = metav1.ConditionFalse + conditionProgrammed.Reason = string(gatewayv1.ListenerReasonInvalid) + } + + for _, ref := range frontendValidation.CACertificateRefs { + if ref.Group != "" && string(ref.Group) != corev1.GroupName { + setInvalid(gatewayv1.ListenerReasonInvalidCertificateRef, + fmt.Sprintf(`Invalid Group for caCertificateRef, expect "", got "%s"`, ref.Group)) + return + } + kind := KindConfigMap + if ref.Kind != "" { + kind = string(ref.Kind) + } + if kind != KindConfigMap && kind != KindSecret { + setInvalid(gatewayv1.ListenerReasonInvalidCertificateRef, + fmt.Sprintf(`Invalid Kind for caCertificateRef, expect "ConfigMap" or "Secret", got "%s"`, ref.Kind)) + return + } + if permitted := checkReferenceGrant(ctx, + mrgc, + v1beta1.ReferenceGrantFrom{ + Group: gatewayv1.GroupName, + Kind: KindGateway, + Namespace: v1beta1.Namespace(gateway.Namespace), + }, + gatewayv1.ObjectReference{ + Group: corev1.GroupName, + Kind: gatewayv1.Kind(kind), + Name: ref.Name, + Namespace: ref.Namespace, + }, + ); !permitted { + setInvalid(gatewayv1.ListenerReasonRefNotPermitted, "caCertificateRefs cross namespaces is not permitted") + return + } + nn := k8stypes.NamespacedName{ + Namespace: string(*cmp.Or(ref.Namespace, (*gatewayv1.Namespace)(&gateway.Namespace))), + Name: string(ref.Name), + } + switch kind { + case KindConfigMap: + var configMap corev1.ConfigMap + if err := mrgc.Get(ctx, nn, &configMap); err != nil { + setInvalid(gatewayv1.ListenerReasonInvalidCertificateRef, err.Error()) + return + } + if _, err := sslutils.ExtractCAFromConfigMap(&configMap); err != nil { + setInvalid(gatewayv1.ListenerReasonInvalidCertificateRef, + fmt.Sprintf("Malformed CA ConfigMap referenced: %s", err.Error())) + return + } + case KindSecret: + var secret corev1.Secret + if err := mrgc.Get(ctx, nn, &secret); err != nil { + setInvalid(gatewayv1.ListenerReasonInvalidCertificateRef, err.Error()) + return + } + if _, err := sslutils.ExtractCAFromSecret(&secret); err != nil { + setInvalid(gatewayv1.ListenerReasonInvalidCertificateRef, + fmt.Sprintf("Malformed CA Secret referenced: %s", err.Error())) + return + } + } + } +} + // SplitMetaNamespaceKey returns the namespace and name that // MetaNamespaceKeyFunc encoded into key. func SplitMetaNamespaceKey(key string) (namespace, name string, err error) { diff --git a/internal/manager/controllers.go b/internal/manager/controllers.go index f8e70890..ec88cda4 100644 --- a/internal/manager/controllers.go +++ b/internal/manager/controllers.go @@ -51,6 +51,7 @@ import ( // +kubebuilder:rbac:groups="",resources=services,verbs=get;list;watch // +kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch // +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch +// +kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch // +kubebuilder:rbac:groups="",resources=namespaces,verbs=get;list;watch // +kubebuilder:rbac:groups="",resources=endpoints,verbs=get;list;watch diff --git a/internal/provider/provider.go b/internal/provider/provider.go index 92f36a87..51d27cb8 100644 --- a/internal/provider/provider.go +++ b/internal/provider/provider.go @@ -50,6 +50,7 @@ type TranslateContext struct { EndpointSlices map[k8stypes.NamespacedName][]discoveryv1.EndpointSlice Secrets map[k8stypes.NamespacedName]*corev1.Secret + ConfigMaps map[k8stypes.NamespacedName]*corev1.ConfigMap PluginConfigs map[k8stypes.NamespacedName]*v1alpha1.PluginConfig ApisixPluginConfigs map[k8stypes.NamespacedName]*apiv2.ApisixPluginConfig Services map[k8stypes.NamespacedName]*corev1.Service @@ -70,6 +71,7 @@ func NewDefaultTranslateContext(ctx context.Context) *TranslateContext { Context: ctx, EndpointSlices: make(map[k8stypes.NamespacedName][]discoveryv1.EndpointSlice), Secrets: make(map[k8stypes.NamespacedName]*corev1.Secret), + ConfigMaps: make(map[k8stypes.NamespacedName]*corev1.ConfigMap), PluginConfigs: make(map[k8stypes.NamespacedName]*v1alpha1.PluginConfig), ApisixPluginConfigs: make(map[k8stypes.NamespacedName]*apiv2.ApisixPluginConfig), Services: make(map[k8stypes.NamespacedName]*corev1.Service), diff --git a/internal/ssl/util.go b/internal/ssl/util.go index f5fc6b19..ad34dd40 100644 --- a/internal/ssl/util.go +++ b/internal/ssl/util.go @@ -83,6 +83,63 @@ func ExtractCertificate(secret *corev1.Secret) ([]byte, error) { return cert, err } +// ExtractCAFromConfigMap extracts the CA certificate from a ConfigMap. +// +// Following the Gateway API conformance for frontendValidation, the CA certificate +// is read from the `ca.crt` key. Both Data and BinaryData are supported. +func ExtractCAFromConfigMap(cm *corev1.ConfigMap) ([]byte, error) { + if cm == nil { + return nil, ErrMissingCert + } + var ca []byte + if v, ok := cm.Data[corev1.ServiceAccountRootCAKey]; ok && v != "" { + ca = []byte(v) + } else if v, ok := cm.BinaryData[corev1.ServiceAccountRootCAKey]; ok && len(v) > 0 { + ca = v + } + if len(ca) == 0 { + return nil, ErrMissingCert + } + // Reject whitespace-only or otherwise malformed data so an invalid trust + // anchor never reaches the downstream mTLS configuration. + if !hasCertificatePEMBlock(ca) { + return nil, ErrInvalidPEM + } + return ca, nil +} + +// ExtractCAFromSecret extracts the CA certificate from a Secret's `ca.crt` key +// (e.g. a cert-manager-issued TLS Secret) and validates it contains a PEM +// CERTIFICATE block. +func ExtractCAFromSecret(secret *corev1.Secret) ([]byte, error) { + if secret == nil { + return nil, ErrMissingCert + } + ca, ok := secret.Data[corev1.ServiceAccountRootCAKey] + if !ok || len(ca) == 0 { + return nil, ErrMissingCert + } + if !hasCertificatePEMBlock(ca) { + return nil, ErrInvalidPEM + } + return ca, nil +} + +// hasCertificatePEMBlock reports whether data contains at least one PEM-encoded +// CERTIFICATE block. +func hasCertificatePEMBlock(data []byte) bool { + for { + var block *pem.Block + block, data = pem.Decode(data) + if block == nil { + return false + } + if block.Type == "CERTIFICATE" { + return true + } + } +} + // ExtractHostsFromCertificate parses the certificate PEM block and returns the DNS names. func ExtractHostsFromCertificate(certPEM []byte) ([]string, error) { block, _ := pem.Decode(certPEM) diff --git a/internal/types/k8s.go b/internal/types/k8s.go index 3fc34a3e..7bc4cb81 100644 --- a/internal/types/k8s.go +++ b/internal/types/k8s.go @@ -50,6 +50,7 @@ const ( KindIngressClass = "IngressClass" KindGatewayProxy = "GatewayProxy" KindSecret = "Secret" + KindConfigMap = "ConfigMap" KindService = "Service" KindApisixRoute = "ApisixRoute" KindApisixGlobalRule = "ApisixGlobalRule" diff --git a/test/e2e/framework/manifests/ingress.yaml b/test/e2e/framework/manifests/ingress.yaml index af36fa81..a4e244a4 100644 --- a/test/e2e/framework/manifests/ingress.yaml +++ b/test/e2e/framework/manifests/ingress.yaml @@ -80,6 +80,7 @@ rules: - apiGroups: - "" resources: + - configmaps - namespaces - pods - secrets diff --git a/test/e2e/gatewayapi/gateway.go b/test/e2e/gatewayapi/gateway.go index a5f19cd7..c971a5ae 100644 --- a/test/e2e/gatewayapi/gateway.go +++ b/test/e2e/gatewayapi/gateway.go @@ -36,6 +36,8 @@ import ( const _secretName = "test-apisix-tls" +const _hostAPI6 = "api6.com" + var Cert = strings.TrimSpace(framework.TestServerCert) var Key = strings.TrimSpace(framework.TestServerKey) @@ -45,6 +47,16 @@ func createSecret(s *scaffold.Scaffold, secretName string) { assert.Nil(GinkgoT(), err, "create secret error") } +// indentLines indents every line of s with the given prefix, for embedding a +// multi-line PEM block inside a YAML block scalar. +func indentLines(s, prefix string) string { + lines := strings.Split(strings.TrimRight(s, "\n"), "\n") + for i, line := range lines { + lines[i] = prefix + line + } + return strings.Join(lines, "\n") +} + var _ = Describe("Test Gateway", Label("networking.k8s.io", "gateway"), func() { s := scaffold.NewDefaultScaffold() @@ -166,7 +178,7 @@ spec: By("create secret") secretName := _secretName - host := "api6.com" + host := _hostAPI6 createSecret(s, secretName) gatewayClassName := s.Namespace() var defaultGatewayClass = ` @@ -219,6 +231,98 @@ spec: assert.ElementsMatch(GinkgoT(), []string{host}, tls[0].Snis) }) + It("Check downstream mTLS via frontendValidation", func() { + By("create GatewayProxy") + gatewayProxy := fmt.Sprintf(gatewayProxyYaml, s.Namespace(), s.Deployer.GetAdminEndpoint(), s.AdminKey()) + err := s.CreateResourceFromString(gatewayProxy) + Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy") + time.Sleep(5 * time.Second) + + By("create server cert secret") + secretName := _secretName + host := _hostAPI6 + createSecret(s, secretName) + + By("create CA ConfigMap for frontendValidation") + caConfigMapName := "test-client-ca" + caConfigMap := fmt.Sprintf(` +apiVersion: v1 +kind: ConfigMap +metadata: + name: %s +data: + ca.crt: | +%s +`, caConfigMapName, indentLines(framework.TestCACert, " ")) + err = s.CreateResourceFromString(caConfigMap) + Expect(err).NotTo(HaveOccurred(), "creating CA ConfigMap") + + gatewayClassName := s.Namespace() + var defaultGatewayClass = ` +apiVersion: gateway.networking.k8s.io/v1 +kind: GatewayClass +metadata: + name: %s +spec: + controllerName: "%s" +` + var defaultGateway = fmt.Sprintf(` +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: %s +spec: + gatewayClassName: %s + listeners: + - name: http1 + protocol: HTTPS + port: 443 + hostname: %s + tls: + certificateRefs: + - kind: Secret + group: "" + name: %s + frontendValidation: + caCertificateRefs: + - kind: ConfigMap + group: "" + name: %s + infrastructure: + parametersRef: + group: apisix.apache.org + kind: GatewayProxy + name: apisix-proxy-config +`, s.Namespace(), gatewayClassName, host, secretName, caConfigMapName) + + By("create GatewayClass") + err = s.CreateResourceFromStringWithNamespace(fmt.Sprintf(defaultGatewayClass, gatewayClassName, s.GetControllerName()), "") + Expect(err).NotTo(HaveOccurred(), "creating GatewayClass") + time.Sleep(5 * time.Second) + + By("create Gateway") + err = s.CreateResourceFromStringWithNamespace(defaultGateway, s.Namespace()) + Expect(err).NotTo(HaveOccurred(), "creating Gateway") + time.Sleep(10 * time.Second) + + Eventually(func() error { + tls, err := s.DefaultDataplaneResource().SSL().List(context.Background()) + if err != nil { + return err + } + if len(tls) != 1 { + return fmt.Errorf("expect 1 ssl, got %d", len(tls)) + } + if tls[0].Client == nil { + return fmt.Errorf("expect client mTLS config, got nil") + } + if got := strings.TrimSpace(tls[0].Client.CA); got != strings.TrimSpace(framework.TestCACert) { + return fmt.Errorf("client CA not expected, got %s", got) + } + return nil + }).WithTimeout(30 * time.Second).ProbeEvery(time.Second).Should(Succeed()) + }) + It("Gateway SSL with and without hostname", func() { By("create GatewayProxy") gatewayProxy := fmt.Sprintf(gatewayProxyYaml, s.Namespace(), s.Deployer.GetAdminEndpoint(), s.AdminKey())