From 998a13e71d2161a245807518dddc80fe1e3450b8 Mon Sep 17 00:00:00 2001 From: hbc Date: Wed, 4 Mar 2026 21:10:12 -0800 Subject: [PATCH 1/4] refactor: drop wireguard references --- plugin/api/api.pb.go | 2 +- plugin/go.mod | 2 +- plugin/pkg/db/internal/testobj/testobj.pb.go | 2 +- .../services/agentpools/api/agentpools.pb.go | 2 +- .../agentpools/api/agentpools_grpc.pb.go | 2 +- .../agentpools/api/features/arc/arc.pb.go | 2 +- .../api/features/capacity/capacity.pb.go | 2 +- .../api/features/kubeadm/kubeadm.pb.go | 4 +- .../api/features/kubeadm/kubeadm.proto | 2 +- .../api/features/wireguard/wireguard.pb.go | 144 ------------ .../api/features/wireguard/wireguard.proto | 11 - .../services/agentpools/api/instances.pb.go | 2 +- .../agentpools/api/instances_grpc.pb.go | 2 +- .../aws/ubuntu2404instance/agentpools.pb.go | 2 +- .../aws/ubuntu2404instance/instances.pb.go | 2 +- .../azure/ubuntu2404vmss/agentpools.pb.go | 2 +- .../azure/ubuntu2404vmss/instances.pb.go | 2 +- .../agentpools/nebius/instance/agentpools.go | 29 --- .../nebius/instance/agentpools.pb.go | 79 ++----- .../nebius/instance/agentpools.proto | 2 - .../nebius/instance/assets/wg-spoke.sh | 213 ------------------ .../networks/api/features/vnet/vnet.pb.go | 2 +- .../pkg/services/networks/api/networks.pb.go | 2 +- .../services/networks/api/networks_grpc.pb.go | 2 +- .../pkg/services/networks/aws/vpc/vpc.pb.go | 2 +- .../services/networks/nebius/vpc/vpc.pb.go | 2 +- .../peerings/api/features/ipsec/ipsec.pb.go | 2 +- .../pkg/services/peerings/api/peerings.pb.go | 2 +- .../services/peerings/api/peerings_grpc.pb.go | 2 +- .../peerings/aws/ipsecvpn/ipsecvpn.pb.go | 2 +- plugin/pkg/util/kubeadm/kubeadm.go | 2 +- plugin/pkg/util/wireguard/wireguard.go | 110 --------- 32 files changed, 51 insertions(+), 589 deletions(-) delete mode 100644 plugin/pkg/services/agentpools/api/features/wireguard/wireguard.pb.go delete mode 100644 plugin/pkg/services/agentpools/api/features/wireguard/wireguard.proto delete mode 100644 plugin/pkg/services/agentpools/nebius/instance/assets/wg-spoke.sh delete mode 100644 plugin/pkg/util/wireguard/wireguard.go diff --git a/plugin/api/api.pb.go b/plugin/api/api.pb.go index 016819d..6c0eabc 100644 --- a/plugin/api/api.pb.go +++ b/plugin/api/api.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.36.11 -// protoc v6.33.4 +// protoc v7.34.0 // source: plugin/api/api.proto package api diff --git a/plugin/go.mod b/plugin/go.mod index ccd23ae..db5bed3 100644 --- a/plugin/go.mod +++ b/plugin/go.mod @@ -18,7 +18,6 @@ require ( github.com/nebius/gosdk v0.0.0-20260218100913-7fb27c45819a github.com/stretchr/testify v1.11.1 go.yaml.in/yaml/v3 v3.0.4 - golang.org/x/crypto v0.47.0 google.golang.org/grpc v1.79.1 google.golang.org/protobuf v1.36.11 gopkg.in/ini.v1 v1.67.1 @@ -91,6 +90,7 @@ require ( go.opentelemetry.io/otel v1.39.0 // indirect go.opentelemetry.io/otel/trace v1.39.0 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect + golang.org/x/crypto v0.47.0 // indirect golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect golang.org/x/net v0.49.0 // indirect golang.org/x/oauth2 v0.34.0 // indirect diff --git a/plugin/pkg/db/internal/testobj/testobj.pb.go b/plugin/pkg/db/internal/testobj/testobj.pb.go index 2065e55..e5f0ce2 100644 --- a/plugin/pkg/db/internal/testobj/testobj.pb.go +++ b/plugin/pkg/db/internal/testobj/testobj.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.36.11 -// protoc v6.33.4 +// protoc v7.34.0 // source: plugin/pkg/db/internal/testobj/testobj.proto package testobj diff --git a/plugin/pkg/services/agentpools/api/agentpools.pb.go b/plugin/pkg/services/agentpools/api/agentpools.pb.go index 6fc1a01..3fa817e 100644 --- a/plugin/pkg/services/agentpools/api/agentpools.pb.go +++ b/plugin/pkg/services/agentpools/api/agentpools.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.36.11 -// protoc v6.33.4 +// protoc v7.34.0 // source: plugin/pkg/services/agentpools/api/agentpools.proto package api diff --git a/plugin/pkg/services/agentpools/api/agentpools_grpc.pb.go b/plugin/pkg/services/agentpools/api/agentpools_grpc.pb.go index a9f5a09..455e1ef 100644 --- a/plugin/pkg/services/agentpools/api/agentpools_grpc.pb.go +++ b/plugin/pkg/services/agentpools/api/agentpools_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.6.1 -// - protoc v6.33.4 +// - protoc v7.34.0 // source: plugin/pkg/services/agentpools/api/agentpools.proto package api diff --git a/plugin/pkg/services/agentpools/api/features/arc/arc.pb.go b/plugin/pkg/services/agentpools/api/features/arc/arc.pb.go index 6b3f82f..eb409cc 100644 --- a/plugin/pkg/services/agentpools/api/features/arc/arc.pb.go +++ b/plugin/pkg/services/agentpools/api/features/arc/arc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.36.11 -// protoc v6.33.4 +// protoc v7.34.0 // source: plugin/pkg/services/agentpools/api/features/arc/arc.proto package arc diff --git a/plugin/pkg/services/agentpools/api/features/capacity/capacity.pb.go b/plugin/pkg/services/agentpools/api/features/capacity/capacity.pb.go index 9dcdd73..59271d7 100644 --- a/plugin/pkg/services/agentpools/api/features/capacity/capacity.pb.go +++ b/plugin/pkg/services/agentpools/api/features/capacity/capacity.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.36.11 -// protoc v6.33.4 +// protoc v7.34.0 // source: plugin/pkg/services/agentpools/api/features/capacity/capacity.proto package capacity diff --git a/plugin/pkg/services/agentpools/api/features/kubeadm/kubeadm.pb.go b/plugin/pkg/services/agentpools/api/features/kubeadm/kubeadm.pb.go index 62fd91d..b0a9706 100644 --- a/plugin/pkg/services/agentpools/api/features/kubeadm/kubeadm.pb.go +++ b/plugin/pkg/services/agentpools/api/features/kubeadm/kubeadm.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.36.11 -// protoc v6.33.4 +// protoc v7.34.0 // source: plugin/pkg/services/agentpools/api/features/kubeadm/kubeadm.proto package kubeadm @@ -343,7 +343,7 @@ type Config_builder struct { Token *string // List of node labels to assign using the kubelet flag NodeLabels map[string]string - // WireGuard IP for --node-ip kubelet argument (optional) + // IP for --node-ip kubelet argument (optional) NodeIp *string // List of taints to assign using the kubelet flag, in the format "key=value:effect" RegisterWithTaints []*Taint diff --git a/plugin/pkg/services/agentpools/api/features/kubeadm/kubeadm.proto b/plugin/pkg/services/agentpools/api/features/kubeadm/kubeadm.proto index 1e7881e..6002bfc 100644 --- a/plugin/pkg/services/agentpools/api/features/kubeadm/kubeadm.proto +++ b/plugin/pkg/services/agentpools/api/features/kubeadm/kubeadm.proto @@ -18,7 +18,7 @@ message Config { // List of node labels to assign using the kubelet flag map node_labels = 4; - // WireGuard IP for --node-ip kubelet argument (optional) + // IP for --node-ip kubelet argument (optional) string node_ip = 5; // List of taints to assign using the kubelet flag, in the format "key=value:effect" diff --git a/plugin/pkg/services/agentpools/api/features/wireguard/wireguard.pb.go b/plugin/pkg/services/agentpools/api/features/wireguard/wireguard.pb.go deleted file mode 100644 index f1dd3e5..0000000 --- a/plugin/pkg/services/agentpools/api/features/wireguard/wireguard.pb.go +++ /dev/null @@ -1,144 +0,0 @@ -// Code generated by protoc-gen-go. DO NOT EDIT. -// versions: -// protoc-gen-go v1.36.11 -// protoc v6.33.4 -// source: plugin/pkg/services/agentpools/api/features/wireguard/wireguard.proto - -package wireguard - -import ( - protoreflect "google.golang.org/protobuf/reflect/protoreflect" - protoimpl "google.golang.org/protobuf/runtime/protoimpl" - reflect "reflect" - unsafe "unsafe" -) - -const ( - // Verify that this generated code is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) - // Verify that runtime/protoimpl is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) -) - -type Config struct { - state protoimpl.MessageState `protogen:"opaque.v1"` - xxx_hidden_PeerIp *string `protobuf:"bytes,1,opt,name=peer_ip,json=peerIp"` - XXX_raceDetectHookData protoimpl.RaceDetectHookData - XXX_presence [1]uint32 - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache -} - -func (x *Config) Reset() { - *x = Config{} - mi := &file_plugin_pkg_services_agentpools_api_features_wireguard_wireguard_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) -} - -func (x *Config) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*Config) ProtoMessage() {} - -func (x *Config) ProtoReflect() protoreflect.Message { - mi := &file_plugin_pkg_services_agentpools_api_features_wireguard_wireguard_proto_msgTypes[0] - if x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -func (x *Config) GetPeerIp() string { - if x != nil { - if x.xxx_hidden_PeerIp != nil { - return *x.xxx_hidden_PeerIp - } - return "" - } - return "" -} - -func (x *Config) SetPeerIp(v string) { - x.xxx_hidden_PeerIp = &v - protoimpl.X.SetPresent(&(x.XXX_presence[0]), 0, 1) -} - -func (x *Config) HasPeerIp() bool { - if x == nil { - return false - } - return protoimpl.X.Present(&(x.XXX_presence[0]), 0) -} - -func (x *Config) ClearPeerIp() { - protoimpl.X.ClearPresent(&(x.XXX_presence[0]), 0) - x.xxx_hidden_PeerIp = nil -} - -type Config_builder struct { - _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. - - // The IP address to use for this peer. - PeerIp *string -} - -func (b0 Config_builder) Build() *Config { - m0 := &Config{} - b, x := &b0, m0 - _, _ = b, x - if b.PeerIp != nil { - protoimpl.X.SetPresentNonAtomic(&(x.XXX_presence[0]), 0, 1) - x.xxx_hidden_PeerIp = b.PeerIp - } - return m0 -} - -var File_plugin_pkg_services_agentpools_api_features_wireguard_wireguard_proto protoreflect.FileDescriptor - -const file_plugin_pkg_services_agentpools_api_features_wireguard_wireguard_proto_rawDesc = "" + - "\n" + - "Eplugin/pkg/services/agentpools/api/features/wireguard/wireguard.proto\x12\twireguard\"!\n" + - "\x06Config\x12\x17\n" + - "\apeer_ip\x18\x01 \x01(\tR\x06peerIpBQZOgithub.com/Azure/aks-flex/plugin/pkg/services/agentpools/api/features/wireguardb\beditionsp\xe9\a" - -var file_plugin_pkg_services_agentpools_api_features_wireguard_wireguard_proto_msgTypes = make([]protoimpl.MessageInfo, 1) -var file_plugin_pkg_services_agentpools_api_features_wireguard_wireguard_proto_goTypes = []any{ - (*Config)(nil), // 0: wireguard.Config -} -var file_plugin_pkg_services_agentpools_api_features_wireguard_wireguard_proto_depIdxs = []int32{ - 0, // [0:0] is the sub-list for method output_type - 0, // [0:0] is the sub-list for method input_type - 0, // [0:0] is the sub-list for extension type_name - 0, // [0:0] is the sub-list for extension extendee - 0, // [0:0] is the sub-list for field type_name -} - -func init() { file_plugin_pkg_services_agentpools_api_features_wireguard_wireguard_proto_init() } -func file_plugin_pkg_services_agentpools_api_features_wireguard_wireguard_proto_init() { - if File_plugin_pkg_services_agentpools_api_features_wireguard_wireguard_proto != nil { - return - } - type x struct{} - out := protoimpl.TypeBuilder{ - File: protoimpl.DescBuilder{ - GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: unsafe.Slice(unsafe.StringData(file_plugin_pkg_services_agentpools_api_features_wireguard_wireguard_proto_rawDesc), len(file_plugin_pkg_services_agentpools_api_features_wireguard_wireguard_proto_rawDesc)), - NumEnums: 0, - NumMessages: 1, - NumExtensions: 0, - NumServices: 0, - }, - GoTypes: file_plugin_pkg_services_agentpools_api_features_wireguard_wireguard_proto_goTypes, - DependencyIndexes: file_plugin_pkg_services_agentpools_api_features_wireguard_wireguard_proto_depIdxs, - MessageInfos: file_plugin_pkg_services_agentpools_api_features_wireguard_wireguard_proto_msgTypes, - }.Build() - File_plugin_pkg_services_agentpools_api_features_wireguard_wireguard_proto = out.File - file_plugin_pkg_services_agentpools_api_features_wireguard_wireguard_proto_goTypes = nil - file_plugin_pkg_services_agentpools_api_features_wireguard_wireguard_proto_depIdxs = nil -} diff --git a/plugin/pkg/services/agentpools/api/features/wireguard/wireguard.proto b/plugin/pkg/services/agentpools/api/features/wireguard/wireguard.proto deleted file mode 100644 index 64bdd86..0000000 --- a/plugin/pkg/services/agentpools/api/features/wireguard/wireguard.proto +++ /dev/null @@ -1,11 +0,0 @@ -edition = "2024"; - -package wireguard; - -option go_package = "github.com/Azure/aks-flex/plugin/pkg/services/agentpools/api/features/wireguard"; - -message Config { - // The IP address to use for this peer. - string peer_ip = 1; - // NOTE: the hub peer ip is discovered within the cluster automatically -} \ No newline at end of file diff --git a/plugin/pkg/services/agentpools/api/instances.pb.go b/plugin/pkg/services/agentpools/api/instances.pb.go index 34b3642..618f917 100644 --- a/plugin/pkg/services/agentpools/api/instances.pb.go +++ b/plugin/pkg/services/agentpools/api/instances.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.36.11 -// protoc v6.33.4 +// protoc v7.34.0 // source: plugin/pkg/services/agentpools/api/instances.proto package api diff --git a/plugin/pkg/services/agentpools/api/instances_grpc.pb.go b/plugin/pkg/services/agentpools/api/instances_grpc.pb.go index 20808cb..77546e0 100644 --- a/plugin/pkg/services/agentpools/api/instances_grpc.pb.go +++ b/plugin/pkg/services/agentpools/api/instances_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.6.1 -// - protoc v6.33.4 +// - protoc v7.34.0 // source: plugin/pkg/services/agentpools/api/instances.proto package api diff --git a/plugin/pkg/services/agentpools/aws/ubuntu2404instance/agentpools.pb.go b/plugin/pkg/services/agentpools/aws/ubuntu2404instance/agentpools.pb.go index 0c2c924..3e2818c 100644 --- a/plugin/pkg/services/agentpools/aws/ubuntu2404instance/agentpools.pb.go +++ b/plugin/pkg/services/agentpools/aws/ubuntu2404instance/agentpools.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.36.11 -// protoc v6.33.4 +// protoc v7.34.0 // source: plugin/pkg/services/agentpools/aws/ubuntu2404instance/agentpools.proto package ubuntu2404instance diff --git a/plugin/pkg/services/agentpools/aws/ubuntu2404instance/instances.pb.go b/plugin/pkg/services/agentpools/aws/ubuntu2404instance/instances.pb.go index b27b8e8..38bc142 100644 --- a/plugin/pkg/services/agentpools/aws/ubuntu2404instance/instances.pb.go +++ b/plugin/pkg/services/agentpools/aws/ubuntu2404instance/instances.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.36.11 -// protoc v6.33.4 +// protoc v7.34.0 // source: plugin/pkg/services/agentpools/aws/ubuntu2404instance/instances.proto package ubuntu2404instance diff --git a/plugin/pkg/services/agentpools/azure/ubuntu2404vmss/agentpools.pb.go b/plugin/pkg/services/agentpools/azure/ubuntu2404vmss/agentpools.pb.go index cc19b11..9122a7d 100644 --- a/plugin/pkg/services/agentpools/azure/ubuntu2404vmss/agentpools.pb.go +++ b/plugin/pkg/services/agentpools/azure/ubuntu2404vmss/agentpools.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.36.11 -// protoc v6.33.4 +// protoc v7.34.0 // source: plugin/pkg/services/agentpools/azure/ubuntu2404vmss/agentpools.proto package ubuntu2404vmss diff --git a/plugin/pkg/services/agentpools/azure/ubuntu2404vmss/instances.pb.go b/plugin/pkg/services/agentpools/azure/ubuntu2404vmss/instances.pb.go index 41e9c4f..dd3aec4 100644 --- a/plugin/pkg/services/agentpools/azure/ubuntu2404vmss/instances.pb.go +++ b/plugin/pkg/services/agentpools/azure/ubuntu2404vmss/instances.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.36.11 -// protoc v6.33.4 +// protoc v7.34.0 // source: plugin/pkg/services/agentpools/azure/ubuntu2404vmss/instances.proto package ubuntu2404vmss diff --git a/plugin/pkg/services/agentpools/nebius/instance/agentpools.go b/plugin/pkg/services/agentpools/nebius/instance/agentpools.go index 1d0e93c..2cd3f14 100644 --- a/plugin/pkg/services/agentpools/nebius/instance/agentpools.go +++ b/plugin/pkg/services/agentpools/nebius/instance/agentpools.go @@ -2,7 +2,6 @@ package instance import ( "context" - _ "embed" "fmt" "strings" @@ -18,13 +17,9 @@ import ( agentpools "github.com/Azure/aks-flex/plugin/pkg/services/agentpools/api" "github.com/Azure/aks-flex/plugin/pkg/services/agentpools/userdata/flex" "github.com/Azure/aks-flex/plugin/pkg/topology" - "github.com/Azure/aks-flex/plugin/pkg/util/cloudinit" utilnebius "github.com/Azure/aks-flex/plugin/pkg/util/nebius" ) -//go:embed assets/wg-spoke.sh -var wgSpokeScript string - var _ api.Object = (*AgentPool)(nil) type agentPoolsServer struct { @@ -62,14 +57,6 @@ func (srv *agentPoolsServer) CreateOrUpdate( topology.NodeLabelKeyInstanceType: apSpec.GetPlatform(), }) - wireguardIP := apSpec.GetWireguard().GetPeerIp() - - if wireguardIP != "" { - // for wireguard enabled instance, the node IP needs to be set to the WireGuard peer IP, - // so the network routing can work between nodes. - kubeadmConfig.SetNodeIp(wireguardIP) - } - // TODO: get gpu info from spec (might need to infer from SKU) hasGPU := strings.Contains(apSpec.GetImageFamily(), "cuda") // TODO: get the k8s version from spec @@ -81,22 +68,6 @@ func (srv *agentPoolsServer) CreateOrUpdate( return nil, fmt.Errorf("failed to generate userdata: %w", err) } - if wireguardIP != "" { - // TODO: this part should move to flex node bootstrap setup task - ud.Packages = append(ud.Packages, "wireguard", "wireguard-tools") - ud.WriteFiles = append(ud.WriteFiles, &cloudinit.WriteFile{ - Path: "/root/wg-spoke.sh", - Content: wgSpokeScript, - Permissions: "0755", - }) - ud.RunCmd = append(ud.RunCmd, strings.Join([]string{ - "export ANNOTATION_PREFIX='stretch.azure.com/wireguard-'", - fmt.Sprintf("export WG_ADDRESS='%s/32'", wireguardIP), - "export WG_DAEMONIZE='true'", - "/root/wg-spoke.sh", - }, "\n")) - } - userdataContent, err := ud.Marshal() if err != nil { return nil, fmt.Errorf("failed to marshal cloud-init: %w", err) diff --git a/plugin/pkg/services/agentpools/nebius/instance/agentpools.pb.go b/plugin/pkg/services/agentpools/nebius/instance/agentpools.pb.go index 2ca5e3f..da27eb7 100644 --- a/plugin/pkg/services/agentpools/nebius/instance/agentpools.pb.go +++ b/plugin/pkg/services/agentpools/nebius/instance/agentpools.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.36.11 -// protoc v6.33.4 +// protoc v7.34.0 // source: plugin/pkg/services/agentpools/nebius/instance/agentpools.proto package instance @@ -9,7 +9,6 @@ package instance import ( api "github.com/Azure/aks-flex/plugin/api" kubeadm "github.com/Azure/aks-flex/plugin/pkg/services/agentpools/api/features/kubeadm" - wireguard "github.com/Azure/aks-flex/plugin/pkg/services/agentpools/api/features/wireguard" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" timestamppb "google.golang.org/protobuf/types/known/timestamppb" @@ -152,7 +151,6 @@ type AgentPoolSpec struct { xxx_hidden_ImageFamily *string `protobuf:"bytes,6,opt,name=image_family,json=imageFamily"` xxx_hidden_OsDiskSizeGibibytes int64 `protobuf:"varint,7,opt,name=os_disk_size_gibibytes,json=osDiskSizeGibibytes"` xxx_hidden_Kubeadm *kubeadm.Config `protobuf:"bytes,8,opt,name=kubeadm"` - xxx_hidden_Wireguard *wireguard.Config `protobuf:"bytes,9,opt,name=wireguard"` XXX_raceDetectHookData protoimpl.RaceDetectHookData XXX_presence [1]uint32 unknownFields protoimpl.UnknownFields @@ -258,56 +256,45 @@ func (x *AgentPoolSpec) GetKubeadm() *kubeadm.Config { return nil } -func (x *AgentPoolSpec) GetWireguard() *wireguard.Config { - if x != nil { - return x.xxx_hidden_Wireguard - } - return nil -} - func (x *AgentPoolSpec) SetProjectId(v string) { x.xxx_hidden_ProjectId = &v - protoimpl.X.SetPresent(&(x.XXX_presence[0]), 0, 9) + protoimpl.X.SetPresent(&(x.XXX_presence[0]), 0, 8) } func (x *AgentPoolSpec) SetRegion(v string) { x.xxx_hidden_Region = &v - protoimpl.X.SetPresent(&(x.XXX_presence[0]), 1, 9) + protoimpl.X.SetPresent(&(x.XXX_presence[0]), 1, 8) } func (x *AgentPoolSpec) SetSubnetId(v string) { x.xxx_hidden_SubnetId = &v - protoimpl.X.SetPresent(&(x.XXX_presence[0]), 2, 9) + protoimpl.X.SetPresent(&(x.XXX_presence[0]), 2, 8) } func (x *AgentPoolSpec) SetPlatform(v string) { x.xxx_hidden_Platform = &v - protoimpl.X.SetPresent(&(x.XXX_presence[0]), 3, 9) + protoimpl.X.SetPresent(&(x.XXX_presence[0]), 3, 8) } func (x *AgentPoolSpec) SetPreset(v string) { x.xxx_hidden_Preset = &v - protoimpl.X.SetPresent(&(x.XXX_presence[0]), 4, 9) + protoimpl.X.SetPresent(&(x.XXX_presence[0]), 4, 8) } func (x *AgentPoolSpec) SetImageFamily(v string) { x.xxx_hidden_ImageFamily = &v - protoimpl.X.SetPresent(&(x.XXX_presence[0]), 5, 9) + protoimpl.X.SetPresent(&(x.XXX_presence[0]), 5, 8) } func (x *AgentPoolSpec) SetOsDiskSizeGibibytes(v int64) { x.xxx_hidden_OsDiskSizeGibibytes = v - protoimpl.X.SetPresent(&(x.XXX_presence[0]), 6, 9) + protoimpl.X.SetPresent(&(x.XXX_presence[0]), 6, 8) } func (x *AgentPoolSpec) SetKubeadm(v *kubeadm.Config) { x.xxx_hidden_Kubeadm = v } -func (x *AgentPoolSpec) SetWireguard(v *wireguard.Config) { - x.xxx_hidden_Wireguard = v -} - func (x *AgentPoolSpec) HasProjectId() bool { if x == nil { return false @@ -364,13 +351,6 @@ func (x *AgentPoolSpec) HasKubeadm() bool { return x.xxx_hidden_Kubeadm != nil } -func (x *AgentPoolSpec) HasWireguard() bool { - if x == nil { - return false - } - return x.xxx_hidden_Wireguard != nil -} - func (x *AgentPoolSpec) ClearProjectId() { protoimpl.X.ClearPresent(&(x.XXX_presence[0]), 0) x.xxx_hidden_ProjectId = nil @@ -410,10 +390,6 @@ func (x *AgentPoolSpec) ClearKubeadm() { x.xxx_hidden_Kubeadm = nil } -func (x *AgentPoolSpec) ClearWireguard() { - x.xxx_hidden_Wireguard = nil -} - type AgentPoolSpec_builder struct { _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. @@ -425,7 +401,6 @@ type AgentPoolSpec_builder struct { ImageFamily *string OsDiskSizeGibibytes *int64 Kubeadm *kubeadm.Config - Wireguard *wireguard.Config } func (b0 AgentPoolSpec_builder) Build() *AgentPoolSpec { @@ -433,35 +408,34 @@ func (b0 AgentPoolSpec_builder) Build() *AgentPoolSpec { b, x := &b0, m0 _, _ = b, x if b.ProjectId != nil { - protoimpl.X.SetPresentNonAtomic(&(x.XXX_presence[0]), 0, 9) + protoimpl.X.SetPresentNonAtomic(&(x.XXX_presence[0]), 0, 8) x.xxx_hidden_ProjectId = b.ProjectId } if b.Region != nil { - protoimpl.X.SetPresentNonAtomic(&(x.XXX_presence[0]), 1, 9) + protoimpl.X.SetPresentNonAtomic(&(x.XXX_presence[0]), 1, 8) x.xxx_hidden_Region = b.Region } if b.SubnetId != nil { - protoimpl.X.SetPresentNonAtomic(&(x.XXX_presence[0]), 2, 9) + protoimpl.X.SetPresentNonAtomic(&(x.XXX_presence[0]), 2, 8) x.xxx_hidden_SubnetId = b.SubnetId } if b.Platform != nil { - protoimpl.X.SetPresentNonAtomic(&(x.XXX_presence[0]), 3, 9) + protoimpl.X.SetPresentNonAtomic(&(x.XXX_presence[0]), 3, 8) x.xxx_hidden_Platform = b.Platform } if b.Preset != nil { - protoimpl.X.SetPresentNonAtomic(&(x.XXX_presence[0]), 4, 9) + protoimpl.X.SetPresentNonAtomic(&(x.XXX_presence[0]), 4, 8) x.xxx_hidden_Preset = b.Preset } if b.ImageFamily != nil { - protoimpl.X.SetPresentNonAtomic(&(x.XXX_presence[0]), 5, 9) + protoimpl.X.SetPresentNonAtomic(&(x.XXX_presence[0]), 5, 8) x.xxx_hidden_ImageFamily = b.ImageFamily } if b.OsDiskSizeGibibytes != nil { - protoimpl.X.SetPresentNonAtomic(&(x.XXX_presence[0]), 6, 9) + protoimpl.X.SetPresentNonAtomic(&(x.XXX_presence[0]), 6, 8) x.xxx_hidden_OsDiskSizeGibibytes = *b.OsDiskSizeGibibytes } x.xxx_hidden_Kubeadm = b.Kubeadm - x.xxx_hidden_Wireguard = b.Wireguard return m0 } @@ -607,11 +581,11 @@ var File_plugin_pkg_services_agentpools_nebius_instance_agentpools_proto protore const file_plugin_pkg_services_agentpools_nebius_instance_agentpools_proto_rawDesc = "" + "\n" + - "?plugin/pkg/services/agentpools/nebius/instance/agentpools.proto\x12\x1aagentpools.nebius.instance\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x14plugin/api/api.proto\x1aAplugin/pkg/services/agentpools/api/features/kubeadm/kubeadm.proto\x1aEplugin/pkg/services/agentpools/api/features/wireguard/wireguard.proto\"\xba\x01\n" + + "?plugin/pkg/services/agentpools/nebius/instance/agentpools.proto\x12\x1aagentpools.nebius.instance\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x14plugin/api/api.proto\x1aAplugin/pkg/services/agentpools/api/features/kubeadm/kubeadm.proto\"\xba\x01\n" + "\tAgentPool\x12)\n" + "\bmetadata\x18\x01 \x01(\v2\r.api.MetadataR\bmetadata\x12=\n" + "\x04spec\x18\x02 \x01(\v2).agentpools.nebius.instance.AgentPoolSpecR\x04spec\x12C\n" + - "\x06status\x18\x03 \x01(\v2+.agentpools.nebius.instance.AgentPoolStatusR\x06status\"\xcb\x02\n" + + "\x06status\x18\x03 \x01(\v2+.agentpools.nebius.instance.AgentPoolStatusR\x06status\"\x9a\x02\n" + "\rAgentPoolSpec\x12\x1d\n" + "\n" + "project_id\x18\x01 \x01(\tR\tprojectId\x12\x16\n" + @@ -621,8 +595,7 @@ const file_plugin_pkg_services_agentpools_nebius_instance_agentpools_proto_rawDe "\x06preset\x18\x05 \x01(\tR\x06preset\x12!\n" + "\fimage_family\x18\x06 \x01(\tR\vimageFamily\x123\n" + "\x16os_disk_size_gibibytes\x18\a \x01(\x03R\x13osDiskSizeGibibytes\x12)\n" + - "\akubeadm\x18\b \x01(\v2\x0f.kubeadm.ConfigR\akubeadm\x12/\n" + - "\twireguard\x18\t \x01(\v2\x11.wireguard.ConfigR\twireguard\"\x8b\x01\n" + + "\akubeadm\x18\b \x01(\v2\x0f.kubeadm.ConfigR\akubeadm\"\x8b\x01\n" + "\x0fAgentPoolStatus\x12\x1f\n" + "\vinstance_id\x18\x01 \x01(\tR\n" + "instanceId\x12\x1c\n" + @@ -638,21 +611,19 @@ var file_plugin_pkg_services_agentpools_nebius_instance_agentpools_proto_goTypes (*AgentPoolStatus)(nil), // 2: agentpools.nebius.instance.AgentPoolStatus (*api.Metadata)(nil), // 3: api.Metadata (*kubeadm.Config)(nil), // 4: kubeadm.Config - (*wireguard.Config)(nil), // 5: wireguard.Config - (*timestamppb.Timestamp)(nil), // 6: google.protobuf.Timestamp + (*timestamppb.Timestamp)(nil), // 5: google.protobuf.Timestamp } var file_plugin_pkg_services_agentpools_nebius_instance_agentpools_proto_depIdxs = []int32{ 3, // 0: agentpools.nebius.instance.AgentPool.metadata:type_name -> api.Metadata 1, // 1: agentpools.nebius.instance.AgentPool.spec:type_name -> agentpools.nebius.instance.AgentPoolSpec 2, // 2: agentpools.nebius.instance.AgentPool.status:type_name -> agentpools.nebius.instance.AgentPoolStatus 4, // 3: agentpools.nebius.instance.AgentPoolSpec.kubeadm:type_name -> kubeadm.Config - 5, // 4: agentpools.nebius.instance.AgentPoolSpec.wireguard:type_name -> wireguard.Config - 6, // 5: agentpools.nebius.instance.AgentPoolStatus.created_at:type_name -> google.protobuf.Timestamp - 6, // [6:6] is the sub-list for method output_type - 6, // [6:6] is the sub-list for method input_type - 6, // [6:6] is the sub-list for extension type_name - 6, // [6:6] is the sub-list for extension extendee - 0, // [0:6] is the sub-list for field type_name + 5, // 4: agentpools.nebius.instance.AgentPoolStatus.created_at:type_name -> google.protobuf.Timestamp + 5, // [5:5] is the sub-list for method output_type + 5, // [5:5] is the sub-list for method input_type + 5, // [5:5] is the sub-list for extension type_name + 5, // [5:5] is the sub-list for extension extendee + 0, // [0:5] is the sub-list for field type_name } func init() { file_plugin_pkg_services_agentpools_nebius_instance_agentpools_proto_init() } diff --git a/plugin/pkg/services/agentpools/nebius/instance/agentpools.proto b/plugin/pkg/services/agentpools/nebius/instance/agentpools.proto index 01b46a9..e9dacee 100644 --- a/plugin/pkg/services/agentpools/nebius/instance/agentpools.proto +++ b/plugin/pkg/services/agentpools/nebius/instance/agentpools.proto @@ -7,7 +7,6 @@ option go_package = "github.com/Azure/aks-flex/plugin/pkg/services/agentpools/ne import "google/protobuf/timestamp.proto"; import "plugin/api/api.proto"; import "plugin/pkg/services/agentpools/api/features/kubeadm/kubeadm.proto"; -import "plugin/pkg/services/agentpools/api/features/wireguard/wireguard.proto"; message AgentPool { api.Metadata metadata = 1; @@ -26,7 +25,6 @@ message AgentPoolSpec { string image_family = 6; int64 os_disk_size_gibibytes = 7; kubeadm.Config kubeadm = 8; - wireguard.Config wireguard = 9; } message AgentPoolStatus { diff --git a/plugin/pkg/services/agentpools/nebius/instance/assets/wg-spoke.sh b/plugin/pkg/services/agentpools/nebius/instance/assets/wg-spoke.sh deleted file mode 100644 index 89d1d40..0000000 --- a/plugin/pkg/services/agentpools/nebius/instance/assets/wg-spoke.sh +++ /dev/null @@ -1,213 +0,0 @@ -#!/usr/bin/env bash -# -# wg-spoke.sh — WireGuard spoke peer agent for Kubernetes nodes. -# -# Runs on a k8s node host. Generates a WireGuard key pair, publishes the -# public key to the node's annotations via kubectl, then watches for the -# hub node's public key and endpoint. Configures the local WireGuard -# interface and restarts it whenever the hub configuration changes. -# -# Environment variables: -# KUBECONFIG — path to kubeconfig (default: /etc/kubernetes/kubelet.conf) -# NODE_NAME — this node's name (default: $(hostname)) -# WG_INTERFACE — WireGuard interface name (default: wg0) -# WG_ADDRESS — local WireGuard address CIDR (default: 100.96.0.2/32) -# WG_LISTEN_PORT — local listen port (default: 51820) -# WG_ALLOWED_IPS — CIDRs to advertise to the hub (default: WG_ADDRESS) -# HUB_ALLOWED_IPS — CIDRs to route through the hub (default: hub's WG address) -# ANNOTATION_PREFIX — annotation/label prefix (default: wireguard.kube/) -# POLL_INTERVAL — seconds between poll iterations (default: 10) -# WG_DAEMONIZE — if set to "true", fork the poll loop into the background -# so cloud-init or other callers don't block (default: false) -# WG_LOG_FILE — log file path when daemonized (default: /var/log/wg-spoke.log) -# - -set -euo pipefail - -KUBECONFIG="${KUBECONFIG:-/etc/kubernetes/kubelet.conf}" -NODE_NAME="${NODE_NAME:-$(hostname)}" -WG_INTERFACE="${WG_INTERFACE:-wg0}" -WG_ADDRESS="${WG_ADDRESS:-100.96.0.2/32}" -WG_LISTEN_PORT="${WG_LISTEN_PORT:-51820}" -HUB_ALLOWED_IPS="${HUB_ALLOWED_IPS:-}" -ANNOTATION_PREFIX="${ANNOTATION_PREFIX:-wireguard.kube/}" -POLL_INTERVAL="${POLL_INTERVAL:-10}" -WG_DAEMONIZE="${WG_DAEMONIZE:-false}" -WG_LOG_FILE="${WG_LOG_FILE:-/var/log/wg-spoke.log}" - -KUBECTL="kubectl --kubeconfig=${KUBECONFIG}" -WG_CONFIG_DIR="/etc/wireguard" -WG_CONFIG="${WG_CONFIG_DIR}/${WG_INTERFACE}.conf" - -KEY_ANNOTATION="${ANNOTATION_PREFIX}public-key" -ENDPOINT_ANNOTATION="${ANNOTATION_PREFIX}endpoint" -ALLOWED_IPS_ANNOTATION="${ANNOTATION_PREFIX}allowed-ips" -PEER_LABEL="${ANNOTATION_PREFIX}peer" -HUB_LABEL="${ANNOTATION_PREFIX}hub" - -# State tracking for change detection -CURRENT_HUB_KEY="" -CURRENT_HUB_ENDPOINT="" -CURRENT_HUB_ALLOWED_IPS="" - -log() { echo "[$(date -u '+%Y-%m-%dT%H:%M:%SZ')] $*"; } - -# --- Key generation --- - -generate_keys() { - log "Generating WireGuard key pair..." - mkdir -p "${WG_CONFIG_DIR}" - local privkey pubkey - privkey=$(wg genkey) - pubkey=$(echo "${privkey}" | wg pubkey) - echo "${privkey}" > "${WG_CONFIG_DIR}/private.key" - echo "${pubkey}" > "${WG_CONFIG_DIR}/public.key" - chmod 600 "${WG_CONFIG_DIR}/private.key" - log "Public key: ${pubkey}" -} - -# --- Node registration --- - -register_peer() { - local pubkey - pubkey=$(cat "${WG_CONFIG_DIR}/public.key") - - local allowed_ips="${WG_ALLOWED_IPS:-${WG_ADDRESS}}" - - log "Registering as peer on node ${NODE_NAME}..." - ${KUBECTL} label node "${NODE_NAME}" "${PEER_LABEL}=true" --overwrite - ${KUBECTL} annotate node "${NODE_NAME}" \ - "${KEY_ANNOTATION}=${pubkey}" \ - "${ALLOWED_IPS_ANNOTATION}=${allowed_ips}" \ - --overwrite - log "Peer registered" -} - -# --- Hub discovery --- - -# discover_hub finds the hub node by label and returns its name. -discover_hub() { - ${KUBECTL} get nodes -l "${HUB_LABEL}=true" \ - -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true -} - -# get_node_annotation reads an annotation from a given node. -get_node_annotation() { - local node="$1" - local annotation="$2" - # Escape dots and slashes for jsonpath (both are special characters) - local escaped="${annotation//\./\\.}" - escaped="${escaped//\//\\/}" - ${KUBECTL} get node "${node}" \ - -o jsonpath="{.metadata.annotations.${escaped}}" \ - 2>/dev/null || true -} - -# --- WireGuard config --- - -write_config() { - local hub_key="$1" - local hub_endpoint="$2" - local hub_allowed_ips="$3" - local privkey - privkey=$(cat "${WG_CONFIG_DIR}/private.key") - - mkdir -p "${WG_CONFIG_DIR}" - cat > "${WG_CONFIG}" </dev/null || true - wg-quick up "${WG_INTERFACE}" - log "WireGuard interface ${WG_INTERFACE} is up" -} - -# --- Main loop --- - -poll_loop() { - log "Watching for hub node (label ${HUB_LABEL}=true), polling every ${POLL_INTERVAL}s..." - - while true; do - hub_node=$(discover_hub) - if [[ -z "${hub_node}" ]]; then - log "No hub node found (looking for label ${HUB_LABEL}=true), waiting..." - sleep "${POLL_INTERVAL}" - continue - fi - - hub_key=$(get_node_annotation "${hub_node}" "${KEY_ANNOTATION}") - hub_endpoint=$(get_node_annotation "${hub_node}" "${ENDPOINT_ANNOTATION}") - - if [[ -z "${hub_key}" || -z "${hub_endpoint}" ]]; then - log "Hub node ${hub_node} not ready yet (key=${hub_key:-}, endpoint=${hub_endpoint:-}), waiting..." - sleep "${POLL_INTERVAL}" - continue - fi - - # Resolve what CIDRs to route through the hub: - # 1. Explicit HUB_ALLOWED_IPS env var (user override) - # 2. Hub's allowed-ips annotation (hub advertises its own CIDRs) - # 3. Hub's WireGuard address from the address annotation - if [[ -n "${HUB_ALLOWED_IPS}" ]]; then - hub_allowed_ips="${HUB_ALLOWED_IPS}" - else - hub_allowed_ips=$(get_node_annotation "${hub_node}" "${ALLOWED_IPS_ANNOTATION}") - if [[ -z "${hub_allowed_ips}" ]]; then - hub_allowed_ips="${WG_ADDRESS%.*}.1/32" - log "Hub has no allowed-ips annotation, defaulting to ${hub_allowed_ips}" - fi - fi - - if [[ "${hub_key}" != "${CURRENT_HUB_KEY}" || "${hub_endpoint}" != "${CURRENT_HUB_ENDPOINT}" || "${hub_allowed_ips}" != "${CURRENT_HUB_ALLOWED_IPS}" ]]; then - log "Hub config changed: node=${hub_node} key=${hub_key} endpoint=${hub_endpoint} allowed-ips=${hub_allowed_ips}" - write_config "${hub_key}" "${hub_endpoint}" "${hub_allowed_ips}" - restart_wg - CURRENT_HUB_KEY="${hub_key}" - CURRENT_HUB_ENDPOINT="${hub_endpoint}" - CURRENT_HUB_ALLOWED_IPS="${hub_allowed_ips}" - fi - - sleep "${POLL_INTERVAL}" - done -} - -main() { - generate_keys - register_peer - - if [[ "${WG_DAEMONIZE}" == "true" ]]; then - log "Daemonizing poll loop (log: ${WG_LOG_FILE})..." - poll_loop >> "${WG_LOG_FILE}" 2>&1 & - local pid=$! - echo "${pid}" > /var/run/wg-spoke.pid - log "Poll loop running in background (PID ${pid})" - # Disown so the shell can exit without waiting for the child - disown "${pid}" - else - poll_loop - fi -} - -# Handle shutdown -cleanup() { - log "Shutting down..." - wg-quick down "${WG_INTERFACE}" 2>/dev/null || true - rm -f /var/run/wg-spoke.pid - exit 0 -} -trap cleanup SIGINT SIGTERM - -main diff --git a/plugin/pkg/services/networks/api/features/vnet/vnet.pb.go b/plugin/pkg/services/networks/api/features/vnet/vnet.pb.go index 3cf62ce..6f3525c 100644 --- a/plugin/pkg/services/networks/api/features/vnet/vnet.pb.go +++ b/plugin/pkg/services/networks/api/features/vnet/vnet.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.36.11 -// protoc v6.33.4 +// protoc v7.34.0 // source: plugin/pkg/services/networks/api/features/vnet/vnet.proto package vnet diff --git a/plugin/pkg/services/networks/api/networks.pb.go b/plugin/pkg/services/networks/api/networks.pb.go index 97c4d13..32fb5b2 100644 --- a/plugin/pkg/services/networks/api/networks.pb.go +++ b/plugin/pkg/services/networks/api/networks.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.36.11 -// protoc v6.33.4 +// protoc v7.34.0 // source: plugin/pkg/services/networks/api/networks.proto package api diff --git a/plugin/pkg/services/networks/api/networks_grpc.pb.go b/plugin/pkg/services/networks/api/networks_grpc.pb.go index d94e3a3..707fd22 100644 --- a/plugin/pkg/services/networks/api/networks_grpc.pb.go +++ b/plugin/pkg/services/networks/api/networks_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.6.1 -// - protoc v6.33.4 +// - protoc v7.34.0 // source: plugin/pkg/services/networks/api/networks.proto package api diff --git a/plugin/pkg/services/networks/aws/vpc/vpc.pb.go b/plugin/pkg/services/networks/aws/vpc/vpc.pb.go index 842ae03..28feb35 100644 --- a/plugin/pkg/services/networks/aws/vpc/vpc.pb.go +++ b/plugin/pkg/services/networks/aws/vpc/vpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.36.11 -// protoc v6.33.4 +// protoc v7.34.0 // source: plugin/pkg/services/networks/aws/vpc/vpc.proto package vpc diff --git a/plugin/pkg/services/networks/nebius/vpc/vpc.pb.go b/plugin/pkg/services/networks/nebius/vpc/vpc.pb.go index 42d8667..1d2a825 100644 --- a/plugin/pkg/services/networks/nebius/vpc/vpc.pb.go +++ b/plugin/pkg/services/networks/nebius/vpc/vpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.36.11 -// protoc v6.33.4 +// protoc v7.34.0 // source: plugin/pkg/services/networks/nebius/vpc/vpc.proto package vpc diff --git a/plugin/pkg/services/peerings/api/features/ipsec/ipsec.pb.go b/plugin/pkg/services/peerings/api/features/ipsec/ipsec.pb.go index 761464b..c7db991 100644 --- a/plugin/pkg/services/peerings/api/features/ipsec/ipsec.pb.go +++ b/plugin/pkg/services/peerings/api/features/ipsec/ipsec.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.36.11 -// protoc v6.33.4 +// protoc v7.34.0 // source: plugin/pkg/services/peerings/api/features/ipsec/ipsec.proto package ipsec diff --git a/plugin/pkg/services/peerings/api/peerings.pb.go b/plugin/pkg/services/peerings/api/peerings.pb.go index ff81bd0..17ffcba 100644 --- a/plugin/pkg/services/peerings/api/peerings.pb.go +++ b/plugin/pkg/services/peerings/api/peerings.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.36.11 -// protoc v6.33.4 +// protoc v7.34.0 // source: plugin/pkg/services/peerings/api/peerings.proto package api diff --git a/plugin/pkg/services/peerings/api/peerings_grpc.pb.go b/plugin/pkg/services/peerings/api/peerings_grpc.pb.go index 3119ea7..94567ee 100644 --- a/plugin/pkg/services/peerings/api/peerings_grpc.pb.go +++ b/plugin/pkg/services/peerings/api/peerings_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.6.1 -// - protoc v6.33.4 +// - protoc v7.34.0 // source: plugin/pkg/services/peerings/api/peerings.proto package api diff --git a/plugin/pkg/services/peerings/aws/ipsecvpn/ipsecvpn.pb.go b/plugin/pkg/services/peerings/aws/ipsecvpn/ipsecvpn.pb.go index eb97099..8daf39f 100644 --- a/plugin/pkg/services/peerings/aws/ipsecvpn/ipsecvpn.pb.go +++ b/plugin/pkg/services/peerings/aws/ipsecvpn/ipsecvpn.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.36.11 -// protoc v6.33.4 +// protoc v7.34.0 // source: plugin/pkg/services/peerings/aws/ipsecvpn/ipsecvpn.proto package ipsecvpn diff --git a/plugin/pkg/util/kubeadm/kubeadm.go b/plugin/pkg/util/kubeadm/kubeadm.go index 8671762..8718ace 100644 --- a/plugin/pkg/util/kubeadm/kubeadm.go +++ b/plugin/pkg/util/kubeadm/kubeadm.go @@ -68,7 +68,7 @@ func JoinConfig(cfg *kubeadm.Config, kubeConfigPath string) ([]byte, error) { }) } - // Add --node-ip if configured (for WireGuard nodes) + // Add --node-ip if configured if cfg.HasNodeIp() { kubeletArgs = append(kubeletArgs, upstreamv1beta4.Arg{ Name: "node-ip", diff --git a/plugin/pkg/util/wireguard/wireguard.go b/plugin/pkg/util/wireguard/wireguard.go deleted file mode 100644 index a1cb392..0000000 --- a/plugin/pkg/util/wireguard/wireguard.go +++ /dev/null @@ -1,110 +0,0 @@ -package wireguard - -import ( - "crypto/rand" - "encoding/base64" - "fmt" - "strings" - - "golang.org/x/crypto/curve25519" -) - -// KeyPair represents a WireGuard key pair. -type KeyPair struct { - PrivateKey string - PublicKey string -} - -// GenerateKeyPair generates a new WireGuard key pair. -// ref: https://github.com/WireGuard/wireguard-tools/blob/0b7d9821f2815973a2930ace28a3f73c205d0e5c/src/genkey.c#L75 -func GenerateKeyPair() (*KeyPair, error) { - // Generate 32 random bytes for private key - var privateKey [32]byte - if _, err := rand.Read(privateKey[:]); err != nil { - return nil, fmt.Errorf("failed to generate random bytes: %w", err) - } - - // Clamp the private key per WireGuard spec (Curve25519) - privateKey[0] &= 248 - privateKey[31] &= 127 - privateKey[31] |= 64 - - // Derive public key from private key - var publicKey [32]byte - curve25519.ScalarBaseMult(&publicKey, &privateKey) - - return &KeyPair{ - PrivateKey: base64.StdEncoding.EncodeToString(privateKey[:]), - PublicKey: base64.StdEncoding.EncodeToString(publicKey[:]), - }, nil -} - -// Peer represents a WireGuard peer configuration. -type Peer struct { - PublicKey string - Endpoint string // empty for server waiting for client - AllowedIPs []string - PersistentKeepalive int // seconds, 0 to disable -} - -// Config holds WireGuard configuration for an interface. -type Config struct { - // Interface settings - Address string // e.g., "100.96.0.1/32" - ListenPort int // e.g., 51820 (0 for client mode) - PrivateKey string - - // Peers configuration - Peers []Peer - - // Routes to add via PostUp (e.g., ["172.16.0.0/16", "172.18.0.0/16"]) - Routes []string -} - -// GenerateConfig generates a WireGuard configuration file content. -func GenerateConfig(cfg *Config) string { - var sb strings.Builder - - sb.WriteString(fmt.Sprintf("[Interface]\nAddress = %s\nPrivateKey = %s\n", cfg.Address, cfg.PrivateKey)) - - if cfg.ListenPort > 0 { - sb.WriteString(fmt.Sprintf("ListenPort = %d\n", cfg.ListenPort)) - } - - // Add IP forwarding for gateway mode - if cfg.ListenPort > 0 { - sb.WriteString(`PostUp = sysctl -w net.ipv4.ip_forward=1 -PostUp = iptables -A FORWARD -i wg0 -j ACCEPT -PostUp = iptables -A FORWARD -o wg0 -j ACCEPT -PostDown = iptables -D FORWARD -i wg0 -j ACCEPT -PostDown = iptables -D FORWARD -o wg0 -j ACCEPT -`) - } - - // Add custom routes via PostUp/PostDown - for _, route := range cfg.Routes { - sb.WriteString(fmt.Sprintf("PostUp = ip route add %s dev wg0\n", route)) - sb.WriteString(fmt.Sprintf("PostDown = ip route del %s dev wg0 || true\n", route)) - } - - // Add peers - for _, peer := range cfg.Peers { - sb.WriteString(fmt.Sprintf("\n[Peer]\nPublicKey = %s\n", peer.PublicKey)) - - if peer.Endpoint != "" { - sb.WriteString(fmt.Sprintf("Endpoint = %s\n", peer.Endpoint)) - } - - if len(peer.AllowedIPs) > 0 { - sb.WriteString("AllowedIPs = ") - sb.WriteString(strings.Join(peer.AllowedIPs, ", ")) - sb.WriteString("\n") - } - - if peer.PersistentKeepalive > 0 { - sb.WriteString(fmt.Sprintf("PersistentKeepalive = %d\n", peer.PersistentKeepalive)) - } - } - - return sb.String() -} From d4458da78205442016627fd27f050f426619661a Mon Sep 17 00:00:00 2001 From: hbc Date: Wed, 4 Mar 2026 21:16:47 -0800 Subject: [PATCH 2/4] refactor: remove wireguard references in cli --- cli/README.md | 2 +- cli/go.mod | 2 +- cli/internal/aks/deploy/assets/aks.json | 94 ----- .../deploy/assets/wireguard-deployment.yaml | 125 ------- cli/internal/aks/deploy/deploy.go | 20 +- cli/internal/aks/deploy/wireguard.go | 350 ------------------ cli/internal/config/agentpools/nebius.go | 4 - 7 files changed, 3 insertions(+), 594 deletions(-) delete mode 100644 cli/internal/aks/deploy/assets/wireguard-deployment.yaml delete mode 100644 cli/internal/aks/deploy/wireguard.go diff --git a/cli/README.md b/cli/README.md index f5fb546..b3cec21 100644 --- a/cli/README.md +++ b/cli/README.md @@ -15,7 +15,7 @@ $ aks-flex-cli network deploy ## Initializing AKS Cluster ``` -$ aks-flex-cli aks deploy --cilium --wireguard +$ aks-flex-cli aks deploy --cilium ``` ## Initializing Remote Cloud Network diff --git a/cli/go.mod b/cli/go.mod index ab6bbaa..dfa582f 100644 --- a/cli/go.mod +++ b/cli/go.mod @@ -8,7 +8,6 @@ require ( github.com/Azure/aks-flex/plugin v0.0.0-00010101000000-000000000000 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 - github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v8 v8.0.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources/v2 v2.1.0 github.com/joho/godotenv v1.5.1 github.com/nebius/gosdk v0.0.0-20260218100913-7fb27c45819a @@ -27,6 +26,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v7 v7.3.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v8 v8.2.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v8 v8.0.0 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 // indirect github.com/aws/aws-sdk-go-v2 v1.41.1 // indirect github.com/aws/aws-sdk-go-v2/config v1.32.9 // indirect diff --git a/cli/internal/aks/deploy/assets/aks.json b/cli/internal/aks/deploy/assets/aks.json index 8f7e52a..22fb51a 100644 --- a/cli/internal/aks/deploy/assets/aks.json +++ b/cli/internal/aks/deploy/assets/aks.json @@ -22,10 +22,6 @@ "type": "string", "defaultValue": "Standard_D16ds_v5" }, - "deployWireguard": { - "type": "bool", - "defaultValue": false - }, "deployUnboundedCNI": { "type": "bool", "defaultValue": false @@ -171,81 +167,6 @@ "[variables('karpenterMIId')]" ] }, - { - "condition": "[parameters('deployWireguard')]", - "apiVersion": "2023-09-01", - "type": "Microsoft.Network/networkSecurityGroups/securityRules", - "name": "nsg/AllowWireGuard", - "properties": { - "priority": 100, - "direction": "Inbound", - "access": "Allow", - "protocol": "Udp", - "sourcePortRange": "*", - "destinationPortRange": "51820", - "sourceAddressPrefix": "*", - "destinationAddressPrefix": "*" - } - }, - { - "condition": "[parameters('deployWireguard')]", - "apiVersion": "2023-09-01", - "type": "Microsoft.Network/publicIPPrefixes", - "name": "wg-pips", - "location": "[resourceGroup().location]", - "sku": { - "name": "Standard" - }, - "properties": { - "prefixLength": 31, - "publicIPAddressVersion": "IPv4" - } - }, - { - "condition": "[parameters('deployWireguard')]", - "apiVersion": "2025-10-01", - "type": "Microsoft.ContainerService/managedClusters/agentPools", - "name": "[concat(parameters('clusterName'), '/wireguard')]", - "properties": { - "count": 1, - "vmSize": "[parameters('gatewayVMSize')]", - "mode": "User", - "osType": "Linux", - "vnetSubnetId": "[resourceId('Microsoft.Network/virtualNetworks/subnets', 'vnet', 'nodes')]", - "enableNodePublicIP": true, - "nodePublicIPPrefixID": "[resourceId('Microsoft.Network/publicIPPrefixes', 'wg-pips')]", - "networkProfile": { - "allowedHostPorts": [ - { - "portStart": 51820, - "portEnd": 51820, - "protocol": "UDP" - } - ] - }, - "nodeLabels": { - "stretch.azure.com/wireguard-gateway": "true", - "stretch.azure.com/wireguard-hub": "true" - }, - "nodeTaints": [ - "stretch.azure.com/wireguard-gateway=true:NoSchedule" - ] - }, - "dependsOn": [ - "[resourceId('Microsoft.ContainerService/managedClusters', parameters('clusterName'))]", - "[resourceId('Microsoft.Network/publicIPPrefixes', 'wg-pips')]" - ] - }, - { - "condition": "[parameters('deployWireguard')]", - "apiVersion": "2023-09-01", - "type": "Microsoft.Network/routeTables", - "name": "wg-routes", - "location": "[resourceGroup().location]", - "properties": { - "routes": [] - } - }, { "condition": "[parameters('deployUnboundedCNI')]", "apiVersion": "2023-09-01", @@ -321,21 +242,6 @@ "oidcIssuerUrl": { "type": "string", "value": "[reference(variables('aksClusterId'), '2024-01-01').oidcIssuerProfile.issuerUrl]" - }, - "nodePoolName": { - "condition": "[parameters('deployWireguard')]", - "type": "string", - "value": "wireguard" - }, - "publicIpPrefixId": { - "condition": "[parameters('deployWireguard')]", - "type": "string", - "value": "[resourceId('Microsoft.Network/publicIPPrefixes', 'wg-pips')]" - }, - "routeTableId": { - "condition": "[parameters('deployWireguard')]", - "type": "string", - "value": "[resourceId('Microsoft.Network/routeTables', 'wg-routes')]" } } } diff --git a/cli/internal/aks/deploy/assets/wireguard-deployment.yaml b/cli/internal/aks/deploy/assets/wireguard-deployment.yaml deleted file mode 100644 index a13bbcb..0000000 --- a/cli/internal/aks/deploy/assets/wireguard-deployment.yaml +++ /dev/null @@ -1,125 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: wireguard ---- -apiVersion: v1 -kind: Secret -metadata: - name: wireguard-keys - namespace: wireguard -type: Opaque -data: - private.key: {{ .privateKeyBase64 }} - public.key: {{ .publicKeyBase64 }} ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: wg-kube - namespace: wireguard ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: wg-kube -rules: - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get", "list", "watch", "patch"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: wg-kube -subjects: - - kind: ServiceAccount - name: wg-kube - namespace: wireguard -roleRef: - kind: ClusterRole - name: wg-kube - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: wg-kube - namespace: wireguard - labels: - app: wg-kube -spec: - selector: - matchLabels: - app: wg-kube - template: - metadata: - labels: - app: wg-kube - spec: - hostNetwork: true - dnsPolicy: ClusterFirstWithHostNet - serviceAccountName: wg-kube - nodeSelector: - stretch.azure.com/wireguard-hub: "true" - tolerations: - - key: stretch.azure.com/wireguard-gateway - operator: Equal - value: "true" - effect: NoSchedule - initContainers: - - name: load-module - image: alpine:3.21 - command: ["modprobe", "wireguard"] - securityContext: - privileged: true - volumeMounts: - - name: lib-modules - mountPath: /lib/modules - readOnly: true - containers: - - name: wg-kube - image: {{ .wgKubeImage }} - args: - - --annotation-prefix=stretch.azure.com/wireguard- - - --listen-port=51820 - - --address=100.96.0.1/32 - - --interface=wg0 - - --private-key-file=/keys/private.key - # Advertise the WireGuard overlay (100.96.0.0/12) and the Azure - # VNet range (172.16.0.0/16) so spokes route both through the hub. - - --allowed-ips=100.96.0.0/12,172.16.0.0/16 - env: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - securityContext: - privileged: true - ports: - - containerPort: 51820 - hostPort: 51820 - protocol: UDP - name: wireguard - resources: - requests: - memory: "64Mi" - cpu: "100m" - limits: - memory: "256Mi" - cpu: "500m" - volumeMounts: - - name: wireguard-keys - mountPath: /keys - readOnly: true - - name: lib-modules - mountPath: /lib/modules - readOnly: true - volumes: - - name: wireguard-keys - secret: - secretName: wireguard-keys - - name: lib-modules - hostPath: - path: /lib/modules - type: Directory diff --git a/cli/internal/aks/deploy/deploy.go b/cli/internal/aks/deploy/deploy.go index 67fae5a..025386d 100644 --- a/cli/internal/aks/deploy/deploy.go +++ b/cli/internal/aks/deploy/deploy.go @@ -27,9 +27,6 @@ var ( if deploycilium { return fmt.Errorf("--cilium cannot be used with --unbounded-cni") } - if deployWireguard { - return fmt.Errorf("--wireguard cannot be used with --unbounded-cni") - } } return nil }, @@ -39,7 +36,6 @@ var ( } deploycilium bool - deployWireguard bool unboundedCNI bool deployGPUOperator bool deployGPUDevicePlugin bool @@ -52,8 +48,7 @@ var ( func init() { Command.Flags().BoolVar(&deploycilium, "cilium", false, "deploy Cilium CNI") // default to true to allow minimal networking to work - Command.Flags().BoolVar(&deployWireguard, "wireguard", false, "deploy WireGuard gateway node pool and DaemonSet") - Command.Flags().BoolVar(&unboundedCNI, "unbounded-cni", false, "deploy unbounded cni (mutually exclusive with --cilium and --wireguard)") + Command.Flags().BoolVar(&unboundedCNI, "unbounded-cni", false, "deploy unbounded cni (mutually exclusive with --cilium)") Command.Flags().BoolVar(&deployGPUOperator, "gpu-operator", false, "install NVIDIA GPU Operator via Helm") Command.Flags().BoolVar(&deployGPUDevicePlugin, "gpu-device-plugin", false, "install NVIDIA GPU Device Plugin via Helm") Command.Flags().BoolVar(&skipARM, "skip-arm", false, "skip the ARM template deployment step") @@ -116,12 +111,6 @@ func run(ctx context.Context) error { "vmSize": { Value: cfg.SystemVMSize, }, - "gatewayVMSize": { - Value: cfg.GatewayVMSize, - }, - "deployWireguard": { - Value: deployWireguard, - }, "deployUnboundedCNI": { Value: unboundedCNI, }, @@ -150,13 +139,6 @@ func run(ctx context.Context) error { log.Printf("Cilium deployment complete") } - if deployWireguard { - if err := deployWireGuard(ctx, credentials, cfg); err != nil { - return err - } - log.Printf("WireGuard deployment complete") - } - if unboundedCNI { if err := unboundedcni.Deploy(ctx, kubeconfigPath, cfg); err != nil { return err diff --git a/cli/internal/aks/deploy/wireguard.go b/cli/internal/aks/deploy/wireguard.go deleted file mode 100644 index fb61e42..0000000 --- a/cli/internal/aks/deploy/wireguard.go +++ /dev/null @@ -1,350 +0,0 @@ -package deploy - -import ( - "bytes" - "context" - _ "embed" - "encoding/base64" - "fmt" - "log" - "text/template" - "time" - - "github.com/Azure/azure-sdk-for-go/sdk/azcore" - "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" - "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v8" - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" - - utilconfig "github.com/Azure/aks-flex/plugin/pkg/util/config" - "github.com/Azure/aks-flex/plugin/pkg/util/k8s" - "github.com/Azure/aks-flex/plugin/pkg/util/wireguard" -) - -var ( - //go:embed assets/wireguard-deployment.yaml - wireguardDeploymentTemplate string -) - -const ( - // wgKubeImage is the container image for the wg-kube controller. - // NOTE: this is a temporary workaround for setting up sites-to-sites via WireGuard. - // We are working on a more robust CNI based implementation and will replace - // with that when it's ready. - wgKubeImage = "ghcr.io/b4fun/wg-kube:sha-11e4656" -) - -func deployWireGuard(ctx context.Context, credentials azcore.TokenCredential, cfg *utilconfig.Config) error { - // Step 1: Get or generate WireGuard keys for the hub - log.Print("Getting WireGuard keys...") - - keys, err := getOrCreateWireGuardKeys(ctx, credentials, cfg) - if err != nil { - return fmt.Errorf("failed to get WireGuard keys: %w", err) - } - - log.Printf(" Public Key: %s", keys.PublicKey) - - // Step 2: Get the WireGuard gateway node's public IP (retry until the node registers) - log.Print("Waiting for WireGuard gateway node to register...") - var gatewayIP, gatewayPrivateIP string - for { - var err error - gatewayIP, gatewayPrivateIP, err = getWireGuardNodeIP(ctx, credentials, cfg) - if err == nil { - break - } - log.Printf(" %v, retrying in 15s...", err) - select { - case <-ctx.Done(): - return ctx.Err() - case <-time.After(15 * time.Second): - } - } - - log.Printf("WireGuard gateway node ready") - log.Printf(" Public IP: %s", gatewayIP) - log.Printf(" Private IP: %s", gatewayPrivateIP) - - // Step 3: Update route table with the gateway node's private IP - log.Print("Updating route table...") - if err := updateRouteTable(ctx, credentials, cfg, gatewayPrivateIP); err != nil { - return fmt.Errorf("failed to update route table: %w", err) - } - - // Step 4: Associate route table with subnets - log.Print("Associating route table with subnets...") - if err := associateRouteTableWithSubnets(ctx, credentials, cfg); err != nil { - return fmt.Errorf("failed to associate route table with subnets: %w", err) - } - - // Step 5: Deploy WireGuard DaemonSet to Kubernetes - log.Print("Deploying WireGuard DaemonSet...") - if err := deployWireGuardToK8s(ctx, credentials, cfg, keys); err != nil { - return fmt.Errorf("failed to deploy WireGuard to Kubernetes: %w", err) - } - - return nil -} - -// getOrCreateWireGuardKeys checks if the wireguard-keys secret exists and returns those keys, -// otherwise generates new keys. -func getOrCreateWireGuardKeys(ctx context.Context, credentials azcore.TokenCredential, cfg *utilconfig.Config) (*wireguard.KeyPair, error) { - loader, err := k8s.Loader(ctx, credentials, cfg) - if err != nil { - return nil, err - } - - restconfig, err := loader.ClientConfig() - if err != nil { - return nil, err - } - - cli, err := client.New(restconfig, client.Options{}) - if err != nil { - return nil, err - } - - // Try to get existing secret - secret := &corev1.Secret{} - err = cli.Get(ctx, types.NamespacedName{ - Namespace: "wireguard", - Name: "wireguard-keys", - }, secret) - - if err == nil { - // Secret exists, extract keys - privateKey, ok := secret.Data["private.key"] - if !ok { - return nil, fmt.Errorf("wireguard-keys secret missing private.key") - } - publicKey, ok := secret.Data["public.key"] - if !ok { - return nil, fmt.Errorf("wireguard-keys secret missing public.key") - } - - log.Print(" Using existing WireGuard keys from secret") - return &wireguard.KeyPair{ - PrivateKey: string(privateKey), - PublicKey: string(publicKey), - }, nil - } - - if !apierrors.IsNotFound(err) { - return nil, fmt.Errorf("failed to get wireguard-keys secret: %w", err) - } - - // Secret doesn't exist, generate new keys - log.Print(" Generating new WireGuard keys") - return wireguard.GenerateKeyPair() -} - -// getWireGuardNodeIP retrieves the public and private IP of the WireGuard gateway node from Kubernetes. -func getWireGuardNodeIP(ctx context.Context, credentials azcore.TokenCredential, cfg *utilconfig.Config) (publicIP, privateIP string, err error) { - loader, err := k8s.Loader(ctx, credentials, cfg) - if err != nil { - return "", "", err - } - - restconfig, err := loader.ClientConfig() - if err != nil { - return "", "", err - } - - cli, err := client.New(restconfig, client.Options{}) - if err != nil { - return "", "", err - } - - // List nodes with the wireguard gateway label - nodeList := &unstructured.UnstructuredList{} - nodeList.SetGroupVersionKind(schema.GroupVersionKind{ - Group: "", - Version: "v1", - Kind: "NodeList", - }) - - if err := cli.List(ctx, nodeList, client.MatchingLabels{ - "stretch.azure.com/wireguard-gateway": "true", - }); err != nil { - return "", "", fmt.Errorf("failed to list wireguard nodes: %w", err) - } - - if len(nodeList.Items) == 0 { - return "", "", fmt.Errorf("no wireguard gateway nodes found (waiting for node to register)") - } - - node := nodeList.Items[0] - - // Extract addresses from the node status - addresses, found, err := unstructured.NestedSlice(node.Object, "status", "addresses") - if err != nil || !found { - return "", "", fmt.Errorf("failed to get node addresses: %w", err) - } - - for _, addr := range addresses { - addrMap, ok := addr.(map[string]any) - if !ok { - continue - } - addrType, _, err := unstructured.NestedString(addrMap, "type") - if err != nil { - continue - } - addrValue, _, err := unstructured.NestedString(addrMap, "address") - if err != nil { - continue - } - - switch addrType { - case "ExternalIP": - publicIP = addrValue - case "InternalIP": - privateIP = addrValue - } - } - - if publicIP == "" { - return "", "", fmt.Errorf("wireguard node has no ExternalIP (public IP not assigned yet)") - } - if privateIP == "" { - return "", "", fmt.Errorf("wireguard node has no InternalIP") - } - - return publicIP, privateIP, nil -} - -// updateRouteTable updates the route table with the gateway node's private IP. -func updateRouteTable(ctx context.Context, credentials azcore.TokenCredential, cfg *utilconfig.Config, gatewayPrivateIP string) error { - routeTablesClient, err := armnetwork.NewRouteTablesClient(cfg.SubscriptionID, credentials, nil) - if err != nil { - return err - } - - routeTable, err := routeTablesClient.Get(ctx, cfg.ResourceGroupName, "wg-routes", nil) - if err != nil { - return fmt.Errorf("failed to get route table: %w", err) - } - - // Update routes to point to the gateway node - routeTable.Properties.Routes = []*armnetwork.Route{ - { - Name: to.Ptr("to-nebius-wg"), - Properties: &armnetwork.RoutePropertiesFormat{ - AddressPrefix: to.Ptr("100.96.0.0/12"), - NextHopType: to.Ptr(armnetwork.RouteNextHopTypeVirtualAppliance), - NextHopIPAddress: to.Ptr(gatewayPrivateIP), - }, - }, - } - - poller, err := routeTablesClient.BeginCreateOrUpdate(ctx, cfg.ResourceGroupName, "wg-routes", routeTable.RouteTable, nil) - if err != nil { - return fmt.Errorf("failed to update route table: %w", err) - } - - if _, err := poller.PollUntilDone(ctx, nil); err != nil { - return fmt.Errorf("failed to wait for route table update: %w", err) - } - - log.Printf(" Route table updated with gateway IP: %s", gatewayPrivateIP) - return nil -} - -// associateRouteTableWithSubnets associates the wg-routes route table with the aks and nodes subnets. -func associateRouteTableWithSubnets(ctx context.Context, credentials azcore.TokenCredential, cfg *utilconfig.Config) error { - subnetsClient, err := armnetwork.NewSubnetsClient(cfg.SubscriptionID, credentials, nil) - if err != nil { - return err - } - - routeTableID := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/routeTables/wg-routes", - cfg.SubscriptionID, cfg.ResourceGroupName) - - // Subnets to update - subnets := []string{"aks", "nodes"} - - for _, subnetName := range subnets { - // Get current subnet configuration - subnet, err := subnetsClient.Get(ctx, cfg.ResourceGroupName, "vnet", subnetName, nil) - if err != nil { - return fmt.Errorf("failed to get subnet %s: %w", subnetName, err) - } - - // Skip if route table is already associated - if subnet.Properties.RouteTable != nil && subnet.Properties.RouteTable.ID != nil { - if *subnet.Properties.RouteTable.ID == routeTableID { - log.Printf(" Route table already associated with subnet %s", subnetName) - continue - } - } - - // Associate route table - subnet.Properties.RouteTable = &armnetwork.RouteTable{ - ID: to.Ptr(routeTableID), - } - - log.Printf(" Associating route table with subnet %s...", subnetName) - poller, err := subnetsClient.BeginCreateOrUpdate(ctx, cfg.ResourceGroupName, "vnet", subnetName, subnet.Subnet, nil) - if err != nil { - return fmt.Errorf("failed to update subnet %s: %w", subnetName, err) - } - - if _, err := poller.PollUntilDone(ctx, nil); err != nil { - return fmt.Errorf("failed to wait for subnet %s update: %w", subnetName, err) - } - - log.Printf(" Route table associated with subnet %s", subnetName) - } - - return nil -} - -// deployWireGuardToK8s deploys the WireGuard DaemonSet to the AKS cluster. -func deployWireGuardToK8s( - ctx context.Context, - credentials azcore.TokenCredential, - cfg *utilconfig.Config, - keys *wireguard.KeyPair, -) error { - loader, err := k8s.Loader(ctx, credentials, cfg) - if err != nil { - return err - } - - restconfig, err := loader.ClientConfig() - if err != nil { - return err - } - - cli, err := client.New(restconfig, client.Options{}) - if err != nil { - return err - } - - // Template the YAML with keys and wg-kube image - tmpl, err := template.New("wireguard").Parse(wireguardDeploymentTemplate) - if err != nil { - return fmt.Errorf("failed to parse WireGuard deployment template: %w", err) - } - - var buf bytes.Buffer - if err := tmpl.Execute(&buf, map[string]string{ - "privateKeyBase64": base64.StdEncoding.EncodeToString([]byte(keys.PrivateKey)), - "publicKeyBase64": base64.StdEncoding.EncodeToString([]byte(keys.PublicKey)), - "wgKubeImage": wgKubeImage, - }); err != nil { - return fmt.Errorf("failed to execute WireGuard deployment template: %w", err) - } - - if err := k8s.ApplyYAMLSpec(ctx, cli, &buf, "stretch"); err != nil { - return fmt.Errorf("failed to apply WireGuard deployment YAML: %w", err) - } - - log.Printf(" WireGuard DaemonSet deployed successfully") - return nil -} diff --git a/cli/internal/config/agentpools/nebius.go b/cli/internal/config/agentpools/nebius.go index 0fa96be..f41f4ff 100644 --- a/cli/internal/config/agentpools/nebius.go +++ b/cli/internal/config/agentpools/nebius.go @@ -8,7 +8,6 @@ import ( "github.com/Azure/aks-flex/cli/internal/config/configcmd" "github.com/Azure/aks-flex/plugin/api" - "github.com/Azure/aks-flex/plugin/pkg/services/agentpools/api/features/wireguard" nebiusap "github.com/Azure/aks-flex/plugin/pkg/services/agentpools/nebius/instance" ) @@ -33,9 +32,6 @@ func newNebiusAgentPool(ctx context.Context) proto.Message { ImageFamily: to.Ptr(configcmd.OrPlaceholder("")), OsDiskSizeGibibytes: to.Ptr(int64(128)), Kubeadm: configcmd.DefaultKubeadmConfig(ctx), - Wireguard: wireguard.Config_builder{ - PeerIp: to.Ptr(configcmd.OrPlaceholder("")), - }.Build(), }.Build(), }.Build() } From ed5cee5f01f6f39ff6b048b16f524f0e2600b204 Mon Sep 17 00:00:00 2001 From: hbc Date: Wed, 4 Mar 2026 21:34:46 -0800 Subject: [PATCH 3/4] refactor: drop wireguard from karpenter --- karpenter/cmd/controller/main.go | 9 - karpenter/examples/nebius/nodeclass.yaml | 1 - .../flex.aks.azure.com_nebiusnodeclasses.yaml | 6 - karpenter/pkg/apis/v1alpha1/nebius.go | 6 - .../apis/v1alpha1/zz_generated.deepcopy.go | 5 - .../pkg/cloudproviders/kaito/cloudprovider.go | 13 +- .../pkg/cloudproviders/kaito/nodeclaim.go | 19 - .../cloudproviders/nebius/cloudprovider.go | 30 +- .../pkg/cloudproviders/nebius/nodeclaim.go | 3 - karpenter/pkg/options/kaito.go | 12 +- karpenter/pkg/utils/wireguard/allocator.go | 247 ---------- .../pkg/utils/wireguard/allocator_test.go | 465 ------------------ 12 files changed, 12 insertions(+), 804 deletions(-) delete mode 100644 karpenter/pkg/utils/wireguard/allocator.go delete mode 100644 karpenter/pkg/utils/wireguard/allocator_test.go diff --git a/karpenter/cmd/controller/main.go b/karpenter/cmd/controller/main.go index d36ad92..85ccd9d 100644 --- a/karpenter/cmd/controller/main.go +++ b/karpenter/cmd/controller/main.go @@ -31,7 +31,6 @@ import ( flexcontrollers "github.com/Azure/aks-flex/karpenter/pkg/controllers" flexoptions "github.com/Azure/aks-flex/karpenter/pkg/options" utilsk8s "github.com/Azure/aks-flex/karpenter/pkg/utils/k8s" - wireguard "github.com/Azure/aks-flex/karpenter/pkg/utils/wireguard" ) func init() { @@ -89,30 +88,22 @@ func main() { // nebius cloud provider... { - wgAlloc := wireguard.NewIPAllocator(op.Manager.GetCache(), nebius.GroupKind, 30*time.Second) - defer wgAlloc.Close() - err := nebius.Register( ctx, hubCloudProvider, flexoptions.MustNewNebiusSDK(ctx), op.GetClient(), clusterCA, - wgAlloc, ) lo.Must0(err, "registering nebius cloud provider") } // kaito { - wgAlloc := wireguard.NewIPAllocator(op.Manager.GetCache(), kaito.GroupKind, 30*time.Second) - defer wgAlloc.Close() - err := kaito.Register( ctx, hubCloudProvider, clusterCA, - wgAlloc, ) lo.Must0(err, "registering kaito cloud provider") } diff --git a/karpenter/examples/nebius/nodeclass.yaml b/karpenter/examples/nebius/nodeclass.yaml index 1fd6187..3a4bb82 100644 --- a/karpenter/examples/nebius/nodeclass.yaml +++ b/karpenter/examples/nebius/nodeclass.yaml @@ -7,4 +7,3 @@ spec: region: "" subnetID: "" osDiskSizeGB: 128 - wireguardPeerCIDR: "100.96.1.0/24" diff --git a/karpenter/pkg/apis/crds/flex.aks.azure.com_nebiusnodeclasses.yaml b/karpenter/pkg/apis/crds/flex.aks.azure.com_nebiusnodeclasses.yaml index 6040ab6..ba7df0d 100644 --- a/karpenter/pkg/apis/crds/flex.aks.azure.com_nebiusnodeclasses.yaml +++ b/karpenter/pkg/apis/crds/flex.aks.azure.com_nebiusnodeclasses.yaml @@ -73,12 +73,6 @@ spec: SubnetID is the nebius subnet id to launch nodes in. Node will be auto-assigned an IP from this subnet. type: string - wireguardPeerCIDR: - description: |- - WireguardPeerCIDR is the CIDR to use for Wireguard peer IPs. - When this is set to a non-empty value, nodes will be allocated with a IP address - from this CIDR and configured to use Wireguard for node to node networking. - type: string required: - projectID - region diff --git a/karpenter/pkg/apis/v1alpha1/nebius.go b/karpenter/pkg/apis/v1alpha1/nebius.go index 481cec4..00a69cb 100644 --- a/karpenter/pkg/apis/v1alpha1/nebius.go +++ b/karpenter/pkg/apis/v1alpha1/nebius.go @@ -70,12 +70,6 @@ type NebiusNodeClassSpec struct { // +optional AllocateNodePublicIP *bool `json:"allocateNodePublicIP,omitempty"` - // WireguardPeerCIDR is the CIDR to use for Wireguard peer IPs. - // When this is set to a non-empty value, nodes will be allocated with a IP address - // from this CIDR and configured to use Wireguard for node to node networking. - // +optional - WireguardPeerCIDR *string `json:"wireguardPeerCIDR,omitempty"` - // TODO: other fields (kublet etc) } diff --git a/karpenter/pkg/apis/v1alpha1/zz_generated.deepcopy.go b/karpenter/pkg/apis/v1alpha1/zz_generated.deepcopy.go index c4741ee..76577b3 100644 --- a/karpenter/pkg/apis/v1alpha1/zz_generated.deepcopy.go +++ b/karpenter/pkg/apis/v1alpha1/zz_generated.deepcopy.go @@ -86,11 +86,6 @@ func (in *NebiusNodeClassSpec) DeepCopyInto(out *NebiusNodeClassSpec) { *out = new(bool) **out = **in } - if in.WireguardPeerCIDR != nil { - in, out := &in.WireguardPeerCIDR, &out.WireguardPeerCIDR - *out = new(string) - **out = **in - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NebiusNodeClassSpec. diff --git a/karpenter/pkg/cloudproviders/kaito/cloudprovider.go b/karpenter/pkg/cloudproviders/kaito/cloudprovider.go index 01d62fc..5ce5beb 100644 --- a/karpenter/pkg/cloudproviders/kaito/cloudprovider.go +++ b/karpenter/pkg/cloudproviders/kaito/cloudprovider.go @@ -25,21 +25,19 @@ import ( "github.com/Azure/aks-flex/karpenter/pkg/cloudproviders" nebiuscloudprovider "github.com/Azure/aks-flex/karpenter/pkg/cloudproviders/nebius" flexopts "github.com/Azure/aks-flex/karpenter/pkg/options" - wgallocator "github.com/Azure/aks-flex/karpenter/pkg/utils/wireguard" ) func Register( ctx context.Context, hub *cloudproviders.CloudProvidersHub, clusterCA []byte, - wgAlloc *wgallocator.IPAllocator, ) error { stretchPluginConn, err := stretchservices.NewConnection() if err != nil { return fmt.Errorf("creating stretch plugin connection: %w", err) } - cp := newCloudProvider(stretchPluginConn, clusterCA, wgAlloc) + cp := newCloudProvider(stretchPluginConn, clusterCA) hub.Register(cp, GroupKind, ProviderIDScheme) return nil @@ -49,21 +47,18 @@ type CloudProvider struct { stretchPluginConn *grpc.ClientConn stretchAgentPoolsClient agentpoolsapi.AgentPoolsClient - clusterCA []byte - wgAllocator *wgallocator.IPAllocator + clusterCA []byte } func newCloudProvider( stretchPluginConn *grpc.ClientConn, clusterCA []byte, - wgAlloc *wgallocator.IPAllocator, ) *CloudProvider { return &CloudProvider{ stretchPluginConn: stretchPluginConn, stretchAgentPoolsClient: agentpoolsapi.NewAgentPoolsClient(stretchPluginConn), - clusterCA: clusterCA, - wgAllocator: wgAlloc, + clusterCA: clusterCA, } } @@ -84,7 +79,7 @@ func (c *CloudProvider) Create(ctx context.Context, nodeClaim *v1.NodeClaim) (*v } nebiusAgentPoolSettings, err := resolveNebiusAgentPoolSettings( - ctx, kaitoOpts, c.wgAllocator, + ctx, kaitoOpts, nodeClaim, nodeClaimReqs, ) if err != nil { diff --git a/karpenter/pkg/cloudproviders/kaito/nodeclaim.go b/karpenter/pkg/cloudproviders/kaito/nodeclaim.go index f56aa35..89e7795 100644 --- a/karpenter/pkg/cloudproviders/kaito/nodeclaim.go +++ b/karpenter/pkg/cloudproviders/kaito/nodeclaim.go @@ -13,13 +13,11 @@ import ( stretchapi "github.com/Azure/aks-flex/plugin/api" "github.com/Azure/aks-flex/plugin/pkg/services/agentpools/api/features/kubeadm" - "github.com/Azure/aks-flex/plugin/pkg/services/agentpools/api/features/wireguard" nebiusinstance "github.com/Azure/aks-flex/plugin/pkg/services/agentpools/nebius/instance" "github.com/Azure/aks-flex/plugin/pkg/topology" "github.com/Azure/aks-flex/karpenter/pkg/cloudproviders" flexopts "github.com/Azure/aks-flex/karpenter/pkg/options" - wgallocator "github.com/Azure/aks-flex/karpenter/pkg/utils/wireguard" ) // ``` @@ -114,7 +112,6 @@ type resolvedNebiusAgentPoolSettings struct { Preset string ImageFamily string OSDiskSizeGibibytes int64 - Wireguard *wireguard.Config } var azureGPUInstanceTypeToNebiusPlatformPreset = map[string]struct { @@ -133,7 +130,6 @@ var azureGPUInstanceTypeToNebiusPlatformPreset = map[string]struct { func resolveNebiusAgentPoolSettings( ctx context.Context, kaitoOpts *flexopts.KaitoOptions, - wgAllocator *wgallocator.IPAllocator, nodeClaim *v1.NodeClaim, nodeClaimReqs nodeClaimRequirements, ) (resolvedNebiusAgentPoolSettings, error) { @@ -154,20 +150,6 @@ func resolveNebiusAgentPoolSettings( minimalOSDiskSizeGibibytes, ) - if cidr := kaitoOpts.NebiusWireguardPeerCIDR; cidr != "" { - wgPeerIP, err := wgAllocator.AllocateIP( - ctx, cidr, - kaitoNodeClassName, nodeClaim.Name, - ) - if err != nil { - var zero resolvedNebiusAgentPoolSettings - return zero, fmt.Errorf("failed to allocate wireguard peer IP: %w", err) - } - rv.Wireguard = wireguard.Config_builder{ - PeerIp: lo.ToPtr(wgPeerIP), - }.Build() - } - return rv, nil } @@ -221,7 +203,6 @@ func nodeClaimToNebiusAgentPool( // TODO: support more remote cloud types ImageFamily: lo.ToPtr(resolvedSettings.ImageFamily), OsDiskSizeGibibytes: lo.ToPtr(resolvedSettings.OSDiskSizeGibibytes), Kubeadm: kubeadmConfig, - Wireguard: resolvedSettings.Wireguard, } return nebiusinstance.AgentPool_builder{ diff --git a/karpenter/pkg/cloudproviders/nebius/cloudprovider.go b/karpenter/pkg/cloudproviders/nebius/cloudprovider.go index 6069001..fe39cd2 100644 --- a/karpenter/pkg/cloudproviders/nebius/cloudprovider.go +++ b/karpenter/pkg/cloudproviders/nebius/cloudprovider.go @@ -23,13 +23,11 @@ import ( stretchhelper "github.com/Azure/aks-flex/plugin/pkg/helper" stretchservices "github.com/Azure/aks-flex/plugin/pkg/services" agentpoolsapi "github.com/Azure/aks-flex/plugin/pkg/services/agentpools/api" - "github.com/Azure/aks-flex/plugin/pkg/services/agentpools/api/features/wireguard" nebiusinstance "github.com/Azure/aks-flex/plugin/pkg/services/agentpools/nebius/instance" "github.com/Azure/aks-flex/karpenter/pkg/apis" "github.com/Azure/aks-flex/karpenter/pkg/apis/v1alpha1" "github.com/Azure/aks-flex/karpenter/pkg/cloudproviders" - wgallocator "github.com/Azure/aks-flex/karpenter/pkg/utils/wireguard" ) type CloudProvider struct { @@ -37,9 +35,8 @@ type CloudProvider struct { stretchPluginConn *grpc.ClientConn stretchAgentPoolsClient agentpoolsapi.AgentPoolsClient - kubeClient client.Client - clusterCA []byte - wgAllocator *wgallocator.IPAllocator + kubeClient client.Client + clusterCA []byte } func newCloudProvider( @@ -47,16 +44,14 @@ func newCloudProvider( stretchPluginConn *grpc.ClientConn, kubeClient client.Client, clusterCA []byte, - wgAlloc *wgallocator.IPAllocator, ) *CloudProvider { return &CloudProvider{ sdk: sdk, stretchPluginConn: stretchPluginConn, stretchAgentPoolsClient: agentpoolsapi.NewAgentPoolsClient(stretchPluginConn), - kubeClient: kubeClient, - clusterCA: clusterCA, - wgAllocator: wgAlloc, + kubeClient: kubeClient, + clusterCA: clusterCA, } } @@ -66,14 +61,13 @@ func Register( sdk *gosdk.SDK, kubeClient client.Client, clusterCA []byte, - wgAlloc *wgallocator.IPAllocator, ) error { stretchPluginConn, err := stretchservices.NewConnection() if err != nil { return fmt.Errorf("creating stretch plugin connection: %w", err) } - cp := newCloudProvider(sdk, stretchPluginConn, kubeClient, clusterCA, wgAlloc) + cp := newCloudProvider(sdk, stretchPluginConn, kubeClient, clusterCA) hub.Register(cp, GroupKind, ProviderIDScheme) return nil @@ -129,26 +123,12 @@ func (c *CloudProvider) Create(ctx context.Context, nodeClaim *v1.NodeClaim) (*v "platformPreset", platformPresetToLaunch.InstanceTypeName(), ) - // Allocate a WireGuard peer IP if enabled for this NodeClass. - var wgConfig *wireguard.Config - if nodeClass.Spec.WireguardPeerCIDR != nil { - peerIP, err := c.wgAllocator.AllocateIP(ctx, *nodeClass.Spec.WireguardPeerCIDR, nodeClass.Name, nodeClaim.Name) - if err != nil { - return nil, err - } - logger.Info("allocated WireGuard peer IP", "peerIP", peerIP) - wgConfig = wireguard.Config_builder{ - PeerIp: lo.ToPtr(peerIP), - }.Build() - } - agentPool := nodeClaimToStretchAgentPool( karpoptions.FromContext(ctx), c.clusterCA, nodeClass, nodeClaim, platformPresetToLaunch, - wgConfig, ) // TODO: create async - we just need to retrieve the resource id for rebuilding the claim agentPoolCreated, err := stretchhelper.CreateOrUpdate( diff --git a/karpenter/pkg/cloudproviders/nebius/nodeclaim.go b/karpenter/pkg/cloudproviders/nebius/nodeclaim.go index 07ab26d..fc159a4 100644 --- a/karpenter/pkg/cloudproviders/nebius/nodeclaim.go +++ b/karpenter/pkg/cloudproviders/nebius/nodeclaim.go @@ -16,7 +16,6 @@ import ( stretchapi "github.com/Azure/aks-flex/plugin/api" "github.com/Azure/aks-flex/plugin/pkg/services/agentpools/api/features/kubeadm" - "github.com/Azure/aks-flex/plugin/pkg/services/agentpools/api/features/wireguard" nebiusinstance "github.com/Azure/aks-flex/plugin/pkg/services/agentpools/nebius/instance" "github.com/Azure/aks-flex/plugin/pkg/topology" @@ -83,7 +82,6 @@ func nodeClaimToStretchAgentPool( nodeClass *v1alpha1.NebiusNodeClass, nodeClaim *v1.NodeClaim, platformPreset *platformPreset, - wgConfig *wireguard.Config, ) *nebiusinstance.AgentPool { mdBuilder := stretchapi.Metadata_builder{ Id: lo.ToPtr(nodeClaim.Name), @@ -141,7 +139,6 @@ func nodeClaimToStretchAgentPool( ImageFamily: lo.ToPtr(imageFamily), OsDiskSizeGibibytes: lo.ToPtr(int64(osDiskSize)), Kubeadm: kubeadmConfig, - Wireguard: wgConfig, } return nebiusinstance.AgentPool_builder{ diff --git a/karpenter/pkg/options/kaito.go b/karpenter/pkg/options/kaito.go index 6a215ea..4a6507b 100644 --- a/karpenter/pkg/options/kaito.go +++ b/karpenter/pkg/options/kaito.go @@ -14,10 +14,9 @@ func init() { type kaitoOptionsKey struct{} type KaitoOptions struct { - NebiusProjectID string - NebiusRegion string - NebiusSubnetID string - NebiusWireguardPeerCIDR string + NebiusProjectID string + NebiusRegion string + NebiusSubnetID string } var _ options.Injectable = (*KaitoOptions)(nil) @@ -26,11 +25,6 @@ func (k *KaitoOptions) AddFlags(fs *options.FlagSet) { fs.StringVar(&k.NebiusProjectID, "flex-kaito.nebius-project-id", "", "The Nebius project ID to use for provisioning instances.") fs.StringVar(&k.NebiusRegion, "flex-kaito.nebius-region", "", "The Nebius region to use for provisioning instances.") fs.StringVar(&k.NebiusSubnetID, "flex-kaito.nebius-subnet-id", "", "The Nebius subnet ID to use for provisioning instances.") - fs.StringVar( - &k.NebiusWireguardPeerCIDR, - "flex-kaito.nebius-wireguard-peer-cidr", "100.96.1.0/24", - "The CIDR for the Nebius Wireguard peer. Sets to empty to disable Wireguard.", - ) } func (k *KaitoOptions) Parse(fs *options.FlagSet, args ...string) error { diff --git a/karpenter/pkg/utils/wireguard/allocator.go b/karpenter/pkg/utils/wireguard/allocator.go deleted file mode 100644 index 47ab202..0000000 --- a/karpenter/pkg/utils/wireguard/allocator.go +++ /dev/null @@ -1,247 +0,0 @@ -package wireguard - -import ( - "context" - "encoding/binary" - "fmt" - "math/rand/v2" - "net" - "sync" - "time" - - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/runtime/schema" - "sigs.k8s.io/controller-runtime/pkg/cache" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/log" - karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" - "sigs.k8s.io/karpenter/pkg/cloudprovider" -) - -// IPAllocator manages WireGuard peer IP allocation for NodeClaims. -// -// Because node creation takes time, we need to track "pre-allocated" IPs that have -// been handed out but haven't yet appeared on actual Nodes. Without this, concurrent -// NodeClaim creations could receive the same IP. -type IPAllocator struct { - cache cache.Cache - groupKind schema.GroupKind - - // mu serializes all IP allocations. This is a heavy lock (one allocation at a - // time across all NodeClasses) but is necessary for correctness since we need - // to read-then-write the pre-allocated state atomically. - mu sync.Mutex - - // preAllocated tracks IPs that have been allocated but not yet confirmed on a - // Node. Keyed by NodeClass name -> (IP -> NodeClaim name). - preAllocated map[string]map[string]string - - // cancel stops the background cleanup goroutine. - cancel context.CancelFunc -} - -// NewIPAllocator creates a new IPAllocator and starts its background cleanup goroutine. -// The caller must call Close() to stop the cleanup goroutine when the allocator is no -// longer needed. -func NewIPAllocator(kubeCache cache.Cache, groupKind schema.GroupKind, cleanupInterval time.Duration) *IPAllocator { - ctx, cancel := context.WithCancel(context.Background()) - a := &IPAllocator{ - cache: kubeCache, - groupKind: groupKind, - preAllocated: make(map[string]map[string]string), - cancel: cancel, - } - a.startCleanup(ctx, cleanupInterval) - return a -} - -// Close stops the background cleanup goroutine. -func (a *IPAllocator) Close() { - a.cancel() -} - -// AllocateIP allocates a WireGuard peer IP for a NodeClaim under the given NodeClass. -// The cidr parameter is the WireGuard peer CIDR to allocate from. -// Returns the allocated IP string, or an error (InsufficientCapacityError if CIDR is exhausted). -func (a *IPAllocator) AllocateIP( - ctx context.Context, - cidr string, - nodeClassName string, - nodeClaimName string, -) (string, error) { - a.mu.Lock() - defer a.mu.Unlock() - - // TODO: replace this per-call List with a k8s informer-based approach. We can - // use cache.GetInformer() to register an event handler on Nodes and maintain - // the confirmed IP set reactively, avoiding the List cost on every allocation. - - // Step 1: Read current Nodes for this NodeClass from the informer-backed cache. - confirmedIPs := make(map[string]struct{}) - nodeClassLabelKey := karpv1.NodeClassLabelKey(a.groupKind) - nodeList := &corev1.NodeList{} - if err := a.cache.List(ctx, nodeList, client.MatchingLabels{nodeClassLabelKey: nodeClassName}); err != nil { - return "", fmt.Errorf("listing nodes for nodeclass %q: %w", nodeClassName, err) - } - for _, node := range nodeList.Items { - for _, addr := range node.Status.Addresses { - if addr.Type == corev1.NodeInternalIP { - confirmedIPs[addr.Address] = struct{}{} - } - } - } - - // Step 2: Reconcile the pre-allocated list — remove entries whose IPs have - // appeared on actual Nodes (they are now confirmed). - preAlloc := a.preAllocated[nodeClassName] - if preAlloc == nil { - preAlloc = make(map[string]string) - a.preAllocated[nodeClassName] = preAlloc - } - for ip := range preAlloc { - if _, confirmed := confirmedIPs[ip]; confirmed { - delete(preAlloc, ip) - } - } - - // Step 3: Combine confirmed IPs and remaining pre-allocated IPs as the full - // set of taken addresses, then allocate the next available IP. - var allAllocatedIPs []string - for ip := range confirmedIPs { - allAllocatedIPs = append(allAllocatedIPs, ip) - } - for ip := range preAlloc { - allAllocatedIPs = append(allAllocatedIPs, ip) - } - - peerIP, err := AllocateRandomIP(cidr, allAllocatedIPs) - if err != nil { - return "", cloudprovider.NewInsufficientCapacityError(fmt.Errorf("allocating wireguard peer IP: %w", err)) - } - - // Step 4: Record the new IP as pre-allocated, associated with the NodeClaim. - preAlloc[peerIP] = nodeClaimName - - return peerIP, nil -} - -// startCleanup launches a background goroutine that periodically removes -// pre-allocated IPs whose corresponding NodeClaims no longer exist. This handles -// cases where node creation fails and the NodeClaim is cleaned up, but the IP -// was never confirmed on a Node. -func (a *IPAllocator) startCleanup(ctx context.Context, interval time.Duration) { - go func() { - ticker := time.NewTicker(interval) - defer ticker.Stop() - - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - a.cleanup(ctx) - } - } - }() -} - -func (a *IPAllocator) cleanup(ctx context.Context) { - logger := log.FromContext(ctx).WithName("wireguard-ip-cleanup") - - a.mu.Lock() - defer a.mu.Unlock() - - for nodeClassName, preAlloc := range a.preAllocated { - for ip, nodeClaimName := range preAlloc { - nc := &karpv1.NodeClaim{} - err := a.cache.Get(ctx, client.ObjectKey{Name: nodeClaimName}, nc) - if errors.IsNotFound(err) { - logger.V(5).Info( - "removing pre-allocated IP for deleted nodeclaim", - "nodeClass", nodeClassName, - "ip", ip, - "nodeClaim", nodeClaimName, - ) - delete(preAlloc, ip) - } else if err != nil { - logger.V(5).Error(err, "checking nodeclaim existence", - "nodeClass", nodeClassName, - "nodeClaim", nodeClaimName, - ) - } - } - if len(preAlloc) == 0 { - delete(a.preAllocated, nodeClassName) - } - } -} - -// AllocateRandomIP allocates a random free IP from the given CIDR, excluding the allocatedIPs. -// The network address and broadcast address are excluded from allocation (except for /31 and /32). -func AllocateRandomIP(cidr string, allocatedIPs []string) (string, error) { - _, ipNet, err := net.ParseCIDR(cidr) - if err != nil { - return "", fmt.Errorf("parsing CIDR %q: %w", cidr, err) - } - - // Build a set of already-allocated IPs for O(1) lookup. - allocated := make(map[string]struct{}, len(allocatedIPs)) - for _, ip := range allocatedIPs { - parsed := net.ParseIP(ip) - if parsed == nil { - return "", fmt.Errorf("parsing allocated IP %q: invalid IP address", ip) - } - // Normalize to 4-byte representation for consistent map keys. - if v4 := parsed.To4(); v4 != nil { - parsed = v4 - } - allocated[parsed.String()] = struct{}{} - } - - // Convert network address to a uint32 for iteration. - networkIP := ipNet.IP.To4() - if networkIP == nil { - return "", fmt.Errorf("only IPv4 CIDRs are supported, got %q", cidr) - } - networkAddr := binary.BigEndian.Uint32(networkIP) - - ones, bits := ipNet.Mask.Size() - if bits != 32 { - return "", fmt.Errorf("only IPv4 CIDRs are supported, got %q", cidr) - } - hostBits := uint(bits - ones) - totalHosts := uint32(1) << hostBits - - // Determine the usable host offset range. - // For /32, there is exactly one IP (the address itself) — allow it. - // For /31, there are two usable IPs per RFC 3021 — allow both. - // Otherwise skip network (first) and broadcast (last) addresses. - start := uint32(1) - end := totalHosts - 2 - if hostBits <= 1 { - start = 0 - end = totalHosts - 1 - } - - // Collect all free offsets. - rangeSize := end - start + 1 - free := make([]uint32, 0, rangeSize) - for offset := start; offset <= end; offset++ { - candidate := make(net.IP, 4) - binary.BigEndian.PutUint32(candidate, networkAddr+offset) - if _, taken := allocated[candidate.String()]; !taken { - free = append(free, offset) - } - } - - if len(free) == 0 { - return "", fmt.Errorf("no free IP addresses available in CIDR %q", cidr) - } - - // Pick a random free IP. - chosen := free[rand.IntN(len(free))] - ip := make(net.IP, 4) - binary.BigEndian.PutUint32(ip, networkAddr+chosen) - return ip.String(), nil -} diff --git a/karpenter/pkg/utils/wireguard/allocator_test.go b/karpenter/pkg/utils/wireguard/allocator_test.go deleted file mode 100644 index 2bb4aa1..0000000 --- a/karpenter/pkg/utils/wireguard/allocator_test.go +++ /dev/null @@ -1,465 +0,0 @@ -package wireguard - -import ( - "context" - "net" - "testing" - "time" - - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" - - "sigs.k8s.io/controller-runtime/pkg/cache" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -// --------------------------------------------------------------------------- -// AllocateRandomIP tests (pure function, no dependencies) -// --------------------------------------------------------------------------- - -func TestAllocateRandomIP_Basic(t *testing.T) { - ip, err := AllocateRandomIP("10.0.0.0/30", nil) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - // /30 has 4 addresses, usable are .1 and .2 (skip network .0 and broadcast .3) - if ip != "10.0.0.1" && ip != "10.0.0.2" { - t.Fatalf("expected 10.0.0.1 or 10.0.0.2, got %s", ip) - } -} - -func TestAllocateRandomIP_ExcludesAllocated(t *testing.T) { - ip, err := AllocateRandomIP("10.0.0.0/30", []string{"10.0.0.1"}) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if ip != "10.0.0.2" { - t.Fatalf("expected 10.0.0.2 (only free host), got %s", ip) - } -} - -func TestAllocateRandomIP_Exhausted(t *testing.T) { - _, err := AllocateRandomIP("10.0.0.0/30", []string{"10.0.0.1", "10.0.0.2"}) - if err == nil { - t.Fatal("expected error for exhausted CIDR, got nil") - } -} - -func TestAllocateRandomIP_Slash32(t *testing.T) { - ip, err := AllocateRandomIP("192.168.1.5/32", nil) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if ip != "192.168.1.5" { - t.Fatalf("expected 192.168.1.5 for /32, got %s", ip) - } -} - -func TestAllocateRandomIP_Slash32_Exhausted(t *testing.T) { - _, err := AllocateRandomIP("192.168.1.5/32", []string{"192.168.1.5"}) - if err == nil { - t.Fatal("expected error for exhausted /32, got nil") - } -} - -func TestAllocateRandomIP_Slash31(t *testing.T) { - // /31 has 2 usable IPs per RFC 3021 - ip, err := AllocateRandomIP("10.0.0.4/31", nil) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if ip != "10.0.0.4" && ip != "10.0.0.5" { - t.Fatalf("expected 10.0.0.4 or 10.0.0.5 for /31, got %s", ip) - } -} - -func TestAllocateRandomIP_Slash31_OneAllocated(t *testing.T) { - ip, err := AllocateRandomIP("10.0.0.4/31", []string{"10.0.0.4"}) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if ip != "10.0.0.5" { - t.Fatalf("expected 10.0.0.5, got %s", ip) - } -} - -func TestAllocateRandomIP_Slash31_Exhausted(t *testing.T) { - _, err := AllocateRandomIP("10.0.0.4/31", []string{"10.0.0.4", "10.0.0.5"}) - if err == nil { - t.Fatal("expected error for exhausted /31, got nil") - } -} - -func TestAllocateRandomIP_LargerCIDR(t *testing.T) { - // /24 has 254 usable hosts (.1-.254) - ip, err := AllocateRandomIP("172.16.0.0/24", nil) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - parsed := net.ParseIP(ip) - if parsed == nil { - t.Fatalf("returned IP %q is not valid", ip) - } - _, cidrNet, _ := net.ParseCIDR("172.16.0.0/24") - if !cidrNet.Contains(parsed) { - t.Fatalf("IP %s not in CIDR 172.16.0.0/24", ip) - } - // Must not be network or broadcast - if ip == "172.16.0.0" || ip == "172.16.0.255" { - t.Fatalf("IP %s should not be network or broadcast address", ip) - } -} - -func TestAllocateRandomIP_InvalidCIDR(t *testing.T) { - _, err := AllocateRandomIP("not-a-cidr", nil) - if err == nil { - t.Fatal("expected error for invalid CIDR, got nil") - } -} - -func TestAllocateRandomIP_InvalidAllocatedIP(t *testing.T) { - _, err := AllocateRandomIP("10.0.0.0/24", []string{"not-an-ip"}) - if err == nil { - t.Fatal("expected error for invalid allocated IP, got nil") - } -} - -func TestAllocateRandomIP_AllocatesAllHostsInSlash29(t *testing.T) { - // /29 = 8 addresses, 6 usable (.1 through .6) - allocated := make(map[string]struct{}) - var allocList []string - - for i := 0; i < 6; i++ { - ip, err := AllocateRandomIP("10.0.0.0/29", allocList) - if err != nil { - t.Fatalf("allocation %d: unexpected error: %v", i, err) - } - if _, dup := allocated[ip]; dup { - t.Fatalf("allocation %d: duplicate IP %s", i, ip) - } - allocated[ip] = struct{}{} - allocList = append(allocList, ip) - } - - if len(allocated) != 6 { - t.Fatalf("expected 6 unique IPs, got %d", len(allocated)) - } - - // Next allocation should fail - _, err := AllocateRandomIP("10.0.0.0/29", allocList) - if err == nil { - t.Fatal("expected error after exhausting /29, got nil") - } -} - -// --------------------------------------------------------------------------- -// Minimal mock cache for IPAllocator tests -// --------------------------------------------------------------------------- - -// mockCache implements cache.Cache with minimal functionality for testing. -// Only List and Get are implemented; all other methods panic. -type mockCache struct { - cache.Cache // embed to satisfy interface; unused methods will panic via nil dereference - nodes []corev1.Node - nodeClaims map[string]*karpv1.NodeClaim // name -> NodeClaim -} - -func (m *mockCache) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { - nodeList, ok := list.(*corev1.NodeList) - if !ok { - return nil - } - nodeList.Items = append(nodeList.Items[:0], m.nodes...) - return nil -} - -func (m *mockCache) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { - nc, ok := obj.(*karpv1.NodeClaim) - if !ok { - return errors.NewNotFound(schema.GroupResource{}, key.Name) - } - stored, exists := m.nodeClaims[key.Name] - if !exists { - return errors.NewNotFound(schema.GroupResource{Group: "karpenter.sh", Resource: "nodeclaims"}, key.Name) - } - *nc = *stored - return nil -} - -func (m *mockCache) GetInformer(ctx context.Context, obj client.Object, opts ...cache.InformerGetOption) (cache.Informer, error) { - panic("not implemented") -} - -func (m *mockCache) GetInformerForKind(ctx context.Context, gvk schema.GroupVersionKind, opts ...cache.InformerGetOption) (cache.Informer, error) { - panic("not implemented") -} - -func (m *mockCache) RemoveInformer(ctx context.Context, obj client.Object) error { - panic("not implemented") -} - -func (m *mockCache) Start(ctx context.Context) error { - panic("not implemented") -} - -func (m *mockCache) WaitForCacheSync(ctx context.Context) bool { - panic("not implemented") -} - -func (m *mockCache) IndexField(ctx context.Context, obj client.Object, field string, extractValue client.IndexerFunc) error { - panic("not implemented") -} - -func makeNode(name string, internalIP string) corev1.Node { - return corev1.Node{ - ObjectMeta: metav1.ObjectMeta{Name: name}, - Status: corev1.NodeStatus{ - Addresses: []corev1.NodeAddress{ - {Type: corev1.NodeInternalIP, Address: internalIP}, - }, - }, - } -} - -// newTestAllocator creates an IPAllocator with a mockCache that does NOT start -// the background cleanup goroutine (to avoid flakiness in tests). -func newTestAllocator(nodes []corev1.Node, nodeClaims map[string]*karpv1.NodeClaim) *IPAllocator { - mc := &mockCache{ - nodes: nodes, - nodeClaims: nodeClaims, - } - _, cancel := context.WithCancel(context.Background()) - // Cancel immediately — we don't want the cleanup goroutine running in unit tests. - cancel() - return &IPAllocator{ - cache: mc, - groupKind: schema.GroupKind{Group: "test.example.com", Kind: "TestNodeClass"}, - preAllocated: make(map[string]map[string]string), - cancel: cancel, - } -} - -// --------------------------------------------------------------------------- -// IPAllocator.AllocateIP tests -// --------------------------------------------------------------------------- - -func TestIPAllocator_AllocateIP_Basic(t *testing.T) { - a := newTestAllocator(nil, nil) - - ip, err := a.AllocateIP(context.Background(), "10.0.0.0/30", "my-class", "claim-1") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if ip != "10.0.0.1" && ip != "10.0.0.2" { - t.Fatalf("expected 10.0.0.1 or 10.0.0.2, got %s", ip) - } -} - -func TestIPAllocator_AllocateIP_AvoidsDuplicates(t *testing.T) { - a := newTestAllocator(nil, nil) - cidr := "10.0.0.0/30" // 2 usable IPs - - ip1, err := a.AllocateIP(context.Background(), cidr, "my-class", "claim-1") - if err != nil { - t.Fatalf("first allocation: unexpected error: %v", err) - } - - ip2, err := a.AllocateIP(context.Background(), cidr, "my-class", "claim-2") - if err != nil { - t.Fatalf("second allocation: unexpected error: %v", err) - } - - if ip1 == ip2 { - t.Fatalf("two allocations returned the same IP: %s", ip1) - } -} - -func TestIPAllocator_AllocateIP_ExhaustedReturnsSufficientError(t *testing.T) { - a := newTestAllocator(nil, nil) - cidr := "10.0.0.0/30" // 2 usable IPs - - _, err := a.AllocateIP(context.Background(), cidr, "my-class", "claim-1") - if err != nil { - t.Fatalf("allocation 1: unexpected error: %v", err) - } - _, err = a.AllocateIP(context.Background(), cidr, "my-class", "claim-2") - if err != nil { - t.Fatalf("allocation 2: unexpected error: %v", err) - } - - // Third allocation should fail - _, err = a.AllocateIP(context.Background(), cidr, "my-class", "claim-3") - if err == nil { - t.Fatal("expected error for exhausted CIDR, got nil") - } -} - -func TestIPAllocator_AllocateIP_SkipsConfirmedNodeIPs(t *testing.T) { - // Node already has 10.0.0.1 as InternalIP - nodes := []corev1.Node{makeNode("node-1", "10.0.0.1")} - a := newTestAllocator(nodes, nil) - - ip, err := a.AllocateIP(context.Background(), "10.0.0.0/30", "my-class", "claim-1") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - // Only 10.0.0.2 should be available - if ip != "10.0.0.2" { - t.Fatalf("expected 10.0.0.2 (10.0.0.1 taken by node), got %s", ip) - } -} - -func TestIPAllocator_AllocateIP_ReconcilesPreAllocatedOnConfirmation(t *testing.T) { - a := newTestAllocator(nil, nil) - cidr := "10.0.0.0/30" // 2 usable IPs - - // Allocate both IPs - ip1, _ := a.AllocateIP(context.Background(), cidr, "my-class", "claim-1") - _, _ = a.AllocateIP(context.Background(), cidr, "my-class", "claim-2") - - // Now simulate that ip1's node has appeared in the cache - mc := a.cache.(*mockCache) - mc.nodes = []corev1.Node{makeNode("node-1", ip1)} - - // Next allocation should succeed because ip1 is confirmed (removed from pre-allocated) - // but also seen as confirmed, so still only 1 free IP... wait, both are taken. - // Let me think: ip1 is confirmed on a node, ip2 is pre-allocated. - // After reconciliation: ip1 removed from preAlloc (confirmed). ip2 stays in preAlloc. - // Confirmed set = {ip1}, preAlloc set = {ip2}. Both taken. Still exhausted. - // This is correct. Let me test a different scenario. - - // Better test: /29 with 6 usable hosts. Pre-allocate 1, confirm it, allocate another. - a2 := newTestAllocator(nil, nil) - cidr2 := "10.0.0.0/29" // 6 usable IPs - - firstIP, _ := a2.AllocateIP(context.Background(), cidr2, "my-class", "claim-1") - - // Simulate the node showing up with the first IP - mc2 := a2.cache.(*mockCache) - mc2.nodes = []corev1.Node{makeNode("node-1", firstIP)} - - // Allocate again — should succeed and the pre-allocated entry for firstIP should be cleaned up - secondIP, err := a2.AllocateIP(context.Background(), cidr2, "my-class", "claim-2") - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if secondIP == firstIP { - t.Fatalf("second IP should differ from confirmed first IP, both are %s", firstIP) - } - - // Verify internal state: preAllocated should only have the second IP - preAlloc := a2.preAllocated["my-class"] - if len(preAlloc) != 1 { - t.Fatalf("expected 1 pre-allocated entry, got %d", len(preAlloc)) - } - if _, ok := preAlloc[secondIP]; !ok { - t.Fatalf("expected pre-allocated entry for %s, not found", secondIP) - } -} - -func TestIPAllocator_AllocateIP_MultipleNodeClasses(t *testing.T) { - a := newTestAllocator(nil, nil) - cidr := "10.0.0.0/30" // 2 usable IPs per class - - // Allocate from class A - ipA, err := a.AllocateIP(context.Background(), cidr, "class-a", "claim-a1") - if err != nil { - t.Fatalf("class-a allocation: unexpected error: %v", err) - } - - // Allocate from class B — same CIDR but different class, so pre-allocated sets are independent - ipB, err := a.AllocateIP(context.Background(), cidr, "class-b", "claim-b1") - if err != nil { - t.Fatalf("class-b allocation: unexpected error: %v", err) - } - - // Both should get valid IPs (they may even be the same IP since classes are independent) - _ = ipA - _ = ipB - - if len(a.preAllocated["class-a"]) != 1 { - t.Fatalf("expected 1 pre-allocated for class-a, got %d", len(a.preAllocated["class-a"])) - } - if len(a.preAllocated["class-b"]) != 1 { - t.Fatalf("expected 1 pre-allocated for class-b, got %d", len(a.preAllocated["class-b"])) - } -} - -// --------------------------------------------------------------------------- -// IPAllocator.cleanup tests -// --------------------------------------------------------------------------- - -func TestIPAllocator_Cleanup_RemovesStalePreAllocated(t *testing.T) { - // claim-1 exists, claim-2 does not - nodeClaims := map[string]*karpv1.NodeClaim{ - "claim-1": {ObjectMeta: metav1.ObjectMeta{Name: "claim-1"}}, - } - a := newTestAllocator(nil, nodeClaims) - - // Seed pre-allocated state - a.preAllocated["my-class"] = map[string]string{ - "10.0.0.1": "claim-1", // exists - "10.0.0.2": "claim-2", // deleted - } - - a.cleanup(context.Background()) - - preAlloc := a.preAllocated["my-class"] - if _, ok := preAlloc["10.0.0.2"]; ok { - t.Fatal("expected stale IP 10.0.0.2 to be removed after cleanup") - } - if _, ok := preAlloc["10.0.0.1"]; !ok { - t.Fatal("expected IP 10.0.0.1 for existing claim to be retained") - } -} - -func TestIPAllocator_Cleanup_RemovesEmptyNodeClassEntry(t *testing.T) { - // No node claims exist at all - a := newTestAllocator(nil, nil) - - a.preAllocated["my-class"] = map[string]string{ - "10.0.0.1": "claim-gone", - } - - a.cleanup(context.Background()) - - if _, ok := a.preAllocated["my-class"]; ok { - t.Fatal("expected empty nodeclass entry to be deleted from preAllocated map") - } -} - -// --------------------------------------------------------------------------- -// IPAllocator.Close test -// --------------------------------------------------------------------------- - -func TestIPAllocator_Close(t *testing.T) { - mc := &mockCache{ - nodeClaims: make(map[string]*karpv1.NodeClaim), - } - - // Use a very short cleanup interval so the goroutine would fire quickly if not stopped. - a := NewIPAllocator(mc, schema.GroupKind{Group: "test", Kind: "Test"}, 10*time.Millisecond) - a.Close() - - // After Close, the allocator should still be usable for allocation (Close only stops cleanup). - ip, err := a.AllocateIP(context.Background(), "10.0.0.0/30", "my-class", "claim-1") - if err != nil { - t.Fatalf("unexpected error after Close: %v", err) - } - if ip == "" { - t.Fatal("expected a valid IP after Close") - } -} - -// --------------------------------------------------------------------------- -// Helpers: verify mockCache satisfies cache.Cache at compile time -// --------------------------------------------------------------------------- - -var _ cache.Cache = (*mockCache)(nil) - -// Verify the runtime module is only needed to satisfy the interface embed. -var _ runtime.Object = (*corev1.NodeList)(nil) From f1894b4f7e642bbeb379ab5dd767014965fc5005 Mon Sep 17 00:00:00 2001 From: hbc Date: Wed, 4 Mar 2026 21:50:26 -0800 Subject: [PATCH 4/4] doc: remove wireguard mentions from doc --- docs/usages/README.md | 2 +- docs/usages/cli-node-bootstrap.md | 6 +- .../usages/cli-plugin-nebius-unbounded-cni.md | 2 - docs/usages/cli-plugin-nebius.md | 52 ++---------- docs/usages/cli-prepare-aks-cluster.md | 76 +----------------- docs/usages/cli-setup.md | 2 +- .../resource-wg-route.png | Bin 102913 -> 0 bytes docs/usages/karpenter.md | 14 ++-- 8 files changed, 18 insertions(+), 136 deletions(-) delete mode 100644 docs/usages/images/cli-prepare-aks-cluster/resource-wg-route.png diff --git a/docs/usages/README.md b/docs/usages/README.md index 590d068..5524c16 100644 --- a/docs/usages/README.md +++ b/docs/usages/README.md @@ -9,7 +9,7 @@ Start with the CLI setup guide, then follow the scenario that best fits your use | Guide | Description | | ----- | ----------- | | [CLI Setup](cli-setup.md) | Install the `aks-flex-cli` tool and generate a `.env` configuration file | -| [AKS Cluster Setup](cli-prepare-aks-cluster.md) | Deploy Azure network resources, create an AKS cluster with Cilium CNI, and optionally enable WireGuard | +| [AKS Cluster Setup](cli-prepare-aks-cluster.md) | Deploy Azure network resources, create an AKS cluster with Cilium CNI, and optionally enable Unbounded CNI | ## Scenarios diff --git a/docs/usages/cli-node-bootstrap.md b/docs/usages/cli-node-bootstrap.md index 5f1c382..9d4f25a 100644 --- a/docs/usages/cli-node-bootstrap.md +++ b/docs/usages/cli-node-bootstrap.md @@ -377,7 +377,6 @@ $ kubectl get nodes NAME STATUS ROLES AGE VERSION aks-system-32742974-vmss000000 Ready 4h32m v1.33.6 aks-system-32742974-vmss000001 Ready 4h32m v1.33.6 -aks-wireguard-12237243-vmss000000 Ready 4h14m v1.33.6 flex-node-azure Ready 102s v1.33.8 ``` @@ -428,7 +427,6 @@ $ kubectl get nodes NAME STATUS ROLES AGE VERSION aks-system-32742974-vmss000000 Ready 5h19m v1.33.6 aks-system-32742974-vmss000001 Ready 5h19m v1.33.6 -aks-wireguard-12237243-vmss000000 Ready 5h1m v1.33.6 flex-node-azure Ready 48m v1.33.8 ubuntu NotReady 3m15s v1.33.8 ``` @@ -482,7 +480,7 @@ When a new VM boots with the generated cloud-init user data, the following steps │ └─ Contains: CA cert, API server URL, bootstrap token │ ├─ 3. Write kubeadm JoinConfiguration - │ └─ Contains: discovery path, node labels, node IP (if WireGuard) + │ └─ Contains: discovery path, node labels │ ├─ 4. Configure containerd (systemd cgroup) │ @@ -501,4 +499,4 @@ Key settings in the kubeadm `JoinConfiguration`: | -------------------------------- | --------------------------------------------------------- | | `discovery.file.kubeConfigPath` | Points to the bootstrap kubeconfig for cluster discovery | | `nodeRegistration.kubeletExtraArgs.node-labels` | Applies labels such as `aks.azure.com/stretch-managed=true` | -| `nodeRegistration.kubeletExtraArgs.node-ip` | Sets the node's InternalIP (used with WireGuard tunnels) | +| `nodeRegistration.kubeletExtraArgs.node-ip` | Sets the node's InternalIP | diff --git a/docs/usages/cli-plugin-nebius-unbounded-cni.md b/docs/usages/cli-plugin-nebius-unbounded-cni.md index 61eaa41..8c83544 100644 --- a/docs/usages/cli-plugin-nebius-unbounded-cni.md +++ b/docs/usages/cli-plugin-nebius-unbounded-cni.md @@ -175,8 +175,6 @@ Edit the file to configure a CPU node. Update the placeholder fields: | `spec.preset` | `4vcpu-16gb` | VM size preset | | `spec.imageFamily` | `ubuntu24.04-driverless` | OS image family | -> **Note:** When using Unbounded CNI, the `spec.wireguard` section in the agent pool config is not needed. The Unbounded CNI operator handles cross-cloud routing automatically. - The `kubeadm` section is auto-populated from the running AKS cluster when the `.env` is configured correctly. If the cluster is not reachable, placeholder values are generated that must be replaced manually. ### Apply the agent pool config diff --git a/docs/usages/cli-plugin-nebius.md b/docs/usages/cli-plugin-nebius.md index 6a2dc59..46fe449 100644 --- a/docs/usages/cli-plugin-nebius.md +++ b/docs/usages/cli-plugin-nebius.md @@ -118,41 +118,9 @@ $ aks-flex-cli plugin get networks nebius-default ![](./images/cli-plugin-nebius/resource-nebius-network.png) -## Nebius - Azure Network Connectivity +## Cross-Cloud Connectivity -AKS Flex uses WireGuard to establish an encrypted site-to-site tunnel between the Azure VNet and the Nebius VPC. On top of this tunnel, Cilium's VXLAN overlay is used to extend the Kubernetes pod network across clouds so that pods on Azure and Nebius nodes can communicate seamlessly. - -The following diagram illustrates the connectivity: - -``` - Azure Nebius - ┌──────────────────────────────┐ ┌──────────────────────────────┐ - │ VNet: 172.16.0.0/16 │ │ VPC: 172.20.0.0/16 │ - │ │ │ │ - │ ┌────────────┐ │ WireGuard │ ┌────────────┐ │ - │ │ AKS Node │ │ Tunnel │ │ Nebius VM │ │ - │ │ │ │◄───────────►│ │ │ │ - │ └────────────┘ │ (UDP/51820)│ └────────────┘ │ - │ │ │ │ - │ ┌────────────┐ │ │ ┌────────────┐ │ - │ │ WireGuard │──────────────┼─────────────┼──────────────│ WireGuard │ │ - │ │ Gateway │ Peer IP: 100.96.x.x │ │ Peer │ │ - │ └────────────┘ │ │ └────────────┘ │ - │ │ │ │ - │ Cilium VXLAN overlay spans across both clouds │ - └──────────────────────────────┘ └──────────────────────────────┘ -``` - -### Peer IP assignment - -Each node that participates in the WireGuard mesh is assigned a **peer IP** from the `100.96.0.0/12` range. This peer IP is critical because it serves as the node's address within the WireGuard tunnel and must be set as the node's InternalIP in Kubernetes. This allows `kube-proxy` to correctly forward service traffic to the node, since kube-proxy routes based on the node's InternalIP. - -When configuring agent pools, each node must be assigned a unique peer IP from this range. For example: - -| Node | Peer IP | -| ------------ | --------------- | -| CPU node | `100.96.1.111` | -| GPU node | `100.96.1.112` | +This guide assumes the AKS cluster has been deployed with Unbounded CNI for cross-cloud networking. See [Enable with Unbounded CNI](cli-prepare-aks-cluster.md#enable-with-unbounded-cni) and the [Nebius Cloud Integration (Unbounded CNI)](cli-plugin-nebius-unbounded-cni.md) guide for details on setting up cross-cloud connectivity. ## Create Nebius CPU Node @@ -188,9 +156,6 @@ This produces a JSON file like: "aks.azure.com/stretch-managed": "true", ... } - }, - "wireguard": { - "peerIp": "" } } } @@ -205,7 +170,6 @@ Edit the file to configure a CPU node. Update the placeholder fields: | `spec.platform` | `cpu-d3` | Nebius compute platform | | `spec.preset` | `4vcpu-16gb` | VM size preset | | `spec.imageFamily` | `ubuntu24.04-driverless` | OS image family | -| `spec.wireguard.peerIp` | *(unique IP in `100.96.0.0/12`)* | WireGuard peer IP for this node (see [Peer IP assignment](#peer-ip-assignment)) | The `kubeadm` section is auto-populated from the running AKS cluster when the `.env` is configured correctly. If the cluster is not reachable, placeholder values are generated that must be replaced manually. @@ -236,8 +200,7 @@ k get node -o wide NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME aks-system-32742974-vmss000000 Ready 40m v1.33.6 172.16.1.4 Ubuntu 22.04.5 LTS 5.15.0-1102-azure containerd://1.7.30-1 aks-system-32742974-vmss000001 Ready 40m v1.33.6 172.16.1.5 Ubuntu 22.04.5 LTS 5.15.0-1102-azure containerd://1.7.30-1 -aks-wireguard-12237243-vmss000000 Ready 21m v1.33.6 172.16.2.4 Ubuntu 22.04.5 LTS 5.15.0-1102-azure containerd://1.7.30-1 -computeinstance-e00c3m3yvj3rhnvhan Ready 58s v1.33.8 100.96.1.111 Ubuntu 24.04.4 LTS 6.11.0-1016-nvidia containerd://1.7.28 +computeinstance-e00c3m3yvj3rhnvhan Ready 58s v1.33.8 172.20.0.10 Ubuntu 24.04.4 LTS 6.11.0-1016-nvidia containerd://1.7.28 ``` ![](./images/cli-plugin-nebius/resource-nebius-instance-cpu.png) @@ -261,7 +224,6 @@ Edit the file to configure a GPU node: | `spec.platform` | *(GPU platform, e.g. `gpu-h100-sxm`)* | Nebius GPU compute platform | | `spec.preset` | *(GPU preset, e.g. `1gpu-16vcpu-200gb`)* | GPU VM size preset | | `spec.imageFamily` | *(GPU image, e.g. `ubuntu24.04-cuda12`)* | OS image with GPU drivers | -| `spec.wireguard.peerIp` | *(unique IP in `100.96.0.0/12`)* | WireGuard peer IP (must differ from CPU node, see [Peer IP assignment](#peer-ip-assignment)) | ### Apply the agent pool config @@ -287,16 +249,15 @@ k get node -o wide NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME aks-system-32742974-vmss000000 Ready 50m v1.33.6 172.16.1.4 Ubuntu 22.04.5 LTS 5.15.0-1102-azure containerd://1.7.30-1 aks-system-32742974-vmss000001 Ready 50m v1.33.6 172.16.1.5 Ubuntu 22.04.5 LTS 5.15.0-1102-azure containerd://1.7.30-1 -aks-wireguard-12237243-vmss000000 Ready 31m v1.33.6 172.16.2.4 Ubuntu 22.04.5 LTS 5.15.0-1102-azure containerd://1.7.30-1 -computeinstance-e00c3m3yvj3rhnvhan Ready 9m57s v1.33.8 100.96.1.111 Ubuntu 24.04.4 LTS 6.11.0-1016-nvidia containerd://1.7.28 -computeinstance-e00vm3hfp0gac4e5vz Ready 117s v1.33.8 100.96.1.112 Ubuntu 24.04.4 LTS 6.11.0-1016-nvidia containerd://1.7.28 +computeinstance-e00c3m3yvj3rhnvhan Ready 9m57s v1.33.8 172.20.0.10 Ubuntu 24.04.4 LTS 6.11.0-1016-nvidia containerd://1.7.28 +computeinstance-e00vm3hfp0gac4e5vz Ready 117s v1.33.8 172.20.0.11 Ubuntu 24.04.4 LTS 6.11.0-1016-nvidia containerd://1.7.28 ``` ![](./images/cli-plugin-nebius/resource-nebius-instance-gpu.png) ## Validating cross-cloud connectivity -With the WireGuard tunnel and Cilium VXLAN overlay in place, pods running on the Nebius nodes should be able to +With cross-cloud connectivity in place, pods running on the Nebius nodes should be able to communicate with pods on the AKS nodes, and vice versa. We can validate this by checking the logs from pods running on the Nebius nodes: @@ -344,7 +305,6 @@ $ kubectl get nodes NAME STATUS ROLES AGE VERSION aks-system-32742974-vmss000000 Ready 58m v1.33.6 aks-system-32742974-vmss000001 Ready 58m v1.33.6 -aks-wireguard-12237243-vmss000000 Ready 39m v1.33.6 computeinstance-e00c3m3yvj3rhnvhan NotReady 18m v1.33.8 computeinstance-e00vm3hfp0gac4e5vz NotReady 10m v1.33.8 ``` diff --git a/docs/usages/cli-prepare-aks-cluster.md b/docs/usages/cli-prepare-aks-cluster.md index a2c42e8..b9e371a 100644 --- a/docs/usages/cli-prepare-aks-cluster.md +++ b/docs/usages/cli-prepare-aks-cluster.md @@ -10,7 +10,6 @@ This guide walks through preparing a base AKS cluster and the supporting Azure r Optionally, you can also deploy: -- A **WireGuard gateway** for site-to-site encrypted tunneling -- see [Enable with WireGuard](#enable-with-wireguard) - **Unbounded CNI** for cross-cloud networking via the Unbounded CNI operator -- see [Enable with Unbounded CNI](#enable-with-unbounded-cni) ## Setup @@ -50,7 +49,7 @@ The CLI creates an AKS cluster with `networkPlugin: none`, which disables the bu - **Cilium CNI** installed via the Cilium CLI after the cluster is provisioned - A system-assigned managed identity -If you need site-to-site connectivity without a VPN gateway (or for development/testing purposes), you can additionally enable WireGuard or Unbounded CNI. These options are mutually exclusive -- refer to the [Enable with WireGuard](#enable-with-wireguard) or [Enable with Unbounded CNI](#enable-with-unbounded-cni) sections below. +If you need site-to-site connectivity without a VPN gateway (or for development/testing purposes), you can additionally enable Unbounded CNI -- refer to the [Enable with Unbounded CNI](#enable-with-unbounded-cni) section below. ## Create Network Resources @@ -124,72 +123,11 @@ $ aks-flex-cli aks deploy --cilium --kubeconfig ./my-cluster.kubeconfig ![](./images/cli-prepare-aks-cluster/resource-aks.png) -### Enable with WireGuard - -WireGuard provides an encrypted site-to-site tunnel between the AKS cluster and remote cloud nodes. This is useful when: - -- A VPN gateway is not available or not practical for the environment -- You need a lightweight tunnel for development and testing -- You want encrypted node-to-node communication across clouds - -To deploy the cluster with both Cilium and WireGuard: - -```bash -$ aks-flex-cli aks deploy --cilium --wireguard -``` - -In addition to the standard AKS resources, the `--wireguard` flag provisions: - -| Resource | Name | Details | -| ------------------------ | ----------------------- | ------------------------------------------------ | -| NSG rule | `AllowWireGuard` | Allows inbound UDP/51820 | -| Public IP prefix | `wg-pips` | Static public IP prefix for the gateway node | -| Agent pool | `wireguard` | 1-node pool in the `nodes` subnet with public IP (size: `$GATEWAY_VM_SIZE`) | -| Route table | `wg-routes` | Routes remote cloud traffic through the gateway | - -After the ARM deployment, the CLI automatically: - -1. Generates (or reuses) a WireGuard key pair stored as a Kubernetes secret -2. Waits for the WireGuard gateway node to register and receive its public IP -3. Updates the `wg-routes` route table to forward remote cloud traffic (`100.96.0.0/12`) through the gateway node -4. Associates the route table with the `aks` and `nodes` subnets -5. Deploys the WireGuard DaemonSet to the cluster - -Expected output: - -``` -2026/02/21 10:05:00 starting deployment aks in rg-aks-flex- -2026/02/21 10:15:00 deployment aks succeeded -2026/02/21 10:15:01 kubeconfig saved to "/home//.kube/config" -... -✅ Cilium was successfully installed! -2026/02/21 10:16:00 Getting WireGuard keys... -2026/02/21 10:16:00 Generating new WireGuard keys -2026/02/21 10:16:00 Public Key: -2026/02/21 10:16:00 Waiting for WireGuard gateway node to register... -2026/02/21 10:17:30 WireGuard gateway node ready -2026/02/21 10:17:30 Public IP: -2026/02/21 10:17:30 Private IP: -2026/02/21 10:17:30 Updating route table... -2026/02/21 10:17:35 Route table updated with gateway IP: -2026/02/21 10:17:35 Associating route table with subnets... -2026/02/21 10:17:40 Associating route table with subnet aks... -2026/02/21 10:17:45 Route table associated with subnet aks -2026/02/21 10:17:45 Associating route table with subnet nodes... -2026/02/21 10:17:50 Route table associated with subnet nodes -2026/02/21 10:17:50 Deploying WireGuard DaemonSet... -2026/02/21 10:17:52 WireGuard DaemonSet deployed successfully -``` - -Sample route table after deployment: - -![](./images/cli-prepare-aks-cluster/resource-wg-route.png) - ### Enable with Unbounded CNI Unbounded CNI provides cross-cloud networking through the Unbounded CNI operator, which manages Site, GatewayPool, and SiteGatewayPoolAssignment resources to route traffic between clouds. -> **Note:** The `--unbounded-cni` flag is mutually exclusive with `--cilium` and `--wireguard`. +> **Note:** The `--unbounded-cni` flag is mutually exclusive with `--cilium`. To deploy the cluster with Unbounded CNI: @@ -256,16 +194,6 @@ aks-system-32742974-vmss000000 Ready 16m v1.33.6 aks-system-32742974-vmss000001 Ready 16m v1.33.6 ``` -If you deployed with `--wireguard`, you will also see the WireGuard gateway node: - -```bash -$ kubectl get nodes -NAME STATUS ROLES AGE VERSION -aks-system-32742974-vmss000000 Ready 19m v1.33.6 -aks-system-32742974-vmss000001 Ready 19m v1.33.6 -aks-wireguard-12237243-vmss000000 Ready 51s v1.33.6 -``` - If you deployed with `--unbounded-cni`, you will see the gateway node pool: ```bash diff --git a/docs/usages/cli-setup.md b/docs/usages/cli-setup.md index bbea4c7..026f235 100644 --- a/docs/usages/cli-setup.md +++ b/docs/usages/cli-setup.md @@ -5,7 +5,7 @@ The AKS Flex CLI (`aks-flex-cli`) is a command-line tool for managing AKS Flex clusters that span across Azure and remote cloud providers (e.g. Nebius). It handles the end-to-end lifecycle of a multi-cloud Kubernetes environment, including: - **Network provisioning** -- deploy and manage the Azure-side network infrastructure -- **AKS cluster deployment** -- create and configure AKS clusters with options such as Cilium CNI and WireGuard encryption +- **AKS cluster deployment** -- create and configure AKS clusters with options such as Cilium CNI - **Remote cloud integration** -- configure networking and agent pools on remote cloud providers - **Configuration generation** -- bootstrap environment configs, Kubernetes cluster settings, and node bootstrap scripts diff --git a/docs/usages/images/cli-prepare-aks-cluster/resource-wg-route.png b/docs/usages/images/cli-prepare-aks-cluster/resource-wg-route.png deleted file mode 100644 index 5de4d6c0dea682f174623ac0eac8fd95b1c1f298..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 102913 zcmeFZbyQXB*9N*t2@w?$0SQTwI&_E9jesIj3c{wlV*{dubayur(p@55(jC&>9h>{^ zbI$jDzj5!lckFTRKlhK1F?O)mT6?WE-}%n@%x6CD_E%Pv!owlKfj}U5GScEI5D3;j z1cDljg$X{PdS3)SB@Y z8>Rf1V&GP2pysS$@1ozeaj$MI7!A@8z#6rh9uHZTIW%Bvffr;hKYF}DEDFIf@{xViucd@*VSex?k7B}kUo~d@K2pR{NyY_W{9+HInb6G&iGp^1*7Lb`TD8yq<#V}2`+!4^69bmz3zTOePH!O&qs>p-l_Bg+v7){=bt@9x#;A*ZP9t~-19o; zbwM~mh*f?DTIZ8Z)%<18j{9p~e}&=}6MD;rhhaTzrtb8+eei&S8>ROd5x2o@)0*!j z>)ku-f5ZayoFpH+PeeSCzDwzI-`lUU8fDkp_0OOE^*ae1{p~_NN9T)5PV(zCf|Odc zb`LBRiHN!+CHRQ#Uf!gpU1oY{Q_SF))GorSdDMKnc?ozR_o{IdeZ8-K#%Vw6A^WDO zK{)n&r^#S{l#JvsgZH&^_mgiFZHC9vcmjCi4+sqkn~FkSj2(x$*h;xb4h-DnU%qm> zN4n$h?n~CJsNYdi_KtB3{oTat0%FsSdyEI2-&{GX7k!`s#DVm8HUo7ybax9%!)H7SIT(z1D@#}(4cl$GePO0JdntFiqWl|H^&Q(7#}M= z{zPA6dY%n>d3mKYS*1y#oDY&UKD_7t$82JMI32S&P7L*HfoR*GZZwoezV-c$;XCck zPbZq)61_cq}_Efv2Wlj1)5vuB)Nyw<7d{Al$;c&W{P-N6?B8TN#RSVoNf z0d~6R>NB}e_D@_Y{$^q(VH9Cvoz}}tL;f@3yVOFRDV?s5#719}jbK&0>c}vQ^!7=! zURUEJNb-N2-nMR2j#?;rmKwBfeJtogxbrF{mA?QpizD1ePP}9+8;XGWQxCtd2t;*8F|q|Ixx=V-B23u7XLEcm%OHd5jTaL{Yk;* z{QRsXc|m63K{!_27mY6uU-5mU|3mrbky~^vcCo)WRfLg(o$r7iJ6i&8Om$$j|AeTZ z0+DyDmq0Ed-xlD^8%~F!dFWd6>sB%JSm#J*{oOv#= zE*@TBU!X}u=*b*pimQBywBJH|cMQTUb}MW-H7 zTq9a0dlb5A-v?v%8AC6WE)*;@cwbl1Zz42GXRK*_#{{<`prXn&pwmaRlTR+|~*{WTqTkjt$91<#+Zu=GcR-T+Arsi`^igVBf zEO78i|C7PkEd9+6Es`Ti%Yx*TWa(nH;+K2*bFS{uuvAzu{KXC3O&w;nSJKP!7AEgL z%$&O}9h-qCB#ciq`5kU2G<|Z@l0F9h(Q4iC+3~rNY?o}2RLQ`Tqz`lniff0pPqv!| zg*-Wn{Y&0W{uiC8LU6pG{zfTpf!14$Et>BGBfmL+seP>ND{npaj;qk9*9g<_p47I@ z=bIL6bt`mEa8CK^-}WiM`LX@uDkg?#vmU0Nljt1g`F9$cwC~+bM?am6ydQ{%Y8z=o z-8*~?mrOZ^v!_2U!6P0i6bAJ$ULqTc5@)wWqa2YHI4bW;SPCW|(J&XLxFL zp<6#PCVD2If>`8ma$KSJLUYcw&a1AwXV5a25`aNB%+kfi!9S>tJD2_c^PnP@F$Z?xdKcmT_ z@uOCtEuk@@?P0`W7+{WKoM6;p4&VtteuagNt3ha!1pP&@Z{qBpwwrb&rsR|7+vfAe zM+mD5H|A@Vyd=G^q7KJ};unK2H0cZv#Sf{*#pi-bgX4)$Y&qHZU(~+1wC?>9>&%v; zPh2vr@7U28d=#Sjz)@h1PUpdVw5XD_Lau^S#FNl!{t`xGx1sCIqb|&@#pTsUDzdM- z`y=5DUaDgX8;bS{%rXt!qb{4m(IIiMz>7(iN#B;>oIg$-sphaEVU59-y1~v(8+oBd@CZA1emd-DW zuWok}cPjLncT%rqt&oKq$=T4*L!S)93T3F8urh}^^c^SGSQw@-$ycfdcjj~_9N8~~ z!0LWV9t17Iyp&$PX_?l*F1yz|7S9|vE8JnxKg6em+r7I^sL3 zVitXz$!er&)cC{c<5~KG1Xn;_fbHYk_seSKwd=*S{&MfX1$`qT6uG85PpEwu^iZSg z#Im7r_K*yobY2&nMG#~0?y^Shol8+~UP1l#5_?Oj?Yz=?uJ`tWL% z(!;42^KZDmSG*-Ef>j$H##F1p1Y5KxA=99X*f_z`s`?IO3S)V%fku>{nwaG$xt}MHF6VqCI%1wn{lggF& z+p62Hr}|SN!t$otG^Q z)V){G{C;nXT*vy0OXhiJ^We{nn~uy5mk=nq#{_r7&Z&PUfx4n($~G^d!)7n*%cex$ zK0f#3z_YlqvScWJl75OQ|DHo*$!x{VLUB3-{5g|y)m70wlbh`E5{YRT@~GHijcJ;?%(&cZ=y$#Oqp+cQQ^mD6v8ULi zF34XCjbZ!}6(UbL z#Vc~i#1}+9j*{E_UPzG>y3PsN{p!Ipjxte&24T!qQo^A8hAEnON0eUisd1}oO^bWZ zke`+153W^J7ajg{p6saz7~$DEhjM&u8K-^GKVSfDNK zf&af5ypu6hP=LGu@3A20C?pUJ@D2q$gi%QU`@JN}a|qhspQAz`zQz#rfA>)YuZX`; z@IV~%&nsHE59AK`?;&_NrK0}V-dOvoX#e#dH5hDzyi^sFkpZu&`Zfjz7Pdx~cI9nv zzJL#~t)w+R0ue{THuKtWc-t^bE9{;~7lp8`V*;|Q|; z_n-;me4)|52gdQKvAB{Ncm-aD_(PEbPtX5(MZ9k;a3Y{9g+N3hGU6}QoKUuBv1b(8 zZkrESaiX&BtH?f2dpMY{%wXWpWd2F!v^4X#9>Y~psl+68R8qNyY=GJ1n`oFu&rrIA z?0{wCkzH`jSj6$nJbjNEtZA`{k|v!1*s=#0xx;= z{Y-xWA_*jq5N*b}-`q-yg539uLcBO1QF_wj9n^55Vo`cRkaBW#Vm7jxGF7|@|>7BNl zXUN?-7G>xYFU|xndbXg9d&rpTZ<7wPh(eUgMLzE0BRMFP>A+_G>m6cC$T~{8wkIUn z$m8b>1(H`KDvo=}`l17=`aaS?J7UnITj{thbozW5cvDjZqUQoRSJ^@9m=-a~furVyidO5cmLSBaMc z(>}OyGJ1;a=#?WzZ$~Ni3Te|C0n<`l9{lD-cJz#h!01h-{B?%3X$KLeJzh$)Mt1c6 zcVZwNss49jAeEd%%W*RkAjx6WQr*9>i#wdjCrhC*ufF2Q45mx0?AGmtX<9bCVK?Z0 zltWI8te0Ymr;FBNFA)E3YLbuVWP90Ky3*Yh%cf5Y?N7uWQwYGJS4<2Y{jT1FL8L`V zr0Zt=p3}>e3dw{b9x7UUcVT1Zf`RX-mxH$bdLCT+^0Z}RSe9JOWr5?IbIx5-VX8&L zMLw$!O(O-b#bCELjvFE$JV)~Fi{}lwkqu+)*@zQqG=0p#Td<~>jAqu%JfIFTvkF`t zNaE3DsoQ3+l8v|Ao2rn@pSP_tzk?!Z4CggMHmW{AeNE7u5wA+86b-w%d_UoZc1I>C zo{JiF(i9yR^{+fs2k#(yl!@Nx-4= z-qGIrV1`P86C1HRmoE50J-w~^u_hj!1DP)(@J^(OSyf6+u0j$yT)6%Ygag5 znOgnIW6kB2o#`rd9k)Li31?QVJ}M=8X3nQ(lgpQ+7FJ67RUYsQs74d4!ECNx<{lY8 zO9 zTZuWJ)~tOF*n$5@6Em2pR^K^|E}Y)p3Vr6iQgN^(Ha6aC+s`NUwor#axzVocq4VWN z&NXez%um^<7jh|$?zBqDf)XG(XPtg(d z{X|~tSMi+2Os2g|g7K?;x*wng)2#n-Fo8-Qzmd*lwZ-xMuwIeW+%`0Zof+4 zHZSdX(5!EK?v0%Dibsm7q3=HxcAMyp=TKhl3&Q)Hr(G9(yfe-Ux!h~IjdnF1Jx2S$ z?+BBP=QJnz@n|%G!%(_dM`*{IMC<5}F9iluPSgy9y!csdDx4OcxZrF;8|JBIp!W@^Vt5 z<>S~V-EYcFMqhMAGk@1|w`>vKp zg@$=A{3)Le`nt*Q9#8m<`P{#A3;X$*Lwg`GGi%V3kB%uWwE>)5FNJmZ$xsA$rYfRn zDcnD?)>sYJ*=%IRxWrTyllZAMTnafjY6xSPM&A2dYR)#7G!{7Dutv%JK6g(YeKC29gr_Cv9 zD*8Oz@clg(F}GQ@dgL~3c|8FQCt67V58KhGEOYxL4JqgZcGTtB9(@G8La9VTm01PH z4wGi3{8YJx!BPIKd)4uth-j%{AIYZGA+mv!N}5s|bAP1ZO%G<__FRMJSfTcoEtsxy zBr_gOO?wtSUz%jz)_p7)drfESW@0(Ne8jE+G)8Ew|?no&Jb-Ot9$0VP1cjV&JizXPhd;RC_;)n9}QxjM{G z;4ReLdfmyC3Q2}(k$Uev;r?f<;uv~xCe^hp?vo?^zN}%JR(0H{gAfmfxFjda8 z4Cdm&(FC#;thB3FF4n^9_K#w|-PP8`?*|_ z*f2hu^B-C9riOoNwo7~5Dl5(3_Y7sp?Z934JSe~R$qS{03&<(9N(L)Wu!ch z*&6x$^((`HxUDr;(p&gdvD$Zsn5Hjp+)xDFFHa-*?Kho#r^Z2=lqzc6l&wpO70vMU zZr#2<8P`4B7sCf3E5SUdVyh7+HtBiq$4}HqqM$@NL?!;Z_-xKqiQmD)h#y)vxAjJ%r-6s=d+ugTY*wPVD=1-$Uq<{tITQEPZ&0Em{vwIX$&z&D<<)G zFFMXwTv;~U1V^%HPq7XMT^y~-#IPt`Ik%t)-nsWQbUI(H5&!|m(CEVV7xoQ!mKQ9u z6x?oKX6(ncbzV1b0dORLe@G&n1Im(c+r!!<9sHDUaVFw%?V47DVPT^C=_JaU-W_Di z!5#GU7oJkY2_l!dZl{wuZ;!tc=|;MoY)+eNtK=I;v>&d-=uWxg?K_`Nu5B|3kt~_}&k2;IXKPPrJmA-dYbhSx z!DIrIPW!Q&f#^ai2lA7WFW+cYmM{ z6WM??g)6QN8BioxP|1+AXLJR@yt6J~J_1C%$(z}_rWkJXsm|b?>7}S*m8@^a6o38h zwTR$IO~zap`RgDEl*Z@`2$#_PvEc{aS||s-sm*mZq#7tqPYQk8YU= zx^9rlZf>kd!N_`J;@wprT~$ir@56uet%~X^vY0bRfpQjo|1PhTj!nAVL1#X6-%gRfY78aKDO!Q6?$z+LT$9n3J-$%Q0)-@)%5RuZ9QY&!ob zq60m0zP-V}cAtG$WszjJ2Y2iIK%FF%D-jTF)5BclS0z5NIr@F)u4vll2u{kpH(O&i zqi@vjXnB(%7jz+B02c6|2Epu&5E3 zl|+^dc!HpER$u?VkL>sdJ_i>i>9#q;fvjzRVF7-LE#$KgQdKv&)lcH!qSzNz*ODI~ znOLbL5xko7BTjT=hmRUC+Yt{^_|F27wC#O11S8d(k@^hT;Uj(xE-LVRqy~ztZT%hq zFVrK)z?j^6T8p(XS?^pHNW!NAlG6(|Hns=eQb@b;P_byX zd|al@%8Q|Fb4>>ATD$mJ>0fvDjrJA2FHf*$WXh_rh>&!@bWKq-XT!!5hj@$waW{z4 zG`IPujUif_d3XCP*AVh>F4M7fn3R6@Hr$|g|_&| zS*5{Nmr%*z;Ds7hyGujigwd*VY1SE%h+8`%UYsq*hPrN2g&&rT8p^Dg*dg; z>>yHFtM(D*51uSbiZ0gxQl!Oq#flPdr+PiAuG`faMs4m2 zc4w*q;E=~WCk?6jb^ivn%U?-GAu2b)tD{w>?*pk2)Ff%|mp=i!!}dE305`@*LW`H$ zgK*J{AWHJ`flozXi5k8`OQbQ}NVwl8L^gWpG{S%)TbPPNbdF)h7c6mALI4_eT>~8iJ6bUQ-^vR(d{k*M( zR36s{z6Dw}D*0-84?|Ou9oaYU3SPS1ULPsty{~wIBDisZCqIou;!4*+Ofzz869_F2 z7JW^}*Sf5)xJ2EFe70uRO|YBoUV%TGlB&Wef*|b-U&Xm1krVucy+-_$Wc+1E08Q|R zhU*D^-busOuQO0;scAI2)fBSZtt$gA0O~d>7JYL(taRANGv5i2JB@0#yiFHvOedkb z)-&hBWo!*yj9I z38F3qp4mrMy$Db{UZi~p0Bz4)gX;>Yoy*o(R5JY#rB0=V6(nTt>=u3|GFfU6@Kn%A znWcXJRn1{HjNd#RER6v$q()T6to}SGG`YIWNp-wczQlxJmdTFQ`9WEm!#&!@D1;+HkX0 zAH4LQtRM=Jxyn4djUk(@!m6p(lR58!6F|##pYBZWaExBPzT;lRFBc)~tGqE$qF)Ml zGuK;}o1b5C1LC(#0WB*=_KF-S)bj`1!U3+r~xY{cllh;cA2Sl zoh42anqwD^7rgPi&8ub~Cz3hb2Jy^=f|h`6kycHn{nl7zA?P~%I5`n?5mo?x9MT)l zB`18ofL2FwP4BonGgN7=S#vxjhxYAQV}qk7hBak9P$o^KBbaRGX5PAk%;xsyY`Qr$ zC)wqN+M5I-GpFUw2gGc8UyObzTMcKc2u`;^NWS_x7&_sK>hobGhXt!Umlm($1rJ74d*DGg$!#L*qz2?NQF#Ix|CC@ zK|(6E(LR5B%xnS;9Bd{`YHOE6!@TR~$o;H1xzU9VR5bVT^e(}8Y~K}iv8Hc98?v z*;K&+=d_%g!R(np>rDZkD+lUt6K|*ESkT;YkwD?tH(G5tn`7iM8TqkND;|ZJv3{uP zQs3{8@sfrgH;6@Ig*NFxGA8Wsv;RygW#MqS@vvuBweaOSwLnH-^qa!>Z%8D$iYQvc)UJ%s>A8D+D4>% zd=pO4tj~HpPhma)A`0&9X77((NM@^vn+iT$eC;MJM|^^ z{jR!fVIoUDR@M;Y_)6(_3u?b|Xxv64xY%dz3eFV{Byni2_9oL>qy`gBsMhtn+gR@K z>x`;-2s!Pi29j`f_d}MFw9u3}-$}-Z9->;+v*Z03{P0wvHZr2WdojDhbbN~PQQy{J zhA*O^wy^Hp;j>-s1i{LDvf6s-n&09_AFW|~@)}SLu^w^TO%?z&Ju$mGTQ?o&SYN7K zyvtYIaQfbOH2;k#R}yWG`_(D;ghb5AP;<;LL{}x*bvKOP9yMhWu)d7uQ{^Qse6@B_ zEc1kB-^Z<)IiJXQdvYdqLy|#gnvAnQ<_FLt29y(Y*}3$Bg^ zrsLdk2w%_uf(a*g069vZMhTG}XA}vQ-(&7ORTX&gkbFs(6LHRy4x{3Zm4H?rgkz?y zZ}ZecsqKa1Wyw>TU=Qzc} zkj=qpOd{Y=&^Npzuv;x=GG0XX7jDRE1X383(ik=8g)nVN;Rj4|XkNiE7h1@bmECEa zlE}>n=Alt&Fr9X=EVOZOQqN9?2G2#tEMK##D5{r=8s7{>c zZQ3MqA*0+TqG5DA6K#fHE|xI&kN4(v4_Et0b5_p}mdusI`dKo|?$XjZqX_c5hV|n- zN}P%Fw=*vIOvn3NYX~rc%X$AKt%tcx@xS23wqL&%c|*Jf-7@TzCvH&+rdeM6Z&!W)N5*t9&J z(EWv$Z~@0%Uf+87<<{ZFTFUfFOE^h$1fx33AhFZuVP6W^bIZny>^_G*w(8_v5rB#g z0h`tZaPrPT`x3_FFCw+&KFQ3oKQkV;Hz+w{li~1xpk6+w(QnFB`ROv_;cU-(D*+2g z$QYm7`H$R~{&&mSpo259g@}e0+|x+G>rl_;yl+Y}3#BkNj{VGoyMpkDzexEt6+u!k z8v_aZIcN!#mHn*l5txNt?d(3Yt^}4agYJS`@<8ANZYL=aD?FF;zZX4ZtzH5-A_AL$ zZenS6A^OW_^S>MYztg;hSok>h6Z;{_AnkYMzRoby+oz6sVc_n3qsJ>ysm~KF?Cmv+?(jWO6ez=jgh>FzSF~^Cb*07Tz!o? z*W5TBDfhY8Gm=h#Q{y}@Et_sC>jA&d&B}lx=;Ap}j)yg*VfauM8v|qGAV&|AJgIMe z5}H!cH#}+0*-58nvt|4{5>v@IUdXkQ>3zBMXK3?5{~t^aS}xCyB(mot@=|kL505U7 z21Ei_9-M9!H%(OrlM78@TGOt++eC`B^6W$mUI8;OZac9PXKpqKQ|WTTUuAa>)hbLz z!{fP3r%cf9+&hk`+Z@gbSq8Xm7DUuy>6{0%9Z_acDG(%bRvp8n&l5)$J8uUEQ z3F4Ov=^xAJ!pk0vGq-z8u0SH6_419yA*vfEE5IvN_0gn(D`ZWk<18D~RmHP~%4wg= zjRt=*;yBr-7>mCA4Y<~N>Hs3v%9`DyyZP-b$l=}l0PmuIOR&%P9MZ!~F46Cp5)J!C z$oQ>4+ZObCr)Uc~QzgA!=bV=CE_SNsw|7KtrJ05^5EyKiG+iGCXumaC>rbkbzpX~0@W-y? zvs(|f+ZyX5FD@JAG@nvKG}AoBh=e*oMA;tbkBGHR{zmLn?ge0c7y?^YCIdN4dQX_M zoN?aO@y759nDiAZB!6Xjsp{~91iWthJF~}V&cBoQ@Y(lZ$-WQevB$Vn`=Huz4Q!TcI6 z{?4(#esR)}(ehEu=k7s#8zWNorFpfW)|NxFI8=%AuZMX;Pc}!fg%d7MwtA8r=ZKY+ z&=f#;TwF6&_QO272DKe58Pki?WTbP~6yk@O_*B55mv#J7$MZKpJZ9c<<$r<3p-V4t znNL&Y#6(e(iCS28i9IhCX6ScQ%&R}7yTYc{CWJ6St}65Cuf zDki6x#}Ucd3dS7?Z7XihI>wFi*~VETn2mmd!hbkPUkaP`Da zCgU{&mMz>Iq}`xYkvQ5d)b1EiDhw)ag1?`0*gnHmg^sV9KAbZkH4k+kgT;>C&#SL^onR0kkk5`eaT~`^&syml5yAI}6G;7l} zJEfJ}mg!%wLD{n-FFkjFoEArP5nb_(o9to{OZXq?0yc(yKB`l_=?#d8HVe%tB3NSH z7MzeJ(BePM|8?bZy#9K!3>t#B`I|QB_`Elc!&gu;W@0ES)vl5#=_X4#?bWlbEE?6A z^r6%B0udJq44g-P91#k#{VDMsCZuP(v!*fRs<=sQbZjzv*7J?-+)nvMgX!?YH~xI5 zfSls?lSK;}p;H@*vpWSOgfBitfupAtxrmtg97trPriPU?>OE}c+mKFs7K^=4QC zqI4_-+r>hHb^Uu6uXn(8o#>H5ScusW>I3F-<}=|Kq$&)=DLf(Mb7)B6Z-lxeGYxR+ z#(o1?$XNiKfYQ_V49R&4PGF4aosult8Y4ISpf`lI@J9$q-3y2xB#4sA=Uj^f*#SV9 zAOC;N|9_iE{@48fkNwI2KYPEI(IZgb`q=GEy?ODr;7L$bhtR)`L{W6C-}8Q+zM=&G zjJux-$|dpZ%MUfo|4d6NQBRm8y5JWqOH{Cu2JUI}FXz7@BPx)>__pAOg3gZ*C@%$b z&i^zVv~%*gt^a7~PI8>31=3;ZV?xL636_AzETt#5DG8e7&Y9NfAz6!g1iTL0vAu~G z*#(+aiVhx4n*ID&Ovdw#-T=MnR9eon02N>)4q;IQlSXM`t)!J9xE9bnT(=ZIN8`uq zBjqy5Ogvxv(x5Wya*_!ER?eS%EmXubr=-UEJLMW=H>4So`8X<&`!WDTrHF1=OzVP>GnUW|L^kXOIm`#VO?4RH=Uep}p}) zZrZigf@m{_%h@h<96K}`nzCVd#Q2W@7CW~U0l7k7qpw>gx(yLKYh|c)F}nH5MC8OS zjN!nAN&kuMH0qwPb{W1);*!UMLj$}n$Leux`aZXwdks~Upw^zBzdfC{G?*;Sf;UzH zP6jFqWJRI4to6#@2w7PW1&?}Ub}^!XDqA(TSoNnhB$|dHHGz#SB^Zc~c!N-nITWcK z_?y&x@D}pk9T@W2@SYNFUEb9!H)0^;ces1SJMZooCODZF!91>8HIU4~W2TMsk8^bz zq3W!u0#1t9sb1Zl(obr+V!XX4w8i_Ts*P_gkvhd4{x#8gH4dg@!wsmA;P=bqM+<29 zotG!NZN{mX`)=0`O3$BMdZUqScw1eI)gVh@kq8NQ)H zQ*{rK5uacKd(Qc=(Lzln1d)*6ZQMicxmNz#_8IzpX<9Hv^3da6DvCiuj#B62xHN%q zeq%XthQ@O(6GQW#QK$yDTK4^d;*z8~<7&5ZtgrK>Ab*l2(6OIFyAQS{gGrd*M!(%^6gDYm+cpSVP9OA4P=^6b609XKW_tZyKW3Z z`-SlIdB3Yj@1}RdBpU`=OF&H-d2zJX1tgkLzT_?wFycpN_gdG)O(9CM+CKwc|1F!k zp8~lFtmZ%F&{F_{t*g1;w>p^j_%Em{s<{;1?*S#-Tr49W3%YMDpNY<|YOO)H z459wZwE7c(b~uUX@}_GylWnIrSsxnvUg|M71e)5x4rMgTjnW$8I>=35LIylSxUyL2 z=gG|ej%YM*!HhOy0M5@s%;qTtopLr;u6pR(2as&6YKrb&Au+6HDN+}vJy~!J zUTVaYR|K)|i-*;QvsNq9>s^mspr53=7s2eeYRXh4`Zj<)*RMrf}0{!^`$^4;6> zr4W=IwlWg>jmtoPTK|A0MuzempT;U_;ST*RhW>o2nkfUwMB$cV%T;mp4n1^GPi~u zQ`e#%mZq5^%XoSv@|?#B|He8UN?w~80d^xF?b8 zeQ&dw>X((~vvgpo8^=vRsZG?z6@v9e=%|aeQ$M`asI`lm55BoN>*-Gt*nvB*CDDUE zTet}PTuk_KUu;m9?+s5G6XA2^=Hg;fnH|gZQe|DYgRf@u@Inyf?{tCY@gqhZt#6MT zIG68ofaYfy9_d(5W5X5AmBwV&yFoo0CXrW~`W@~&#jC<@qh%P9sc zkNc1BMYU7K8Ki)LiUw?zPVYiopxcUpW_FXqks)-*9<;AmS<|M9qs9v~m^*@R3_^y5 zGb%-RH*B7|6vyK{W}*bdD}K>W#ImgWVW`JEiyln7tV^n_K=^okMq< zVP&n&xDI2R!aP-D=1n|kOGrBGPJ9$GwbzpDu$GNsVFq0}i$t-Ln6=BeLI%0$UYk}t zo_LC!j!u?g%lGE(HV%96Ih~_?#^Uhwwr_2G!dgE46S1K^K$Edkym0cg#%3jOIA_1V z@J8~cqMnXydPrD57Z#|aRQz3Ed8%q?GNC()ksgr`9rJmM{oO5p@rB++L|#;kb?C2K zW$<+A1iBMzFrPX9%K2-oUXOKt;T2dhr{nI^4MW4Ru}FGL9S)Au9GJFcMhk{BG;Y-0B29{f1T+K<^wEuc_lCVg6J9to7Xt*uaR5o2<8l&~WP<2e$h|ac zz;+rH&g%blEy6*5e552I^Ctjzx+GpN`!;s`ypADn3A$TiMh!v*BGL<78CA2R2(B_T z9v}AT!47EYa3+O^w~Fc%>Esh>;(#PO#rhSBAb?Jh{scN_O{qut*OnPTMLm?J-{HAE zR`@=Sbs&1O{vMfL59k&@R+e(4Q;cV-R3M=1O!Y!rj1(EFE&)0duGEJ2GEJNSDf!7K zvD5amUAF!9L_u4roVE;6mHb!#j*acqqqV{Q*aC;BIm7-Z*%Ep2^x&iBqq)VlAt2Rv z4IrT@`_!cZT9H3uiEqu4VXqHQ1@Q=^8~Dfen(qFCx;Ea1 zRn57`x!8FKQz|(g*W!_Ih9(QSPLwV{lKlzjyPHvN%qLHHz`~vc5xNjAkCELi3OtrU z)@e@KoTic?^j*81vC+FzPk*do!OI+G)!~#KUT9YzWNGAspPbD?*Px{cI3%7cTHT0t6EtyW6;={WvP9xCmg(k0yT)O3 z{v!=GPdhSQWC-L61ypKr32T`9>(Iq(j3a}$e!_36EC!@Aq{yxDE2c^AacUHm)So^kK5+X}x6>o}IU zG?bwh28O+$1^r1Wkol=zgH(R4^c5yqpu~C1ajd8UAQ<71P(0>GS7V?&MY*A2op-+h zD`c{O)`{J6SBqzQeoR<>Z_Zu3L-upTsSvqGy>_QH>CRHQ#psX-jT%JBrsLP=D}{rg z+oyjmy`Swpf1BHw3YRvY`0>XYwL3>XnTfT5^XFdi1(SiE}0Tte@k4M_&df=+r04>QXKg@np<-DPx-0}f00 zXWmIA9`|GTiC*vCyKZ1zR!xP~? z@E7&4hFX+y{M|dwy3!l0t$TIF!P{(QBVXjYuonieMwjPQ_b(cBC!H@WniSAdW899z z)IaHhb|{Y#W9sL!jBJkH;#qdkdo`Su(SQs9%ALw_29_`%QlOSs8c5F3uA|_Q^`>~& zeO3K~?zB7E{oxkshX_P72N2CK;g@Ft6(di^r8lAPY=pfZ66B_faLYcgK@lX1BQ{*z zVq|(({hX3g!dr_RD8oGCVq3M9ykiC8MDapHe*QV&0z-swOe7<**(`8N2f6Fteb zrTu;K$eH9m-`{>W6ahf;M2>_A_1H6#OJW!Lya&z0l?tXDu16}ks`+ZyBGLh)1Oj!o zL7Qhzbsu%T(X2SFvW$o6K5pG1LZ?UxXJf0He>8s{62BQ6aMhr*e;!?Fd(n|{&LL#_ zmjhVeiWa{IR@&{;7+fa&fN zCX+*~!}emPBe~>dup4zYL%P&hkTkhddo|~0ZY#!g zy7yMh@8WFUL%)~P8)a?GG=2jUE!g0nMOj|UIBuZl#Rjyp!5dIv)rf@ScS63&j6(EB zj8AXN-idTnnNJ&9N{NJgf7{T?%25XKsaL4dP475^9ke|udm6EGM?R6aMPzP^x;KSq zKqTb!W4)XEC@e$+{-T|!g|G}PPJFTxQN04d@L|Yk5H`VLGLV0|ur0MWkQ~WZ6|ot1 zF=&eL-2|baVV;kV!NQ66f2<9`{WPc;v*luP%td55fIMw+eDpM)wbM+SYXlDTNoe}L zo<4nY>q@z~iugnvD* zxshl%1uf8eF(vFq+R8Z*9;pOpxp6YUrU%K8XS1NXFGtN zM7Py9uPG?+K=N;^|wZaU$n{Y0SlgSly?B@J#zT z=PUE+6!sqfDd8Z;#KuM-HHDXSxm%voe?SBLi8K^cKYA2TZUP^|p(ZYS&~<{*y~LRT zmogfoW{)`A)!x&dlgT^uWUW>q<80r5Ws(UvOe~d$q%ZLp@+k@^>KQ8DpZChuA}KRch0g$4d$Ou zu}_4SDs(%J>X+xnBa-+p^d&dbmmI_}@lW&H!EEUj@dKr=pA#rb9^Cp-rF+%Bb@Y)> z<4>hi;g)P_J`{duf4@r(L|>!(p$<%+hnHm{u8%f8#3ppS`k=$!G;wxx!h7?xL&3K%hG$M_^ZUI=|F&JF@lg0v=?VMTHQ;N7SA2zB z$8rz$%HCbkw6v5PZZI`zZPn;DJE>cFbCd^@HWNI()*pLvcCW0M|E~oYUz9RLDd2I) z#Y?U^|M2N~t2?}Qe^E6qV`6>9Ne$kcSYYRYNH!ez)GaIv6BTBlY3bnXRv9-dSq*tT z7E-)5Zc<8`1zXB!;xr#+xXYYPc)luI0vOy=YHO3@FnF9*qHkM*+p4E$nDw0uqUtM0 z5CZPud3;4-;eSK$4Wh?$P_^n0tLJmYb0wGgDK%0dNB>KMezOI`I4t{zWk z3HTOJQa!r=LTlT>W_}T0T7Z&4~D5kDFD{qb!$$6cN3V2zHdaK ze^9qKC2CGKjk7CdRD+pELl;L|{u6dPQfW@wwD&POh@Fm~xr3jeK%q$2S))vYknf>v`Tl@YX$@qcG4lUjf<3R(U9KiW zakTm+5F@!-+d&Q#0*s}_Ew2Z3KJI@AgO_T0u&uytsonc7Gv?C%Z6k+L0rx0zyQK9K zz}2<-;fJ|Xn{vRfnw$Wdt_=hd2ud zhWIuiN){d&fcb;lS0Fm{wYLG(XVzkp`>Ri26u%~| zWf$+-V((UAob?<(-8Dm<3bDP?sogoH_wJi*p&+}9t7vg0hOrM;)C;8U>=rlJDF=Q* zfI^Vff+li^Hv_nsW%{cnWrbVAkBB3cQ)OT6sh=Go+Su0lb!FS@y{$Tz;I}N$gjvN- zO3HV8SR0ndZA0+yEnuH{KX>dk_~rL!ZiT>2p;Mv>-F9!5Y`~`J|6uRU!=di~zJC-> z33Vz$s8q6tQ1%u@wxaAy$i8MbGPY15lzp4*$-XaxA!Ij}>-zrwyN~0({yRHP$84YX@>-tH$Lq!r#dWDVQsUet^OAIZu3K}{(!mwE)OdZN87sk>2X^4EYW*>3BVLy81Pm$|*8k`~emm7VDb!WyF4ihzyMS)sQiq*a_ z6*dL8C<_p?T`>eP+e-zQvaRuTN96^<6_0%bt+HQzt_vGxzITlU4&x1-cSV4R4f!tV z@i=z5D@B(@2%;-bat}!iJeQ)I`GAg$&M-BV>Rp!;+ltd|6SO@W&D}uG0z65t`n<-5 zfXX;<2l6u{V9lyorB^YkkKRLFeB-J*yTUK-3ZEVwauZYyb{N`l!Ce@k^)|8t=6%ng z#)&!ejUdP-!5#7OdNX@*nTZt}wde^Nd6Y#|RrIg+a3WFkZysZ~`z8V0-+@#iiH0kzgF-zLdHQ&|BvQmc|XB<==vQG!oFvS4n8a!;m z(YVHFOl0ib>&Sm)I!X8toxBgga7!d-zpb3Qw-o7w;V^+$jBN*@9B@A zI3{6<#Lhs|os7I@JhoP{`cCXSQ=Y%KUQ0MPYk(;>E@3r^1x2r^iaJuY0Z=THh)i$Y z&J)IOhg=pLbsoQAo$iw>B&r&w+!HUfUR~_pWVf3fz%z|E^>1YmGyB#S;5s9ESKmCI zj;?deP3V^2#KGP+?Lq~aFOqEyO1fp5A`%@ET3c+nMJj97PnLOMw=9I{F_AV3`Sbqs zLSxY;${#uwiy^I7J z$4i>BhqbVZSHo81k|wBVJ!QZ37Q7p$?o?2pxV%JvL&3KtC`7ALPgG^q zw^iXXxhk(HefBf#Eb%j_8Av}+RcJlvH0)*SJ2*O(hxFF&j^&%jA=6|Y^%hDVLcc@; zwWKnj$B=8F_s@s*iMI$?oz8(JzW7<{0o?n%!?}%SJu~v^!88O_wQe&M zO)6wkFt`$O>U=rTY4+M33-F8ZP5+Kc5>+NU`4P~29G|Xh#sKYcCs4HsAr%NliXIqM z`7LD&pWi>jlse`bVq-VOaTiR^Dqh+GHI4^|;_u(E0J(hnJz!MXQ~CsCr8CfgD4MZp zmw5)W6W%+j^}uWXd$*C+5HL81A>-X5O&jG`jVLrh|0YMNCmt#{b90#ay*_+qDP%v> zinPDGHFts+adC&r+XQ0!CW1{)dd_GQBRknpC%gA@`G3oHPOb!e7u#K(nK~`2rXHK5 zYnwyY+XuKEGad>(Gj~sy6cgC_13G#IdB)-e(9dJWc-5jgoN5PW36k-``d6&1Sl}F{ z6q-q5W1Tx};tj0_x;2qdD_IUFe~bC=3`2{ndF=;Og8gSTJ+ThKZVf@eHvV3J)+k=Y zbjfnB#Ory%b?{jU!C9oBGaOD7YM`oTSxFbm7&jf5&f4b&x>o5$ON?_ia}_j-aK7;w zA^I9pd$-6n-QscUbaqNv_9w*{uTRAG^c18dIoV17lk7D9Zqifj1}CpVYo&w=>9i{% zQN@-biV*Uam~Qfi#UO@%q~}C|-1ynL060F5BQAayg-#!~#ExmwGtNzvd!+d4`%i(c zi+(};=?gapM8chUM1?1#>JfKO__C9}Emt5B(LkYY*n1j!m_H*QYE@54%uB~!0UF|6 zG=u|4e`DE4aGRcv6fSp}m9cmM(l>9(pb}2a=kLKu6!`WWd6DYCeCE`8Ym)a z{r(lMKA>SHZWSlqb<~%f^Ps&q_u_J`R?g#|cg#n>6&|~FG=V}tr9R?Nz*lXm8k6k^ z_k8t%iM5$|x%WIl*D0Hp1%%Tc0ajMi>rz#{u{$y_eX_qFRj2QUtu5x4x(04m$BpdshKI#+|%k)hZ3omG_3_?Qdg z3e%UDp3@ZR_P`y$r~QjYZ?!@I5Mz#>czy2RY^FQ!T|`@lyg%hJum0*p-odAoNKk@9 z9VVM@gTlw769@?k+1q_-QVK7uvgTazK$Q`=b6wG&2mh{}(4IW1OkP>?tvd$fmDTZ7 z!J>`Vs}U}Y5elPqQ11>Y5?c!qL1TQz{Q!ej^?ZiC3N9n@=hX`9Awgwfc(HYm6My)H>nDqLB8nqC49=a z$fH_6N#mI;5J#D4-)jOw&&U081S5pniAeYA?^1nCOVth&>&y0MG&cmhRX7#4HNS(} z;)86E)xUi$7&G=oS-%JEUc)oZUhE{(JNKwrIu4Z_S07)O;O_TW#MBg|uuJDB#^$P) z`)DgnvBf$0=nC4rvF}j?n*j!^D_xH{5J@E`77>Z!o& z!k?2Rricq9*3?@fpnWU~DV{rrO`L<+)**nWDFFFX3YBOzRypcX6nvDl8cwRmLT0Td zXcSBp=E43Y-mb|;hr<1$0bXU#qj!rInv~S|9qA?2JM6A274*g_j}c5qu(ox9y~=97jvg4E}%Z`g+L;vc(*@Dsrvk0j`TQ!>7M%p z5?_He@g3~XxCB#tlzQ*?j-CW!haEBK)70S;M&KtsqH=SHpujVBTN2-Y*28`}X!Ag>+{=xD{H- z4i^=eB1zLTwPwMvl8zM9u#P)WAa|tn0FGbIJdN6jw)#X7-vfIa#OkIe=799PK&Smq z<5ZOG;IFdVi^+opx|&B9_fcC~k5L?`JIC4ezbZ9ZZ%9TF@zBsD%88@YG5H8+&BAp| ziA~d?8?6o;YL0@1FRfI$36Opqv%pO6bXaBTWU&Gnq2ijA#QYmDK>95|bI4wUe})Z|LVXw}R1mI#b$cS?n^L zW~D6wz*AT^l8R37CkN5$NV|i?D*oYyW)6vNn|>9z{3y68oO~3M7?W$e9_-V`^701o z){!DXIpgGX_?WPii$RTq?Tbp9^^?5f^ii{iPg;aqc0Fbp0tOEUK{x)yS6c*<;mgor z`d-{8(+?UZa#Hd)E{xVi|)-3G?xj zY5cQN6924}1!(9sf_P`I)4aci?Ppu9Peh-TSdr1*A#=)k+5M1kRB%dV$**+(wOBsr zD;EmQ)=(zelhpS9eGK0Qw-10+`f!uZ@#?73)Oz*c)Tl_^^kKM{Tn%@YN0rF0>jZj9 zA+O-CABiXz{WVI#s0>ET#K#PfV@e-Dh&Dss#JD zffWrGpRYO_!l;J}Qk$_Lh7Qz77n^{IO%%9D#}5qI1G9)$FQ`FSE_N3bf26UB+zFyS z)!lMGY<)7R{v@dX3FX}H_yCprhfS&cPEji_pwKKAK0-$L*d47_5eYcwqt$2S=LK{s z`!=_xA$+#W`O>2&)s+_?*X-vx1C33k!$|im6KWi2mZ$Z185Q7;@QxhEsqt$J)2rT9 zK^nZTL9;%2&OWBj&5DSA1HAFaO%z)%^|`mxh60{b0%OxPuQfe&v8200EPuD;!0Ny@eb8dbF6PBZam*Qg}BP6YPfQDUw^tv zmG#IQLo4TUbiz|V7-MxO)K*&)|H(aK8RKQOr)okYl99v9XC-fl9xD;y)i@BaVrdU} z_1P*)oI_UORPyy*h{?Hm);61LQ7XoN1llnp)vZNbXH=LK8p4(2r% zK;~NW{1XwiQQS&)Peh~sEPhc?dSL|OALa0Cm1?E$6S-@NM&1i9Bfz*ja=e;d@N{mHn`InfDi_Z7>UV;%d} zMj$FFWZ>Vl&Z4Af#rqzVpD#vhwf@}raUWy)3aT7K<|ICoB51D!(Jv&J`o}Hy#CNYx zlvzkaGVi!8cL58qFxWTYj+Fl32$chpEoCN6KFpLdMU)TCCWHGbkKIIh2&r_JD**%q zK32(QaGe}oY}49)!hT3Zf`%^0Ml74yb|4SjUHFHE8oBp#G z+vcg!-oTv)&8Eez_$BL`yc}qzJQNbqt5pUb(0gY5+E>1dEcdymXF$WNDM98o`aJ>mUvc-93c zrIDrf5VUI+J?0aEmk#{#-FrVj(lo23i#-R@AhzQ7FdWT>O?3EmoT?^b$e)9OeOg2g zg0HkM_qBY-);w5m_0>WYkxpQwCng(E%7VKlu+sn6t0QTU$Db94$IXnjWWaI7OSIQ* z6$v}5;s+v^Nx^?WdqWwD53$bPQ37SAg|?(O-z6$Fv=mxF%KXx*FXp;c@0Hslh0j%K zPG7i?%4w|t&g}gVRZV8WgHc2A`P8w#T5^)S7#D-5KPLiei_4Q+izk|DH`i9}$+^>t zsD;Ce=oIc*Pc5U-5>pW-c^CiY)p(&Jy{%;=cXXbNlfuIjA3PHSurB$Uvrx&;-9nmc zzO8ccj^sYM9FU<5h^DnspF(fvz%~PqB%^*n9~zw8NM!TMcA6o6!^^l&4-lRE80Mb# z&OvjT4rW6SmDT+#7l2ps(YBV@pJ)&CtR$NDmN;xj3vI{Ct`%kR-45WDl-ijku8qdA znyx!`WPKEavg|u7TnSLjKIMIc5aEIP|DBJ#Iy4yvl2kyZSocp7bvm$j-R_eL^OLlC zF9XUI{LngR*$(I&|0a_CNtiQaIU+32^u~xJ;*#%r84D$%&Fh5sBnj3i69~^Y;J$s> z8BO!}(~|4zdu5y4TQjF4#a`mCyyRz(=DYXh^!X5FlIm5hm?saOT6O8Y61Z|c?v~~u zTHkJ&}yWCS83@ei|YR#Jt?8i%EZrM&gdmMY??0PWWKNY0QbF`Z) zn-VsUfJy0Q!SJ2S-j9&*;=PPUvsOr2>`H(s{N{HUpGeaUKO%~|Fv(%5s)I${s*`$k zbk}4s;f}B^+kM|j;I+w{Hm7<$O`rHEl<|{6m0g_!UYkJnDk;UDsJu8lQYkHUU!;>W z4NnVQq5oRvcBK1r$Nz{>n3%!5F257jHlFugX?+Z);_PFwv%-s^zvqQ=GKCn|NZP#U z(U>$)`4s=@yMmnjUoNgbyV?kG$4OMWOOzP2ZY?uhL~@DbfF?@V%m3wB5U2(if~7?( z!68R#+0UJ}x9zX>ll=DIBAjbXNE}j5f>J1V?FX0oGQDOc$`Po7#t4D>v zqw64HGZ0)IChMOg@X38;U=VM!ToTSWr(gX#v}<6epQ38<;1Gi2>kuW+3iU)sh1XTT?!-Hr zW8adAdy2y6Kc}8OJZzhB?PfFg{WVe_6>Qu;UhP=$OsY{DzVj=VPpj#m&PnGW=0+xm z4|S5*&5X!lKvy_>cB9aP%gduq^*z_~TVE311LWWRBC*JsdM_NyQnwNLjiwV;_d8V_Gs;($C#uxW&)JQ*_VXlk4sk3pV)qp37 zo<-0`Kz=${QcTx%X@~kVOZ=%6?3XHzh4~LMCIEgHo%-qLmt%cb+ZmUTCu39{eoL`) zX%`Y=EyJM)n~$cbUBL5Jo5LWkP~94m{zFEQewM(9%Mr19pJ*n}^z5;w&wbyp!JiGl z9;~Y^d8b9E@=fEy(Blutz8*eK2zq?}!>3PbQmy;0f^vM)0oLE%Sg$YH5oeE_Vzf2o z)IZ!=O59l&>uZ2SLQu1~35#x%j!E>7^bqAgAFGS`b3;rbsRa^h>kK99wt(cT6>qQ- zB`>=^<_EFICx{RtOU2}YXQ8=(*phHj^9`I^4zJ6lemR+U#g($dRpdj+F<+4-5>h-$q{(jdX6<+F|Z`ZLaKU9F( zR2{yL@`XPRj~8k!FTeQb$$@sq)oAd2KME(c{_EvWgAZi>x%B_jDHXVWwbTn|LKX$* z=#`n%3c1{nF;-aqe?uS@oybEqYG%!f|LsvQhGbV_kuwwTzdME_dWsB`#CRp5|J~Uj zouj~_^%6bR+Vzjy0<84^++YOu0eJF6&ZYhT_2mEa|0=Hn3u9s*@8bGjy!8Kk<2v$4 zn`vlRo&0h1@Sndv_`iV;(LK_867_ejNW7K8|uf3SzHxX6RX7xMh!(c86PHs%tmj*`B2TK&%W*B@?at z>vZ*9bAbsXy%NYZvG$w3IMp)q_P9o7_*SOAo<;`O=~o@9Io)v!)!Sf-%y{dzncZSEq0h$GvedNoX1@eY~sPHsZuzh z)gNV9X)yejJG(KpnA)Ec?`4jn!7lqJUd@ldig(?;Gl-gA1w{n6$I!YZ}FM zy?H)*<@v`!j25O-@HfEUp&hw}8TEN%^l-_3>uP#;h@$+DcKKBb%VVtb{UmG7j zFVB{FL;BI2Zhc)X`f1j~$vtn8zjcd@_-1fuu5ES?3%`76_wk!#>gWus|MAJOf@(}@ zz^@gsnE#Z)bpvb+H>)LR18OzySK;-w`**6vExT2>h&V)#&-wnUsWe~*6+=f(yds>f z-U8dCn)uim*JMWQ>D_9$*eu=>E5ik~dArCV|Ldt6qQE-zqvP>S`u5Dg?9#g*_Roq8 zl$vH5Wz46%#W^IM$_uIrGv!u+vP9;V@3OF$3Sh_F8|yGEgY4R|-F^|^FkXtXQq4AV z)A?b4F=CboaV{2(483LR*SB?E0CVe~@1S^T`P9M$1()N@!9euoTqT0JKHRC1%&}TV zwY#U*9sS~u3Hpk#7g8ppojm_40)3_~0L@(~vHI6Ve0X-idMPY-IXPe3^rNkI-ox59 z(B>0(OExx1jTH+$0%qyn)H~YlX}MEQ_Cj7@l1-buc)f9(#$gg5j+tfyZ0OcVf10NX zdekxdpl;C;y==G$Hd<+8|H}K-k_LXrhe2E51p;$Ptl__(4Qod=De`-1+?l|)Oa8&% zJ^>DA?;WdrFd-_#(7DJNd0!wSIk`Lx|2WGsDIrh2VsH1_0-%U8^LJS044z=x`PB94 zdKWOa+(60_{afnnaq3;p8PsZMM0VHxT z#aBIo?Cl%Of1)lQjTgZ0IJ94_gRl2ZVl|FrZ^v=?Z$113I ziJnzBI>otI=sf>3<(hdip-lj9rJkZfW^6S2={QFxobU#WSBoC*5GDZ!Up+wie4?{m z{%4M6pzQVN7)WGGFm0FDtRE}?E?{4sj7$L@Y`$VAMGk$KGvr|8S((eZ`T-y-pxe8aT>JC7jYQyQ$h)%K?hW-0ghiU?QqpO?J43B>_+ zkH8nND*+vd9<}4!BifC6%Yh=w^e-Mzt$+sJaXa4Vg-hVvICzT(T8s>6_wW`CyRXig|Gn@vq6p=JT*>UCF%J372m2dRxKOf5OCmQ);syqz@q>y}u z*Kad4Doip*(MNl}Fru?$ZbFiW~5gN8=`b(q7nm6q=bf zNxiI+Z*;DE^(-BS&`D4G0 z;g9*zw0GU>-A*&J@=dZ)3KUiQL9E^}tpJj4d_rOc*>(iJ44kg^y>QVbJJ2m%9n6h4 za)DaI_&joD9u%6(ieh&0R zObSLk%jq2Ou@i{pw$)+Nm?3Kzr=M-_pEbGgl#=uNuT}rh>>bkpB{!Qh8`oG(Uz$Rm zmpr`>%z9_juC;E1B1NKbE63metF)(_r}%J9;uq&V-?qBR)4SP}o$ASwaX^zp1AKB# zG+D7ihLUjq<~It)uW&Gn~=zlyh)sWe!1#N zV*03CEse>QEe?(Q48QR8N8t2J=GroiW*U2aDU1_Z>r>(y{G%SbKVY#?dEQ4j;1SOk z52GiJed`7Z_G95;2jXvZe8kV)2Nh!;?0l5Mq0RS4GnatUOI6I)O=h7)=jlon#yeMGMj!yn#}~DF2lbRWMY+%zJ}O)U3S<0IN9>i?j-Ec^JYPoZPYhRofKaGRKI|O8 zwSZ`7-RsK^LRB?nyI1#nrwcMUfXoORO-Lh2+s!<^k zWr-(PiN9{k^88(}@*3`m`Pd^;V8^O(2NJPL`8hf55TGY=?BYns-z`>&LmbWrrUIoa ze%TT@CWft#tM66->s+NuWtNDOq9(Fpwmi^MiDKU`=Wo?g>sw|oz6CDJyVVXT!7?fU z6*0QOYUFbF$zkXE6E5i7ptrc7ZlPE#{Gt4x2w*F|8GP{^a!aE=xW7!km1rF}(26w#1&BD5ptm8{Pu7Y8Tbp2!MFPG{H zJSbJwcSXeaec_jP#gA>Rs|k8L$AgVpDt>VYbaPOd>?HD z^cb1XmQzi7R@{Ez-ZgcO*b8b^zRfS;<4FKg+iBJ>YuSrG!KR(UX&h4RVM@PRMoa)) zIcq9BWdOxGnta)&3Z*2cn)bYVX^g0Nk9-c^{!PV}KlauuUZmmXQ<@!|bA_PZ?+>Xi z-{>~W9~GvES-#(CL_`@^+wIbd!b;9^BqsGOdBNqw`^9j&w^n^4!PQY-Zqmwis~^^T zh=ScT3z9t;I5WtVs;#9}C`2jys#!mTdEM*2u}T zk9dVH!80=V(Jt;U zr#*jIpi>-sT^*mmxr889)2+#ODX4v2?QKy0k+vG%kgJ|NPU;8E48NGfwM&A3dLx|` zR8@AP(a7F2X@e~n<=>97_IhI^h-xE0D*!9tLLsBC#@_BZps|jLUFr)myKD&V1`fv| z3JUobDi7{HyT+W(#l6l0xo23Xnfm+QgC{Nz*XZ2eMrPdSR(Y7=XO;}qLk6!NPl4ynOi?C(&c(5DYa(S^(nprO{R3eA}J>Rl% z2}^P4V`A6{X)7%dj*8E7pIVtSK0^Jbl!Y2c*%vP2a-S){sO9=Une#?j=isjouA^xZb%@ispgA$?t#=;%ru|_Q>l|~lb{ZC`os?yFLDLb*Yl>8Y6H3)xsjD8XLwl@o}zq@kxlZ(U*fdvbPGm3wmJtgBw;LX(PIT1x6z)HnJ!~kwtmqt2iCDnSO=plu@ote%-G&*1!`T27EES zMW^JeLI~}w7+g*VF62A4ioL3b<4?r*N=4HOt4K#5zct&u$85t|MAu=%s9Csi>F^gi zKQk%l^lih3NuWFTrHW{6?1hph>=#Oz(@4V^D!MxxPd0+e#dw09(~%kMkiOn&1|DWf zZ(MvoZZ+B%f|xa~yq%#FnjvFB{U=&2W=?zovGxUajdc&(7bP?izg7GT>MzS$Gz!xD zYSckaks^y*&9K}vf5=ini#;X{t9B#$?yN*#NXq9O@Uttq zJLwzE_y`i1&%|d2SsszT$nbD5yT=-t&OB)0Jia^5@aurj9?~V$9Jg0Da)kF5rHcv%zk^X3V@7_Fm_eV3Elba&r7`PTyx3=$aNV|`A^xHvet1!8 z)@Z8ov%(0T<>u6_Gp@En3#^XLeNL#El$Fmyh+p-V^;lQ+iEDEt zYWnY4Qh5(iS<8_hPV?QfHiHeez4Mh09w|nidxz-S@3e_q9>n_*g0@>Iv>&2KAO1CnD^~s6sOe!z%%#;7F1ZojMbn>^N3iPM!K`nW51S-gZK;W`ENO=M<@J`Xi)m|qCJxO01m;=ywoUgv zfZ)1lq`iaNE=3PweqC$)0QCDxF9+>+(H6aDv|1ntqr4^qtmyp`TC~SKi zT?zYVn$K}&R9Qie+FHqv*2iWzio!NNnT|0VE5xy>$~`J#7N?wU`T65p3GF6=*#NOn ztO-a@fzL|QQU=iCt(Ft|MW0^H^33GL*BA|z7Z->RZU0^?D4FIz4un5mno~IXwkBbo z7Jo>XCe@h0idNOLV6Vlzg|F`$%8rVvmD-oWR;+V+U~#+bPOvh0@onGP2(A|;_tAGM z8*)|`ZHL#z{P6z}hLv-hOEZ{os*pgiTwLt59$R4lQK%Q7so8t%G^|k zUMX(BJVk%w)rG_GycxI`y!|YU{Ry+qW#$;s{)mFtgCqb-=6&8@WT?ROVRN#ega!?) z;^zdVEIMd+&5-N|pa}R#4#E$4Lc`dihf`fUR%HC<_c!tM zw@OSy-}UDPDkeWVYOHWE+gW=dUl4Jt^l?Jbi@P=7eV?;RPlfeV>S&qpn zQA)7gf}xRM?8ITn(E$TI{jHj))wd6eV|%|TpFrz0(5!gu42MMtF-Vc6Z=TmH9pc(IdyjS8y3?@ptxB&&`Sn&2} z!aLh*!>XRq{-E-fUk>I#5L))VEf=0~lphyQdZ-VUO(9g?m9NTtbN7(L4>VkcvRQ*1 zeRtQAI@>a8aqhs)Zx9G)SP|lG#zjRc; zRPC-#3J&FqWz#YacO&0I4j zG+VU`x@(EpFEhWM!60(=MLL3Y=l(3TRGCL|@l6ZE7H@JM7+38kU^S2S&O^v^(0kMf zv3GGwtzk^`jLVsOBzx%Yh;ERdZTnL_*I8C7?{9DPK7g2T!7;GZ`inN1^&hmX-jKj7 zZsIG{nK$*)UB+u#4`v`N)_UP&$1CJBLcUPD@x;qtZAXo_-O#(*h9gQf-$%vbdgf`b zbx4PiVG|9TRNh7}im}5bweL};Iv4WaHC-67klXHkvvU7UKb+Nlm2NIc(!*{pdvtP& zo9<5oGw>#*Li9b3oZ4&zY{pG$ed0Vmbqmc$9^75i^9)8S7SN%PDBsifWR_St(%97L zwix}h^^o9evK?j|g(#`am9Wu4>+Q5>-|T)C+~b0hb0uEW?6aCD%mNf9M^W%TGnhc8wM);P*5J5XY}QMft)9RHm6-47ou zG1JYxvg9T8Iy62kbnEWL`614h9i=Nr-KWqLbc{cif~zWW#1!f^_1>yCF4aVDFsn@7 zTxx6&$0#6Z$GxZ4nk2C1b$4|1TI0N@td~82DEXV7drpw&BKnj)d-GWl+l88?kkKhq3|B1>}F`5>++&A2YJ1KDOd}x zan2BwbWxmGnTbwL#|ZVMy~fjZ2{8tnOPM28&*7@YfA=|?eQCt5_$bXv)kG5AMcBbN zy7KyuAnG?cG*yW5<+6*CFjG09J+nQu2#z8rB3LStLuQgQO~w=VU^)Bip4*3|e+idB zzpsftWH4&BHxOFJHlc-mPs^F;4aNInWp}RWHZ1fDHC8*p_U|`y3e-P`_b{V}{UP40GD7K3zOQ&GJwV5y&N}v}Vepb+jjY2;M(Ogi!uPOw+$VJ{-t|qzTuVu+CGP!5j8hAs;;ZH|0s8Zin)Te^A!9Z&Mvs z@C{OMVU+qwHF7cO6FZN=kUO?p$)k>s=%3v9J^8P4{5JL0>NRC9H8(3Hi#TI#<5m@VRQ2#+?*z4WDX_wpRBc;)kB9?Y zL0LUkIZ^i&9`n#=(Z6`pZ#W|wn~M)Y=M2n`%Tuzh+E)4?c*SwDnqf4F>Wl1V+YLVb zud%&}qT=9UA{UiKi&DVc8T`=d2buM+T!b0k+?+I+ko*2yo)6KB0y>ksxu5xSspHa2 z060+#j2};&lEaT@nM1N;EI!ETum~AAY$pcgJ_Gf(G_E_Yu|^)dBc2c(ihCiGUZiq( z9tJfKu3ONO!5qC?cI0Qa?2K+P=Z`&5&-NoSy86OBK}n7hE-P1DVb%B17u(e1R0E1K zTOHg8+U45)V2Q2aD22yhw$=K+7s~i5po4zV_YX{iiMr*2(eB+Eg!NZ2HJH3{cbtfb zc5cJVg!cX6+E+ZHGd?Pxa@BJlMWvUOm!AGp8E2w@BX{5LZ$bkCQ$6ZBHseJs2(lFt zx0%Rrm~eZ!R!OpqjBsLIzZFTR{q!oBN+^o_{t=&4IGkYgI%a0AoliUa^`0BXeMZ{j zWoziJE2e6!`|Ts{@H$-~U)!Y5_d>Q`4lQ(vZm)a`3q5^y*if;}y=b0~L!pl|AYLAorK@~VOaqcR) z?G2^~N)9a+lMqA+?eX39+578={q@G4ADk;5wb_fYSW|=7HGSz?{|VT~ijZ9yJ7Fsq z&LzH&8vhiv*|8W^#y^s|CSljP@smeO$eQv`KGJelc?S$c>KhlecnP2#o-s|Zf%3-< z*~mo}A+2>clWj&0Nm(pB$(#7zq~>K8aj0C09)5ehQCVlyzINWa9p|T8JnXkd{Ftc} z}wba`aosL}c;AleoMq zd*;2nc6a_R3Gr^gv>IscP1UkkLz!fw+L&Fgue6CwR3!CH@>YcKjuHSW=3hG}RMtaXYg=5V9_UE%j0N=q63$+i2CK#!T zU?Mwqu~|ac12Bs!7~hbUK=g5+too5fn{LB)e7!uow9dAM25G{iyxR_#6j(F3|M~}u zuCS%MOpPBz!Q;08p%4!kC_oxuTnsbI2VfS-D8{?&B64!j*Dj|y*DmWoKtn4?wrrHx zBWAG!rpKJ@*bKYAN#K^m7>pC;t~^*CG4ZVAeIzlW11%W3Fe?S|J$~ym8@}&{XAdT) z!hLnn&Ob}f;Zq-Hr{5#6A!yT&c2CEmAVEjI_=&lLDz(zqox^2bjNii&5djFe=mh*| z>}LXIi8EMZc+mD==s;@_h>^6z6Y*hPH$~)vJ^GEq2#l(SaywzC7~oQ%tTk&m7g)Z7 zHrg2%t`k$wl-sB^TdiW`FM<@U)XR9O9p?p%9~7_bvJYDSLfNluOBQ9{*Q@7psYJ;a z(zaCMnX6GJ;WS8(BmLE7WH*Fi##x7i@>;CldA+|#d&Hjf&&+|qb&=*X2DwALI@bc| zJKv7wlsPT)7yfEYd`FP@tKju!%`0kSg@6vGw}X#!9(*#*&k7VO`}jc=UH@_?toAQ~ z>?5GBx%*SMBP%K{6o4qAdE;vkk)G}A{0o%>>4UG(Mq_>KCl{zbR^IEsHryI5$IFS- z!NFJp!cKFo>r81PrZ!818f|@8j5q+C{GXM0z;gzFdN)uFC*88$#uV|&=fJN401TWq zo5&~Ar8H)2dv5?GiKp7VrQIO(l<#O>_N|(+FM+a~Lx7-06%K5c?Y(RGMcU=>y*4aX z-#g<*DrMtQ3nl~Y0=)G<^{Uo)0eL!yaAx7STliv@+yCHf!^Y6^dtdA-&Q?g_O%90B zeep_6!{-{gb-6PUMOoTbYjo3RMj+5Zm7WXO#phNhybrz`0-|Pwd@RFPkB(FHbMLCs zreZk$e%A94>WVEZMlKGQr_11hbZPpRpBdqM33?MP%ouB_2PUhPtoWcM&;+rfY!9bo zg(94R0nzNZ1`)`*fxh}T+dVWGR>PETX>~QQmUjhVJt$3rLHGHb4kHTHjw>2F7c15X zO{;z0Fq*p|BsZMaPSV;IneE1AX3lZkwLa5HL**kGU=7g5cPU)~!X@8H35IG6-c6Bo zQmZmywYmAW;E4|*&C7gA#okPBF&trBNWm4U>;){UflJODB&vauA9VIVakR91LW{r7 z5801?aYFLyXg+{23PVhgch+;#{ZaGZ<<1+;g0|dB$z$)ys$1F$UauLdU)ch*B>uww zx85{=_a~o@sGbH#o9A43Xxdu3qfdnWv=W}&HIOEdD`y19} zFT4KsA5I)*lNdtz8?oIp9L$rdV|538Unoue60deqpl)}yh}Q+_U6k-n16m@P#4x8%{26xF=ZelmyTr+%Z94k7 zr_PACmzRr)kt1+eyn@jkWwdyu(Sp`^<0cw%K#_jJl8vQ#d#lnmFds|; z60l7gNSg;!k?&Oy<#R@Hn<>vLXPdMtP$I^rE7>m*ar!$1VxaOn(AKqV{N8fV zh)YKp_}@Ja@+bmRMWU@wcms9Ur7u@60F*uh|6qbyvEe-6Pf2^5Wl?h%6}}RQs@!Vh zmcJ4se5tjG-z-i9dZG@b@x6x%Brrzr3gmEaIoNGr-iArGI8#KtsI>At z+<$uI@qIZaf>!@xxP)4mXZ(y8yPD# zn`#fcV}KoSed~*AgRmvUU~Vbu8(P};SnIM3xDAR8gn!~qDNgtQO9KN6>vRUM(LQf> zpRClUjLKRa?0&wO(ISJLC>AOw&94q{flgeo7ipnew7%qL>yd8%;4&P0KLOn%bd(d{ zG(8w^TVznJHj<+LFk<>lDnt+=y9l#xu+b$ zR4fI00^GXdZ)P|^2G2Bi=Se(Rx(24ds5*BJfMl#f)7QsStlN)ckN{?+GfvJ4anwo}NFT?_+UHc?-durfbkBmq_ zrqQVcMy@6F0Yjcz`@Paba)1mX9&JenJ#Ik(5L8ES5@S={hg4e1chn+YAG{KsF$~9A ze~(BKl`8VvR^I}?B^A(D&Kc(PI_aZE9h3O}${rYg_aX_~v7^Ua=Fr~#+!fTvr({9I zy_6H5CTv_^IaAWVQu*!_>$YomvCPuA#bSQmcNM@jUT-*jTJiWivSJ&Y#2*Wao1{T~ z^!Y+^LCLA5k+laFWMv_|5QKy;e}1F(gkb7-KPvb`WiZ9+*5uKTd)p-la%i|NO%i|o z`>%YHtcN#YsCNbu_HX&-ruz2r7~WmyMhSCPAYCb9a%_6CYTr~ zadF?#Vw73G37;l)b$ZHS;hJhvqn@l$Nn4!~V+S1?&c~jeDUA+@q?E(BGGu3+O}1y| z&PrZ9t&{zTm^XfHk6S-m`9WQqi2bT0XiL*|Ti}EZ1Ur(&-d^7>e}t_s2v0;&U>Xkj zpugU=G*uRgz4}F;>)`pY)C9J=?=beG(8p~R<#$xsZ{2ny$46YWaWiK!0V@*o1EddR z8R;$O^eez-s9DtoMsV`vK8Jo?G^0Ndd#1+~e=JThraU6x^7GQJ1GZ!w*%cS`<~Z(7 zOGatGR6IItIoz!~a@MZ-jI>r~E8@Kw*k;boG2yqu?YCqZX@^ocOEQWRk-ffZOtNton?10)Gx(<_SSq*2(FWkH}yr0sGNZ+NJxS( z9+@CjS^)PmeRuO>H<<)v^1ie6^l`|3egoz|h@&x2>x$DgG4n@&ne)3_1FcbQ-G2+B zK_76Vmw_czjf~OX>a94k9t$0@Z<}-nnvxt#FpD?tf3WwKaaFJBzc7M;1-1xC7=V%@ zB@Lni0)isV0+ep0VUe;0MHHk%x-Tq&VObP&p9v8yxAZ5 zz+USY_kG=0d}BY=yEz;HH{`ozG#MIKx6}uBR)aM!a6Bn8P%+u`3N&N?y}^OzCiV-d zHk6Sd6;uoL5!IvQ#nNO0U`s!K&M#xE&z|OFGNcs~+WJLq^t((vY8z)U72~;;($1Au z#jkoOE|9!K)1GT0-+Hz)tLifNe)y46@!cKkD_JKdUmb}aA^cd89b5Ds^ug-K6aDHG z`dLuLPgEh(>KoF?l!Ih1v+*N2`R(|?#~>Pagi(G#NM?ca@lxqA8{8WP@rLtC>OU0w zTC8Wg`^Zew@gI=Qyv>I(j{JLpWr?Dd_Yb+F_Lrw&7uSF6v%K;$meXIc84T|)cd^fr zwlC>_UHm~`Y2-cltUTF{f9Y$j2$QhHY>X#$(5!jk<%uB>_%YJJw<%!zp-!|*YViCK z_*aex_48o7^4~}B^{v?0Z~W#riV(Itcx*jLB)-)4;ChE2pK@x;B@#|-Urz8jR`uIZ z`p4Ur)cG^KF(fy)W!Ilq=f&oBU~`35imm3)tn@ho-yE6x?^Y*8GmpErIS}4Gz6>>eCl>R4W_YPm%_Na_Ajqdv3)~qBFBzNdWlq zq{6}879Va?zy68P)aef~)d2rgwdvoL^l1xX2_GktZ~HXObyf#eUm7iCZq%RqCRIJ_ zj%vOU-THu~`RAs*OF9mZr1Jz!+%!FoHB?*tnuL`yX*uMa?O=LGSP0C$b{q~f`s~lm zM`fI8x-eAs^@Q;GOX8G;-dyk8)axY#-SRomMsv#K>{qhg-U;n{3at7X4p^0>s+B03 znfEkz^Au{oLiwoC%yz8aHfW$ooD6>5MMA+H(3~h>wRDfHe_Zw*B^#KYWOcN$-OL-3 zXi~4t(`&Dc1TEhOua0fc%FTJ~@4Nu{v=KF|mr;9?Y!t1pc#gqMS;PzeVMOts9#R8u zSPM!8vHAnwyB{{-(yI8! zcgh6a2F3g#W(TU3--nzmt>7BPo@K0Y!MvZJ-WdO`_7iEllY)}%F*x{e*ZuX&*;=uY2wWcZJ(bJmW#T}l=MOEr>9 zKN`rG7ZAExlVZef}bCq)PtBiug>%_>rtm&Y7f>na)>=T#M_j6UFzB{4CC^7 zqIWh=W{Gn5HT9s}$-|_%!!=%ky2Z0{wtYV_Qr^3r<&H&`CPA0QsL!OS|4m*<<&YCE zVDf3gv+JJXHd9+Z-hJ$l-!Yk2KEk^-6(^5XUC#mGGWOAn63IEu$5^f=^e;vcs+I#_ z8N!iabH$TZN-ZmAr}sueES|3dMb)WUd?Iw4=` z{qsAJlYag@Ftyf))Y6&X9?tEF$n3UMebE#DLYu$wH7ObHJ7rGCpqS`|2>Oygi6m~! zXF|A)g9a?uQTeYn&ly`^7V(ZUK_+OMX9c|#8WiK`T z=*;#Md3)go?k)wF5wosL!5|-JFOXt%fbg>1>RV-Fz||yj+^OL!&7aG<)RojdrrOEC zi|3E~?JXGb#X6Get_av-k`;9gxHvCX zvd;W@wSWI#EYgwdULSh#Cm{ImTr?e9{pJL5O}K`=0tL^hBk4UKQWJQ!v=AI$yj! zZ)GZln2r7OozxBjjdyH*yP-=rs}g*KZa`!6L!6IL*k!Kl-k`yQaC?1~5bkh_ZRG4h z-5P5zAl%`%m`cgNKY_pg*O&r4pXHf|$>M{O3A0}z+~J?x&(HkhQ~F;EKty`rjVfpFg^pTt>LVg(M#jO1c!$Kz2gLkbOGn_x|s%;$MCVt=bWwaX-6X#X4c|LTbR@3Hk)8Ora)_8JzyCx+nzn{$ac6IPVYy3*e;m9N{edK|{+@5HIYz+DI2%y~=H z{R?Kp<)}U2GT3w=mZ=eZ)*RWC`G1z7|2fN{O%!>5bYv4Iy$Prij4tRTu(N$-5m?LZ z(3u~Aj%%W@r5Iid4Cby&1(hQ0p}=ZVe&!DPxxZR_|M(PXXAfIQp{#wBm;sLAO?lwIcPi0uun-Ll+x49>+_3c;W?M$O1&s;PKX6j(l3f)XR@U1T766 ziQwxeh#dzxu{QHVn1hsy@QPpt178FvS@B}4@(#ummo}n1%g2KV7|!JBlJ;b^+0emYFtqhl(Cmq{3tUn1QcP&u!#^PjXBoVLd z;r+dWC#g>)AKw1)$LpPWhar_hIeHm~s`Ob*LDV_nrMvu2v76pP!$|)VA>84pFmzFy z!+pBV+}ms{bW%Y9@CEngqR0>QIDi9?db;*BbQzgDfN+lU;+h%*j>>puHK~-uh`+4R zhj6pfstGC(6i5bR-$v|APX@zmcX}X*oz;WDC82#Rf!#xXtd*qU5VP`vY;G}@dwQVg zE#j=L$qd2~2%EqDwD_gVE5pueAfl=3bN)kJBd1yWU6@j^5*|DET0__U?N-QcV+21> z7awY|w)N@b6vZ?x6riG{Gh4Q_gdBbXYChPL+W9N%5F*ozZW+u5;M0_eP5HLdx+I{y zw9~aOA8?P0OdPhji`Rls8+nGU;v7M;1839_Bq&aOiL(|bnwugs zoynAqvtQ>2OA^cm`F`mVY=-Jb8lq=6!3Xx&^el9boX;erGqp(gbrQH)1Gm;;v)g3W zkLVPYYGGzXmsS*__&?+dB;iws&l$P~;HFkahsv_y^Nb(j`J5d)d%)%HfGqd8;IwLf zF2i;@`b^+;MKu(OJbEOS5__(-qHhQ>Tx!=5kKTtCVBb;{<-FlqPn9hYtNe34REQvBhv z9mmx+@kH^cAY3mZ_gA{N#r#Y5B6!cnEX47Q#q*j$^7A$yKLEdqIAoS}`RId4IIN&}OS-=t?#uOfXlw?pZh0LxQsf_T8t|w6`80|RK`_X zTXPHIy-Hw3@ilzx%fZf&Q1TG=0jdLFsKnKpgyfB1OBPD6nqu;^-bPaJXncOsN8=kO zIBRf{KMavn+$NuGG0rGABIAMR;eZ>UITC-o{n$ufiETz_s>;#HRr5`5fW2r0 z2(EK>$0+}{3;lPM@!yYr-&gp&+^#j`h&lMJsVXqZ{rtzn+;}d_JeVDnT(nla0 zwY?BbL_$$4mb7&V)KR2AZj_?H#EBRbl%DcqVf76JWhjRl@uM=0#vd5A$7i7?i5ELF zG-bg_gwirx`M7ac@k8p(q#9~elp+8a`U+pf4cJdoEq}{5<)e%bxWFBP z%KcKsJiv*TtxQ`(y!feR*kzw(u*90=rQH?|EdpWMJoyT~orjMv03p9#OI1sHkEpqv zxTm1-U5JadHpgk(X0wG$3A4FJFOme~q^jbnH^BPz18ZRUSOV)?XDboZD>Akgd;+pC zaVJBK%0YeTbWaEflh5BpbB3{Ndq~?klu{8jVwFAV!*4GUBInz{mEVB6gIu~Ct!b5zh<2F0MSxPL)-H2!rZ!HGmxpzkm}55 zIzK(?a148MQPN$Vv-e)d;%T1;YcpN$>gl#rRQ)r9_ueKv|M8`YFXH(jZaykD zSMeSnmo3AhF+V9Fvm&PHRx`biKx9BGSSD;IcKTJ)oy;Muu!BJwt$V0)o<#zuVS6vD zgVBH;$ZvT?;LsNs0cav^Z}|BfvvTWj=fl}k%Ah#%=<5sXSN(%9q~UI{Y!PM)WQsCR z8jKP|<7z*MQ{UKJnY4ZT5=?IRYj^Bd=61XSQ^p|sI@!&B;7&FdKJOJ*{kh((ThBIb z@`Bt@0vvXalPz?4r{n$a66iyD@YFNJVLWhJ@vgg0Zc?bW5BNt={MyZhE=R7Dv`%9d z4)Jb|`Rjt7%`Kx&iBQ1J`SPUUQ8#S>0ON6v9t-7>crhzMPMu6d0e5~BVaY#|6(FCJ zWq?f+hp}B4lF>k~DMXwj`8jRrqIp9tKNaMzC34rJ!AZUj?5q+Ti4g0GdMuS)GzBc#7hM? z+muXQt{N!c{8M`PKV}29rIq^xczFB!Hr69tfAgK68$4Qip<};t$z1m+ofM9lB`90b zfX?{Uos`FQs<}onBv{o)25r>$=)Pn~FvdFPU%1jBE15u|;>GMzw7(+AYuVE16j@>& zJZU35xn`k1@sR_Ajtu5)a-95#FG(-?G6`%#>R#7S%k(^I{>UW-zC@*RgAm{A&j)K# z*RPO$R!pPERA5+)w3hB(C4Wc1;Ap&`my5ItB@#QGP zA70dsi-+u=;!JqwGONd`$C>Wvnmrx&I^SMu&rx#Q`ODp8P=jLf10-ZagzeB%AGTU= zcFQgWrG`;ifzyneApC~NMP)l-^5vYNeAAiA{Bg^^{Tey9iNPo4;+v-ZR>^W%&p5+% z51K(UpStb~1Sc3%Uy~K&Qq+hN)`Y=Y!MUU~H+E#=|NWi0AE?qLT<`JD=t=WO&c9$y zXL802rzS2$vaVtu@;<^>%j zKnP>kgnzieoF;@-Z= z@=1Rz58g@XPD@b9nXqVNuPIE`Dt1__0SEaSxl=7NGGP+n60WnkJRVzGkowxY37oDReYgNK!K|`2Fw#=7+0R!a<*DKXLUJ%{G(}Q@ zMG+;Iu4&leIzTY6SgazJ?8_b>b0r{^$a5~egFUaGcGTgx9Jj^Q??mRhOGN z=vLB^j&?mdX4tmet$zHc5cK7gt&i;a%-~q35^MQ!F9W|BE zldr2^EQfaoEyoAW3cKpp(jK?eETAsJ|N8KtYY%7c_O$3y)E&H*z1{7fa|g(=fBi}O zDF_EuZtb@Pybk(J%Lrh`P`z@1=)vgX^@BdZ+EM%>s;DogRPA?&zy3JrfOsw(`K}G! zb5n2Ggk-hfs;D`3tLWy~xSNn1G<=B@-lB-|>mn`~UaPcJYq+3d8Zd*Smvy4=vN(3o zTRx)(VU{yp_P(1sUdy|+nWU_7C#F5H{l16ZzU-<$d`abi5R6(=H}CIBI%`Yw1%iuK zvu*O(#&%331LU`+o}yU*_%ai94Rb)zd<#Y7no2p2jS=_gh@XtM}b-J zL-4erO^S5*{L#8w#iH|(_^X-C zNoW%Xfo~^fYbF-+;s)Oz-Eb9#?^ZYVHz#RsD0~hmBLdW;Wy)dsyq-u}2sG#+0N zi7|6Xk_u)GggT%fsw}18AQrVmTLm?9C?=rM)p%|eDNn>6yF4bT_=qWWa_)I)X*W!R1~eM z-~i8{IIG$2CRXdY-Zr9hMSRY9vUx^bb3&-l@ENYc?fH`Nm&aM3tJuIWsihtmOor}; zpqgs3ySK2v$M!_r{S^rJJR^v|L#QX}V?0*-0rqH+!3UDZ?X#2ko_y9b=T=k;&x^mK zpd+TT^1Vh!S2tGYKR#$XY<(}ruqPd5n)&FuKjA?tJ{$rVZ7(cTAqh}?{!n$?@RHf* zZxqklBTzxi0LkMN}d6;=f_44eItE(c6aUSbC=TLd2ij#-O? zLM!gj?XZWp_w0DTt@g*3{d5Gbj%c(vMALZa+f?y6XTHPTjT7v-WKnzBV#JNOU*j1O z=_P|%R*h2SNa(g}31ZtO$NXi3V1m^*Vn-i4qN3*jY))Z(Zve z`a@kXLTIs)Ky-m!A6g3}uA4xH9*6!_!FKHohzp=3++8`R^EzA;%`0}hDf*;WCP5P8 z_?pR>CJ$HV2vwO)KYuGkNRteS;&hx>e3guIJGBL;okT_sJ&5uom5~`{h=qZ(66;A2 zOv1jIggPylMw;#JaOnO3pTi)^;ghX7?}#hGoOQ=+()8D}vbmV9NVi-DIuZ|1J9)!? zfB{}D8@>uvl;5KQ$1Vb&YDpG}$^0M|lRPs)hp?v~>WdY!O+Jfq`3T~aPie(ac5CQ^ zIIRPJ8grFHGR!mFgVwfoa9*h z4b8=z z+hba-ZgaaFVXydTf1ouk{%ehY?jr=(U(2xsUN$e9Powx~vs5Kbv%)M53h|)Lp9?Z4 zZ^ViF(l@LJ0Te4I?_5MQ zU_e`k%q5 zz#a;jG4k(;ayA5`4Juy|X-xSlX1!Fil3>L4-IKSvn5I>&cYhse%7f}|ICAt?waNkw zFIUXi5GCA8-=Fgq6GYT#&>{>y#FH;Y)m2iHD`W(Du$y#Y9wd}{KH{E^Gy3>i?GN}$ z19159lAhy_<3Q}Z2F1bz;zRlr*dNywm(h|&nEW*L=T(IHVKiXNshB=_&4m)i*ag_8 z+LND_&~)v_DOI=IICbmKA*$$D_8Q^zPMCF4&TmbZ;yj*!3GF#iZPojKl4t+&FruwM zaLC^Sn^~jp9vNwq?-|mEgGZ0Ro>M~HpHw_SaCk~i8CbIq*&9Ekotu<}G{l5WC|wzC z5Gv$UOywo%S?YyoR)cW1drVB06{>h@Zhq&NK@&)twoO-i7R{oKh&@PNnQj>MJC8)R@(Jh1?QO(L<| zyXzJEfxtGytc?;bem$0t=t+$Yp%?OIHwByGZ=Yhj$`3PZ9*6Y)l*W{da*D$uhBxag z-3Z)=oIwTDa*ed5dCTo_+tYFoSJN#3+VH)$AL-8$AcvHS5^^&6ppgLwLQAtgF#h{< z4YPp^&kLvLIM^1v&TF8R%_tZil~Wa5Jj@hEPBIp{-*3De8d@Ab`DyaO0r!TElH&u1 z-6=N)QBbuYO9gLTn-F9HJeX{bFwJj}N+Z5lP#&JHQO0*UDaM^RThswv~(XZe+>Qk_SF!iCz1m}eg zv#W}-68W^vQQlAo$pB+JIQEcQ7@R?85Bn8a9O&K8wE%ED{)KhU@#{QLV@8L8&5DfG z?F8jIn#dUh;sJ<|ej_nYQWB4nb`4!l-)oBPLoX0cuUdTd19&qn&9lBw`j7iaAp@xa z2wR9T1|}8Or1wR3FlbTH?!gX~l?1S&`c3&JI2j=J*Ie$AYKAI@5mXN|YvQYTDdOV< zR|Fu-M+lrA?-kt5xXlI(Rv^w8#E{xrgA}kw_in}hwlGAGw`-k0syIjPqrbHU^x;K< zqeBB=&P)nkAkhRcDszsjJ}ynDzKx~Paq!X3oaEDM3>i^Yw=XrUaNgDFNR&a!n>ydi zVm}qG()?wx{9Qe|?LMR}`6=Ii{N70c-d+M9q7P8?KKQJZ!P+1G5Dqr1uOfw9T$Lcj z+!9PN`i*;X^kvo3k5eee9-J0yUpr>{=6Ri|ehVlNo930H@rS2X%H(Hto4 zV?-AO`c7BoJo|PGGb;mkl>oF@oHD1KK15W}lLAn!c_Jp{L1nOImO&*7vkClWHQ1-r zyb!ywi)z634hDc!|E*nrRXiV3!vw)o5thxo`Fo%hdCe!(H98HG-!~59)dn!sFqmAE z1wjxi6bWbh<#bNNn|`G3i8ZN)FptY8rlaDP_UvU;FU)M}5r*0&tNP7!DRqj*KExyy zBpz;Wx5t!2M)i?zHE7QRK9cM(Rwq7*VUYMh^;0P;kPj%7ZkD)O#1T{X9G9j@Vg?_2 zKgfJNK7*dMWwzbYPg+?5NzM>)0txkH^v5S4iV9Zpwq zo(-r2-Vl{{(|fM0^=MJJpee4$9pZ5A55|jxQP*42 z-EmDZ!4@cpnF7cld(rv`W*@|!F+vV9aB9*~Fi&{sbsRH_o{Sz3%QXXz?-!v*uW*H3b~-7*)15-2?DW)ufAiRt)j2WUAP;vXzRAoHVv0m0qDEm z=I87cGUbe5n%y)VEbbz@ksL>qSNJsXr|F$tZu22#q(X)|U7yU{si`OW`rv{jLxfHX zZ(gy=qeRgM#TC$-=$%D@+!dz_8-h>RjWBe5^yUoR+;f$^tf@>@ia!zrd8Zey=V`IH z2>vPf^w%q_cldXX^l&qun3}MC$mVB4PBSBA#%sreR4dR~bla6c8l|jyW&{Z@?9Lsu z^E~##ZQ#9ZT@SR^mWD|^f9u)&iyZFnNTJ`OPD11)a|^+U7+#|Oc96UWNr>##X$wZ> z;l%gZcfEb!8*@TNc_bD8Cw=%kCk6gb`tal2aulB8><67SQj^tUH_kbX)Qt+$k{;xg zLk>c%VPu_GvlaaR97>3o)g)a5#WXSEx7X+-C)3xZppmk?F+Q<)koGUK2+v4rwL}Uo zPc6n=Av{*=v~JrHC?@qkM@QVE0!7OlPm>Pzax3>3qO17LnM2s3QZ-l-8a}eFQF1m0 z`b)zHLvXI+kWUJEXy<1^_&VYsB?Gd^=7#YJr5E^FREHRAsB9h6KN`RU>XR4I60=(I z6FAq5Jv-{p->^AIU5YH|OqZh}l@aX;4H9`XU6w3%;-V`6gWa(mN za=-x$_#f3EWaSqYHoxr#|8sP36#?yEfw~Kg2MPGU|tK-zT9!A@JlB}l(-{OlE8o+Ftfp=z%NLjx z^Q=KFE|rAl@87DSaw#p{oKTzA2=@B(Ne1CzNj^+>BsI(rR`f|dhhpae%h|8tieW;g zVRL&<1!;=u1AT}CY6o}d)iMWMrahODI=%My_cF#D_KBtHeK{|+BeqsK=tZVuwL(Kb zo*tnX^X5Q{0uSYUbi{{tc@UD@ni1G^80$uj=0(1~P~G$%~Ss3UCqT1a`1w=!L_M z7zM-JkY(LwX6%Bzv?dy>Fx_61)6|oJnJitk@Z!_?-MRABBvc=j_iSpXwi|dNc;x44 z$n{?e+;X&ATxsY;~=h@u~%=-yVEyq{`Nn<@jiyBGaG|_ z&_A3di*V;T==FK-=itCZM4{ z*D1Ns2-K(X%}M>b*jL9BB-&zc8F#Ntqqx*&#}h_Qlk~R*{{aqGL8+=mkA*g#7XhmO znwpcvvr+qLim^Y4w;}KUdLcUEbh!|sK>{12c6Mdb^Y}AnYz6?8@=g7Vt<%qz$s*{$ z<)0se?_9m@2#~|&sQuC_E1GB-v+|-)tF86Xg^PD}ex220mat+rcZhx8bTGKqh;^97 z>o+l>iK2))kEXv=%{eEJ)v?5hvn&fAjX`zN=E<;plq|n}m zwqgHR;<=2;RZr zh2dfdP+V+BW)B;msXQxp6EPMUdkwETpmCveH$T^Gkn#F;RH8Te%MyQex$@FW@DP@v zGh7GJhV?9iG}v2(&%wlEs%{_p!?9KM(5iX3)cnafnAOIBJGA`Qd@h&09j(Cz)|3x1*2%vCsUi=Lwhp~!8BD|MRynChneRZ+%gzJ@lpaw*Q#7+i z?sdJgMm<1jaP&II-6!EF31!x7aT(Gs_ib6m>W>rkr0+7`>>M4+GgeYG6Y9wi3{DqK zznrBFyUiNzF^jDVklJ_F6C?Ot-l(*lpb6CR==V~|uh6%d9^k!sOOR@QA*o^W*Rs*M z<7#uCzE%0#u~q_~Hx}&~UxjwZ{Aq^__hyfe1;~HVj7mr%rhZqu*J>3}2sFUHV%?O% zAwxQ;dO|XFTDm6Au=1Us$2XtWyf0gm=J95Tu<|n*DpgTTSH7(2v=)fUun1&~VV`p0 zToYC=A8-_0e}zOtTrc8oF_kFRfIwrR;dWKic`0GAds_eF<>F?<`tWlN{_l4%eg5_R z_a$o}eSCMtnl~-dj_4#c^OmOS+CctU`_HECQlFJ_P^_~rgm|@W|z7*ha?p4gdgj1(c0}pYfRVmxj?#s4+drl}mGk{e=^+6D$ z?o6;B0qJd8E9R^Vw;Uci>_w5ULrbyVhu=8>!5u#c&6o$QIU}@tv<%4l6A;Lf&I-7( z(ZT@n^a`!K(Eg64idny@@cyd5!M0S7LW>=M0wR`yh!SSJ8@k){5j}7Jc*yaGa_0%j z+vq02@paC@bLcGm|-sL#v}O>v)^wSVPox(}`U-pOOgnBE|K zB#mb{0M6UKt_LCck^s}5b9~w{@kx2<4*!06(eS?M{B~F~|AGKoGw1mJ-e%nZ!`e{N zwC|T!33tAE?FU_x&Bf*p2r%^(Tfa5sId?<#tB`&hjB(M%dYwg>!4ktLMZFBp-w{g1)6(EP~$LBjoCnDAFH}kZ391CBultc@+I}Drk zUxQimi@saSOAIe3Mf#gxnBFqghUPHWr%64S#i06fCTU7OVLil73U;1PSNNnQ>iiP!W@O>APswCC^JMkENk$QrMMj0|KVyxs z`}rn0tk^*sDhs#Na}e=0b4vAcj`igEy+XI`znAN^0;^w|S?Xc}v6AoXEc59-t>MyHw70k5*1h zlxhv{c?-?HSX*@p5So;fZQm}*k64Y6UFH42FErRpKEeByz2?$clUst5R8P1$jG_fu z4An#ht&b==U9kKfs^4$?qo-@rPS3flGhHebm!vw&JI&AGbRj;@z#x3xSB!Ltr$GRA zv+r|A1k)vf}QQsQ-L z;5(2D&);}~4F=eSB)BnXNKZR><<+*nO)t|AOGz0l1v>VjVD9a!DY=% zfHMCXFUhW{cdsCH$g=(NeIh}Dn|6KY2j7=>I+%QwXuGU{Y8P3A)THR%N~?MDnniVB z;mf%9Lj3|`l%^UQi6_c6KxfzrkV=P-V@be;+8|uUlh3eghBcTjgU#;hH}^6!jS!ZT zLZ{Fh0XQnYkTtcWsb&6NS`K(#A}^19HRf*}I<97Qtg^L&o8JJF5hg3x@KlO5mN@*i?eAD1gjGQR0UPvSAt8{?F#YGA){ z)#o*7+mH6y;}`zGk662!nUe9qIX$jQSV_|lI3x;pPkSdlf6fn^y03 zdGhuz2Vx?H)6lzX+#xm--sGbB``*>MMyDQjF$TKl4=jpP2Lda%`+QBrXH()1=S5)& z+rYI;3^HNYfx48Z;_i9k1I*H@&PzEoV-%1a1gyxNZ$v&Sv?$vzFvM`%*~D#bE%lbG z2ODh56TIpluq?pn-t0179sAUVb5E!BN7(wbRfi4quk{kvX?ubnSCJNQi5`-&DN2WN z_fgWN^`XY+H_8?)-8kU zwJ_{{b{TU(9WnMF)i1)V*T>l{ZmutSLnjgA4Vysr2;9<3WesY&s3y0ixb#=e@}JQ$LA7x z+67DAJV22p%T;8qyPK-x$?Ek}ZoN{8^Y}rg(Pra|E=aSo+ksg%*v!?%0P z&)`J9+3J479h?DYj~9Y5FYl4pZ-^ERp38FD)ud&Voh{X)Cc41*q*k{M=Hhn_Jr))H z7;1pmBGO@IT}Ja&@ay!>2!v#AfHd`B3G^!RFWErrrvTm|{+8H^mL~TBk zk+g1trDD+VBI=kT8z-UcJSm8IlmEg>O{x)8PJeU&(+_*sNpU=qjN}&ERrA6h-$#67 z50!4YDXq^7zO<+yPSk4(m&(vUyF2lOu6)*9_^wq_b&Opvw&mT|{tW7SJ&AAfwto`)A{Hy^L(re5Ufp?f{1z0NN-_|E8Lm>JJ6UIorrvT><%?GM^e^Nv*+qA^6Jtm(bgqBWtB&mnsBtqiFW|D1vi#9Ik zJKsnn>?&UJ!EC5BlSmu0OVyRT{0 z6t-4MrH4Uj3fqY-ti6@?eLF;>zP#|Re^jFsk6K9+UDOss4t5mUMF1ul1KQ$;Z|1B) z2O~(W11GX1>}m1)Ou?m%vQwL7)P|U`bpC~a z{XJXr1<~>P2$pJ(lLggx49)Rj_;^I)j$wFu@HCe!da&aMTlwHlnqkr!$$JR`Rnl$S zi>b$DT0~WG?NVs;Auk;5^NhZE{0=4`sV(msnu+)>lr53qvBHxTu{ED(jY=lHn9?(` z8BmN1cRIHYzbB$83HI7s_>SdK46iS+>e2e{J$YyHbJe#kLSY0!AaMSD|ck+6~3S< z{v2a(7Ta>0^WMU6Ski)xI-5(ATCOj^Lk(>{8)i=PE=(uOyYp=|#ypDq4&pnfJDSzb zt;WwW#$Tu$ow?h>Pf?oClR4{z!nQu!N;S+S=WBHOl!%!!jvVGR$}JH8=~Twrwys<3 zgk4*_!tu<*d4#KZ^-h`~s-!a?h3+64z`O-13S`uDtr+?WcPs83wQx*)6v!r@XQbd? zA8DPqyMU#WN_Bgq{;+s1n?lri+XjmtuA~`3Ha2Wmy(U^7-ewv?#%abL5=NwuIrQp0 z)8S1%OWWCQ(NO9JNFB}xglD0|YGTI|G2f!q^}#FPY^sY4R#h}~^O{{bfejq^b}01>G-&gN8Pdcf;}hImT6F6 zK2e`56T9vt9O`*Rv}mZIi45nQ6~R+Q9>ce9KOPv}tTmc3TeP2VHh2cU1xnILEATEP z9Pi&-^3SIlDCXm;=AjEEHgXP(Z_MYJFWkjxVIT=UYRXZ(2qhDtwRT=bEI1TCY~h{W zN=%u)cJ@-hGx^0K{Ua|@lu5PAFv^*%B+VRe;F=dCQ?ax1sif#h9Pi(dsY-q1r zatsy^(tOT&F?wGqpflQeH(DAK)b=E)*;;fHz(uj&n2~|D{bN_$_o0Tw7TO*Ci}YHY zR)T9o)AMSwLQ8i)Nt5lMN7D-OPXvlbDTM@c#aI;FgtShJ2EZ#;eKE=WL=oxASy!GU z9;d&^(Cd-^b|=?zvN_(ag}Fc9^lYlW$E7u;(g+pP2Iq_m%T@6W3jYH?bE@ zX^P@o{@Ah3GaX^rzmyj_j8iMYrL-0U*mqe;CJdl&qD2eZ_@OTyU52+B#r>&EqB5|; zGSIhEyzT;SIo!khD^Lu+ngMi3YIN`Z;1o_f?`&Rng?SkfQxMz9alu8A46X2A##wK! ziSTPLbls#0zZR-B(z;(VxQ|&IF4!u+7Iq6Qyc!iXB_#da$g0Ce#YeJZri;e&JySQD ze7{Jdg?bR{Fz>^El837*CH;EHeJI|BOY$BEH%GW@-xPaKI9n77NJYZ&nQ>wHHgqn1CjISy2hmTd=ly zZ9BYc?wc{5o)rQpQn5}V@A4;YSc9wB@^sW4jO9t`-11@CH(kB0_e1#aN8G2B+Im^T zW?Qp3RI=c)EK?9D32h$BDdBz5NOGy6wE9117fxrQ4;2T?(lAa*(bz^g8_o6lrVwF^ zZ$*UFZp#&II`BU+b67bVIQuf6cgJ+&kbsO?@r~=0H)+Sks8GYJb7wjwnOp}9;&cC| z1u*%Rt-Mv?&W#)_2}h(;rl#PYq(41fDBFe@#^(6S1Rb%QG0Le8mMD?Z#~8y^UHMc? zFEcq!im=NSez{EvRcy}NZ7!!NoHEIbF?&+?_eH}u{e%}=how`AeWDzfCFR7aAT3;t zi7Fp?o4>ibRWf{(tv*&M1Zi(09Z|oGkL0Wz8~de1lf^m?=|`8>grX09ZW|k>ln1cu zw&tq+9~eczI2P_sJ@FBBM{(lm_H(pc+r&nMo14ujkwl;YG#2S(<6sb8JmL~RPHeDU z)wh++k@l$~#pd0*1 zzui^9F!JrZB6a`4u%esi?XZkEzGO#5Jaa~hV>8cP)YZ1)?phi0W& zkpni{*k6)Hx)VP>sJl1~Y8A38JQ^)FGImQpry!BQ{Se|wBFpy zl?l&pPj^CxpU0}A!6L`zwnBw-g|n0U<@*#NF|AwcG+J+E7WkgM%$Rdb#YJD3Iz^|z zsFZEJ>O*-)Dx++Vs6D*Xw7xT`LoLDqNvFOXYaNm(=iJ{O-v8z+#M7}-SP=vH6Sn80Vu z8#U16S4}qVf63~!_``z#oQGPs;oA#M-yC0=7(r)qx-&e zx>7q`jMKH|u6TIXlZ>Mj@3w9wSOnI;0PY|i3Z#!-Zq2vPh#fVRF54eMvq}L}7G$P#yh-yJf>~Wsgg>rLz%eNN<2lTerm#2>3a1q?E@{Nn!OD+J-~(I+naqqQJ{Kr&4;~KTKBuT`BZhj+4?XI z#qdyCe}b_R<;T$=>m7TJYk|xv%Rvpt67!6Df<)bkh^yK}|3CKLGAzn|bSTBRk1uAxy->28pg?q&c*>Fx%lg<iZlJZ^U!gX{s0Km;rjSUh^} z+vq-y_|^9?l7@Q@3eWNn_-2e>TIbB9A3YE)I!_%5_-(H>?#Z3NI1jr%dU~hP9If~Z zm#BaUUvG*PB2R3~K0_T|ur=LDcQH*kf8Jr_fsdtBqi>;xEC4Kl9CRk|1*mFY@V*@X97 zr$;7*<(76zo8iNrn4>`Gsy`+?UZt)wE1Q~AkZ8iygYwhq^k;L%@3IY<(`}8O_peL$ zHiE&v$Ag`pFFamfkMyxhEVVN5xvhLeFC5Fog=_|l3Ypj2WH^%nS4AkVobX3PVG58k zDugqul~d6fviTtn_Gk8wG$HlLu0VXqT95MvuM?gl1tKqBnZ7JE+3sC~ z$l6T0zmCn9C^S#Ft_JAkwRL&=#+5RAWjm~?Z}&_>CgH893qr0sg}@E<#DGpR{=BC| zdNwsH8Iy`QTBYdMA{BwSj;CMKzVv0V__>&6mWT9x^Z>laEplNXylNJ>74x>Rof{n5 z2r?5avZmacBwbwvsNK$6@VAQ4jKPIQoS{n5MOm`*mdr4v2c~_>JwhmpNz^7Fq%iB& zt;3mk4{vo6Q@6jxpgyhXqX#qvpuUzTnBwf&5zWRZb;ITrTkuS;!CsSrpMDgje9^1; zp_MOW%WX#7>=m6#LcFqt)!VXL(1;Opz4-w-3zuE;kz=GoRE^#8MCnQZSHb~;rg0Rb zv8rvzgXiS%gu5^K%Kht|ndsL3#x%5A0!`Zx5e0o6X_Rd&ZPsFm6b&JBL&bt6@BWxU zw>vLTbKIRF4!Fv~K&Eg1+`16~5qdL#a(~H0apzm3oz2<-^2^F`Pooqdujn#A$zM?? z$kOb`4zcPY{?PB(*Q4+FZvRP!4D^a8whgYs=fZE13!SD=QDN=dC(D33Y z2$=M%p8XjvOPGq`RxI(~3P=lL5;}L5dx@u0Qb680a5SnHa4^eVVUFtnX{a%?9$Q-5 zxT%lW0^IbA9aCKs75cBg$HwRv-f6dIyb&2UV-6~&mWzZGL!e(r;qf^9po9NT!0@^L z4$bkIyBwL}yVrG~c}giWsq-%oI?%l zOXV*Jdvbc$G3CZ4`}c+@JS|2qWs0gkC*GyXp$CnM_#2v?uk=JT>MsvtL1j_YZ z5||YrTPO*9EKzVqOR{Q*^w#i+lXBLHtjQ7ssV=pn%5WDmiggGeKUJOpWdfi2R%$tR z4MyQO>I7fc?{ML4+r4tObogZ_#S^YPu$`^k!HCEqceLr^>a~spFq!gj_&GUvMFLPn zkr@a96<`4@bt>El3Kng_GGsHL^x-1$j*%W(GN%gbvnKLItW4@=%8ee%*(M1%S!ms2?|tIxcE|GH7G$ zCu{I6JILAliwvgX8afdHf-m1~3(& zNaeT&jGc(LyTnl_j`!tyu6kXDm9P?E_19NZJ->~e?VuNNPSU(d@qV11v(Nl!rIxFT zYfrn|F^z>r_wg0NryaSVvl_OlZ!V}P><;wMm?qNM!Fce1n11hzS$-3Iy3UDlm75U_ zp+H07f+95rJhv8SHRP38m{?8|A|_z{klxR(dS4&B^h>%p-Rih;^yMn9>gibgl)dW8 z!Dq12qH8FQ7kaUR{C5W3fV1+&67pJMNnc|(-$cAQm$iZ_AA@1{MnH%Ac!1(U7op(? zFQPlgF!`=9bWjb0f_alP} zY8of1D{!(=agSm}=x8=-i9@TpPye+Gx-CwN9@@~yeXWexJ)v2fou_y>@!@gv-06;M zh$GE4lYNheFNY#N?(d##?hfUAd@vm%dP=o^!e-$rL^SMa_`_PGTQU73JK$H@>5sC1 z5%0u1t*l0Lh&Z&KBzr8GX;#=8DQ9tc#CUkbcxm#!ir#3Y z$az-24NZlkTodTD31h+Rz<8&AODCP;TLpw{n+e$tW0zK2vaFa(?6#9IC4V%9T&- z1D&W8*GDF;s+x@eI_eQxj=5~Me|tWds?wuiqGGY6RT5{dB|)awn%JNvkci5kdLFo) zv+BDYNZip!hN+2}u7;Wea*~m?P%Rux3VSw21+GVArzBI@(-#`WgMyq7P`VMFmQ|HQ znhEw#gMfSEprihAE8s8A`cOpQpVsUJjrhD4baSU|5bFR2BU$?<2Kb+5LSjMe#}0&S_0xXNTXSNJUS6q^ZvUph-wcTzP> zrSskazF(0{s?TeS$xplI$cI9RoT4TW3{Ts;el$l^Kkw?2(>EI{_I9f?Q&m=*WIXJ8 zhYoSK-(kQnmLMH@hU|uOhckf2v2Bz0`8Gf=Ux;>(fRT0VD|Kc@sNF1YeTzHG$##YA#_vzh z?W;OoOwAN~v#R`peN{+NaPL0v*o`6~xy=aS#F$RR#GX*iaWMbXi&}sYbGkg(Y^RJo z+Tkb`%SsTT=3l5jxXyZKFfNcNHFIr+i_G(mdUxG?5C0-y-Ies(0`ZP$Xkg|Mn>XoN0#B{Qu+6T;`O^Sd zsVh51arZyybL^^g>g(!Xh$~9zk9FEq46^W*@yH`~m4VKVuYic;PsID8R^k^JF8C#d9OaJTKC zbR!NZfY!>;8fH#S1IDHwaU>Y4()u#V4_}6lIvX5K0FKOkJ}0WdG4Ly{TJS zhm*2FAz_9U49xJT&*8G%L&IZ1*+WF|qE`6zbC17imv4jjaa4Ua`%(>_D`PQRCa zcR-YJcOsMVjotUAewspvdKe64M$9_B(WtchF;wpGW-jPe4wRX2&z+=3>&Q~QhsUDo97qR)>@M9``3B^KKl&_i6jY3o4PGsojua}s!Qu*(9sJO>7n7cQojdC z8ufVEdLJ*=?$IV@IByPURj4f2!t}{so}iYH%H=Cx`6zD_Vo^M_ezsDro4WtBFU^Hu za^CxF>bD@}eVhp+K}=Lw+J%@BbFhE8aK*-mPvNXC@Z|4|W-CM99pz7smYBRYf&HHR z<*gy#_vhB^`>2hB`4!P{mubWMAD-$z0pn_}D^qD}4jV(7L>!LJebF*(UW_vVyr(y& z+_X4$%PrF$#DmGK3&nKOGIC-K7M}_oe!U%j3g8#IaxM=@&P7Y;Al$ z9DO))vZpoWV6r`W6cTiu3BZJG^Zc_W{08;NTZ1TLZmmdHP0#FJ)GYEi?z3Z<7J1o7 z>3Kd!GoF^y$GmXOv`Flf^P_ZsvdJ5--1kv;in$*x-jFwd$VOu<8{?6vUG|^u-(L} zynMl}l~&Ob4{X;Xn5*DTpl{}rM;&jxUkheG6e8$Ew)6c*U(sHf-72IqLWr|SbLHk! zG^=l)%lkMdsU$@=-hwyDjj}0%Dt_Ybbn$cF8$b$gKU4RXpmU#JlF+3AcE%9irRz+3 zN7vE)z*1jt%QG%nC==!=o`b$>otYl~O#6^bwjDi{)+7Hfd`FvU44ait-uX-BShfU8 zaPU2eAwpyl^Sz6IE>y;1lmScC4_oW32ji{ga)sci`R;%;Un0IcOR1{_S%g_Erc|zL zE8o7r#csKaMCr=-?@bu`Irbe#aq3sB5mY70)+)Jwa4N;mRUuVCYZO8^tSp!J>UBfT zJ)ql`^3Uo!Zwc+6T=IA`@cS{gv=UGa3ye+>(xF;R){Kxr@@OXmwb!qf**dGi3kzc zZhJCm?V5au8CR$=Rqg~*grOS*rv1yB8|KHV3U?J8E6K{u8Ufcj9RBDdMp!L{6!r>lk23)+{-jm8{0=zu>H2E9=Gh1|J*-;waj|rzqVJ zH7J9giEE5%o=VoJ9`aFmG#JVXv*+ls;pLNCy0U?(d#8>~R#r(e>FSPGM-)3zWlXYO zs&9vl@zf3pVc`H$mQP3Bbp~p*R4b0{tW`0Cx*?uovg(=Hy%Ptlk#F~eg$IHCzkZ5^ z(&jffZF~_YEQaNL57HVBf+ad)hJL{+B>Db8ZZkOIS$qoak^j@H9QDe{oXvjdSr{hH zJr2-4*m2B%`!l>^VC4{iies8lIJ(DWOJXwpeR7@0&D{}gxzfe??<>U{C#IKNK2Ce? zrwVRX0}oZ}^&52PFHr?ud0SaNC!`pL(#Bz#6UbQ5?8P^D)slBU@{?S@hSc_!F^(N7 zUF?uu>`)VJesTo*JNIn9;kazSW%RN0+)VWs_kTqHvkU>cqVOR19RtfhK7|9++zkpI zEdA8Blsqr5?=}p=q=;)4qode2%6x3&Xj(+Cc11 zyf~4}81O&|G~ODBgU0qH&}P8XpoNBb15eLAqqRR)3Evwnbg|XVRl=NiGjtgT2q3b9 zDR<0e@IQ~UsaH)mzeo-RN)TR$(vSMCV- zKZ`?yidU)5?d6gy`a;i6M{@%BTz`>K|^@_pCX3$D)!K9cl#Vy$eQP-~apo+ed)SsUu<%^K0?Y z_xsQrW>!brzAUJksJTR*PmUhyf>JR$nF@o zi|Olw;o3mSJPb8O*8?<)7hjKaw0)8wSe#Ty@QE0md(R~w0z<2kPFmIJ0&j4N$i>V! zBKTwg5oy*cP`rzov}zROpb~2R)%=Ub2MjNxNDDl4@{Hb!pi$;dItF}*j$YI@2@MH& zvFKE9HdqE${KMLv{X;+;c7#vRgDGyz4VQ&56(DEMI(rJD9bIjR4=zEhS5kAC8_MDj z=CZ94SdA$l@beF=@oNkGdZqsP2Y8qQM|gi+)nC{5Z~oRMfo1@cr0D*xIKoX}{@D0$ zKJ3?0{Nqn;AETet{MR1< zYfs9*xzNA=f}3P$nzw|TBRUNJ*U$NP{|T1=+~0rL-`v&zF8_ag$p813-}@Eu+`*PV z!ZytRw_kw;GVkyj+7)LvGmH2qw}ZKR7cR}!1paXQA6Q8{|KRz(-~xg|lhKu% zO?Kp42d2NhqIjE||AUa%c}K)+`QEpk{(l#N{v}dfn)^EY8@uh-KZya)x7Xz^SAewu`UJsQvbi^&z-4hre;z-V)~8+a3&`1n_y&)e#u9k1zjEwGtnE z0QHeQck%koV-y%;eFE0p zLaHCzE_E;s=Z%K0kC&l_71k3#KHEN4VgOVxoeY6BOgJ#r!y&dg+%87_TtT z&Zyg#ZBO7yiDXyR0vTNN+{s7g8xO5TTw~3PTRQ|93bZPS)agGuNQF>GfyuOteAfeB zhwqVBn4h-!l?BAyp#|1#GS;5ipZF17kwCUgLN=y${{ zHI|zVN(bNI&3v+|qLM$B-cv{d+ZZpKtW;$YKDMW-`RYe`Ya&GUKWx`I1+YQiK7`;n zU-p*xW6g`P7!?yk0Slqy0_l3+_EG{Q4nvl=ZvEtOnz+ruR}^A8?-M&>^0a2!NDQ8^bJy$3#^8X!?Sv6>$|{3RB&C@98;S1t}`1^k|l>*=nI|hTlq15XpL+zVhCI z+P%D5KJh-&N~JOj_Z=>Wjw@gq#$y?k;g`c>HFZ;gf`{mkbML!^ObD6&{nUi=M^$wi z8?%l2$*!f*x)p})Dm6S(MB~(?bkk7#nEBS=n!Lm4&X}T`MT9i$Kp1hSfS}^NcTPNa zN~jpan@l6n#tsLq4CP_XGX*V?QnDX_HMtmXUMRB<&+z`&6>3@pp)&%{9js1M8uM8g z%Xumy#tj5vqigJDBV6p34`p6m*aTt_HxB?U{rG$BEV|TdxTL(utY-LQlfAne;>wQjVa8 z!_bxn<*RFkaP*_V$w}^gVo@~oH|yBQBI3nJGV_lcOr^vtSW6XAW8#iAUTRe4O`%J| z+%%tebbNx%a?+tD)?scY{ zQc~ag!y_?z%Yc?=PW}BS_W|l|_Wm=I-sFmO+%K^?4doWo*9^oY_Z32E_`1jGB&aL3 zB^!iffj)p~40qEGP)d{rn7c*LYu<+Eq_5t$ocK(3d7{R=Dg!_vssybcWq-|8$t>;F zt|FbofAUQby?4Aca1Qt2=Gk-dT@G(F8Ytm^My9`VDfUWm7Ky*PJST&5P3`Qwlm3FG z=q2UL0Rl!e+RwI`_-f@X8T`$aHxCy(1dXKz-{$J;>$#O=D`{Zg(s$c>z0eucTh#;G zT%v@unGWvgV&B!Sw2A-|#A=@Nx{g=^d}+Y_z;azEh?2Y1p&8fNKDPisCZvNCc`c<| zs0{HBA3ayE#TEc;`j{4Ax&_*%hOPVXw?vyqv#DY#1rzi8gIV%P#=Z|WLL4D90?u!q z6g_T+jYAHm$ICLo?b{A^^K`~@)1b0GPr#rtz)cIYv9^a2Ii$O@%n>@}vi&gO8;pqj z-e-BT2jj^cjK$(!jMStoKNmsZmWWp?9pgLWv!>l&NscuMPBfeD3#sAL?yeh~z(-pJ zIljCQ%wPISpj2o!mdm6v|6Wpbg{l8|_Sz8O{Z`*v5MoqHzE8K%NLS|skId_~p9ZMZ znA{Kz0?k=-N)-n$sqD0|Qlo$yytYqw#leml4rkMLWYR3nFzkxuoT<4;29z&1&?>^y z%V^zK;02D_AN6zwYZneoAXFOo9b`8;Tw@m6lca2q8DkC-!x4243S8jERgK*)+>CewKQcqDtpb7;iVVe?te|p*$f)jY-;vp3j=lO)C0){i z*U(KXAK!(``fhVkJuP`*qv(Cr3A6=zN8DQqsf~Oz;1iVmUg*aTBo_fjW>$z=n{ zltC07Z#TjnM)U1xhR|UOu<|=j+)<00VRW{=(@qIc%F{5=aX5VaYR_o`aBf)#vlPP| z)<(vyA;=9II(&enkOU{`l7{e@Ny#7bZtNjmC-6KZW=~4vGk$_;)2vTN;9(g-#NG@CrUSmR75byP^04?NFVr?QCA&b1)Y}GLc`$K@C zU~CPcvLl*@vT5c<22%3OAd}I9S9B&U9fx<-?VbLjWzm5EF7gcnZN(qAx}8Y|g8 zT%)semdN&(m3@1=Y5rB9eFhCxCgaro$1-NV8>HX0SX1XKEc!)93Xh*MYvkN&_7B0+ znkE;Nrg_z)Uwqe;3Z_YZRjt zNHZ@h>7|0yjoiJ32$f7*dhHMlB>{GI^l_1+1vo=@NNdL#_y%UBXn=X2g50ba*+E4UWNrr@h%&Sgga#1x)Oa~X^ZV9Owd{)be%7{ey{dLTk%(00P1O(yjEtJ zSJTMJJmX6B)?_!Gp0cb}`nCzLvac%}^{QM|NhDm!o;3Bd?a zn?HY+pfG9Q++2zY-o9)W$Bz_rsnOujJ{H>v-3aEtMG&+?G=XY(=i-(1=o?U3Y`adn zB!k`&T-I|Kah&e8k#`<{a_f0GUSXGQihxqRm6oFEN?p!VKN=Xod>H5dtj-h4^UX9 zt@MNl0kyiyspJF4@nRd_q%4)Hs}GRUSxPCX0S#$ilUnoCOOQPFMnR0pp>^FW{3(c2 z9iUNWv}UGs;AGrg>YbL%v0p5>QNak~qR;SqlX%Z&VBtO8H{MB?UtS*Gos4r0{;8vv zU6p8o_s(P^(xCVAL{3@Kh(g>wHEJfUAs6%P&5^$1Yr6Vffwjmym(Xm_h_w%H-lOaw zgmJ1q>nh#biwpGgfz)m>jMp2L5wcDadLG|wnjd*a{&UxOFEI9fcz*mo=;c&KP9&=u z-%KTlj~y&9*B0#}mcw1jX9#{AtE1hB3Sja1xoLR+IMQs;w-7ne0fNNpP)_=k=P7m; znL{MnQ`Kqaep5P6eECXq_Q2Ak2vsZ^iq%VlmJ{Wbk8TzWj>=o6(o$u!YSurkb#(vO zpj&HEqfh*RYU8jkRm>4jHe0CZSAG_H4|j^E@iN!zXnrr%JasnmyVlBSUWxrq_SyKP zj50vchP0wll{XwiX-%EYD2O~ta`GS#|Dk8`i`Qj!h>G`dDi&>_sgBi_vmN5BIC=pR zwdQXxgE?3SyRKd+NE~pcEqm$^vWS_YF-xe>BqfJw#>e&WrH5LV4RQu6BzY{JAJ9;Z zDL}x>hHePP&~J-b8P<|E)#9)I2zN7qPwp=(JW5|queB;7dq`G836uA@}u{aNXG={nI99lE#qRO{y?gZeIJ% z_lQ`&t7@4YQI^VDQi4M_1ESpyHb@C`0F?b|Lv0v)bs%-r2W=G znRx(ytD`u!ZZ?m&gR?aojx#H^@XxbO6mR$5^8`?RP0X?+Jb-MWVBd*`^10|5&7MjH z@HfpTA5I>;0f+_JOkmBTja>PJ&Wnd~4_@i&7wJXToNk4e1t2OEld>1f1fdzOK@^UL zFHEp8sik~0&K`a}9Xcj>0vjo0!)%402Pqt<;G;ejw@G(=2s{P_9F!J5ViAum5>fHV zmMwL|Z9mZH@;OC=snEn4IhD;r)vW4gA$1Ldn$n{rb2gpDztq6CN&Bq3+en%hBgu7V zm>g2vdjuFie5(&a)(%dI(Qv=a(78MK@?}5CtI5i>o5a^xuNM`Tee=EA*^TFccnDdn z?25}2YgIJf_hD^t(9@Hyz@$dX3=(vrI~H5>trC!@t8RG{=>@<*cgfIcSIt5AKA+P` zViJQWkK;YEc!h+PZZ<@o#kKNf7m%Z^GrM~Ymo3YI6t1c|@7E=QV_KxieTRh;GdY?~ zt8Jo!W|Deyf>bCxjSlA}F*h7uD3QHJ!TBO%_Kk1YjaFSo5FZ-qUEJzc?g+q8yL&*Y z!c5rqGag!suwdTf&dMZ{4=%FeX$H7d!q3*=&$hSTAI%9dqd4>x8;+MK`8&LW9Yd8} zJZA)4Rx^_;;}rwAT{^DR96O$__5xuvI){`TI)7rCeFecKoCLhh+qb(M!85w)K=Apq zMZ}@8_;a>H{;Usi$)A{lC?qFRoUtpmcmqPeg)6~{T zF()@ZOPvDdRt*Mx1ge1~sxq6`Tht!$7kq4tlL8#I4o6n|!g}Kz zfG;_+zyR}Cnxazag<8_cz)UtU4nB~*%JMW86Z@QO0b`2%rB&?+3&b-?8=FGGT|S9y zOIuM|eCnIxucfgBdaxS-E>vAe4<%C{W;`S!c2_ty=R-K(tx!9*YZkSkmlUe&_Rr~MD)Z2{h4E??6zG!jGU4%5h4Z#UEpcM5s8TrAX4k%$$YJMF_KCWa@S)*~;H-y1VR zN`I;qeT=U;q%r>LDOxUokexdti% zQP4EHY8CUQO5*YyIw32C%g4$Rk53}#K50i3^F%i(gJL*n-twy*>;38yR<40v z)4%x>mH-MNl@B7h7dPcSQI=55oaNouWVN!O8P}C)>ZxR=49+nA1(go~6M70P`Ye=q z1Y0uNBQl?sNI7sC_A^}u7i*Cn+_y?reuedk^o$48wa=fAmiY6zUrE7cqR4HYMy$rj z9Snu+rhz`C>a?$sA6$FKH-m{dbM_#0@Jj2t+b^|mVwM1p*rtig&$?Nm#1zP?E&64OV^d6#{qqc0t z$K$74S%C#hdJ3-FiicCok#m1{K=VAO?0342U-l{`Hotar2svuS6QGS7JL|HDlwN(a zKrLB~5wi2;ejDK1u<4bzX&Pa*k!FauhdbKqC5g&)0+*Oft-bFJd>gL;f~12$_Z#20 zOMe}37+6edH`UHLe;*AAkF}h%@G@TNaysfMP=q>oo}kuil`*N|xVpVv-1=1;T>GM` zERBNhegeQt&7<|g{t{YeOGD3Nd=#p%9MTKm%Eh^VzGGKzU25;X(zDqG}au4nwa1L*#Y-&Y+U;_?lY5`&39$Wu9xr_lB8w zv`&WD=uAr__p<4}o2q7&RT4ZlgRXnnIf$O@Xu5<5~}*{Bv*<7jQOOJ$OM3 z%?CY?$yo=F^F%v-7)sY~JUC-3gv#SidD8LIer?Kq;^`l^Ay1gvvN(7)CsB8 zW9T{p?;U-ur3~R_;zJwJju>c{hhA?o$N0@kRP%C=yOQ3?#@PrVDtgSMs&eMyPmg#dDtkv$0PS0rVP$I&UXZI(|mb9Ipiv@CA+>G}5>F0Xmc9M@P`w1!0IR16pV(T8D;Fcl3+L(QgSiNeQtH#D{Nv*lv> z;FdLrhH4VG8P^>JwC`o6O#}nFlQ6*t>~ivU5u?34SRp;Co`VWJluU9ueoCx2}jS5P#2P@YRb5+zjZ-?0%=n!A)p5$nHyip{pzse$M6MjBgyZ$M#?A;?l$ia2Xj zV$`V1v@yVL6HF{S=p+L8AQH>;A^#4dAqEnOsc4akek7`S8KjqjAuInVk@^!X2QBg)M+W zB4Vt*`q^O1`;$(-%z&>0~dh$P(x$g9%9o; z!T&oH0UOoGh9a#>O~1|#s+lim05IF2lY6Q+&2fUZoG9|51^us59IP%inRFB;wSv3S zbzwcp;9VLOEt;u|)c3@-DlKSJ;Nu3+$O_P)aC*0PcA~An;nS_HpVKZPkON-DWVQB{ zYrCTMtEG#e$E_Flz~LR_y0po8!i%d@yN8#U#uT`-ssHM?;CKLHD-*~kQAUy>X!YJR6u8#5F>_KRzE4w8a}v=%58?2TerhAM=kk=-zc8{M*VsPczml& z<9sM*)v6b_jrWgC?RbShZ}fTdYwCkmZ5kP`<*|}>}}s7-z6;^L1I1CLore!>dNQTCbLzQcTD;JJ0ru-G&EI!teMswUv* za1zo$0>Q^Xg>C=WoblnZN_N@6Om}14X#K(V+q%{Y)#VrUK)_7h{`(t{m3@)d4%0-t zEbf0D`tn0Q*!vek1@HE4YoNW`9?29jW!+4Oucol;2(xkO3lwnPn9^(yqcdq+%?6ws zlQpB>&H_DY9v%H;!%7UA#vFCPKOSqDYqK@92vxX^s1H4x}+k+X^Oe&!j=Oux_Ka7^kG9r1HE6@RSz!TwaT$__K3WgT-f*HA7nc z(H^6=`N+KxN&P7baKJk`Ne}xQSM8QhLJYgEYs|(4t4fjUlr7C$`#<~BNj>xy4j7Ix zN+dpN!1>iD0w5xZt3SGTH(At5Ur{YVKfei6$+2AwaCIZ{`(p{to(_0K_yiACwdTP?a;#d$z8vF^#?+s`ho>WenN%Zk zYNuoZ7dn*Xns)Nvhp*?(z);djCdoDsw(?zIxkyB)<~7JB&`s5bZne&|z3- z2L_{n7xP;FBYAu{+aJ5o2M2F6<9$TC#!pj)e}~@;uk!-Qu$07WOXEt5al_ATqx4Q2 zQ_t5*NPd;+m!z4{m)#M!0R8Kp|K@Mhm)-;gMI8Xa7yw&tGlA~^(8?dwTy9U$07WI; z-WpFR2mz?dKhF!i410j-U^>3Ib9~m7`1>pLHe|g-32l)#VM9Y$45wt{|9qZ0THpg> zMq<)`8x?g)-3vqMDfNSY9^L-A+W&lC)@Ep6k(PdN@V{)Le|}KEepTctxTxnYW4~>3 z)RYM}o}HOHFW;YA`M>+F=is7VEj~vB{r~Ni|Cb-cfcx&v+*@4v-Cw|a1ulv-+CS*G zf8qb(hY{}X#mKGL4o>(no&bJ%O80L+PG>DIjF4<4nxM1h$G?Ly{`Jmo)?#2%7o|L!_+#|^ z-+k4b8F*2Wh1~C>)vx~goPYnfrZo_U>OIbx3IFE4_sRzEiSyxZzw_^X0S0Cf!02rq zl4l(McNhQ1+m(C`92%>GQuBWu8vprTfG?_7>zYaE{_n=D|9ud@@ls%5{qKYLV@v+0 zt^4zhxLq~~PIz8)uPss7Q;*v7$OfPkgAGSS<9`=rZ_SB57UjDGtn)$TqIyBVx>0ao zEqx{wFtc=CrQ72Ddf}?4uz;9WIOyw3pPpc60L$O4c&`6%fL2P%yVU#u3gIdR3QTsO zxhT7NDO)KenhH3lj95+@0+*5SjSd!i$c&pxt-KkNR{2vOl=lt z<^hN0#QktODMg@P7kPzc-&DL6FW(;I#pC5>5$ItJ@WMrlDEtKY){El=U4i`?1$ofuF+-VaG1mM3qj18?KAx4{Mni_9Td|C;<>y3S|hQ zJYN9d36Z? zFrN6hVW%;Yuacue-|ngJ!RvSwi}u$kgzanTPaYH9*v$GhfxW#0uabN6ByyoEPEIl9 zq`y*T)ZA=h*JX96#;z&(<7};%1W*f$j&E57CQT6_10HL7{33X?3PKYnXdVsxw#3o! zA&W0U6p=g$*2{Uae9ef9fxLwc>z=7?XiyBRx;-CL)qaRdB>LKHy(>(or2a z15b1jIKn{~N9!rN>2r3h}PaZK)BJ-2K(Dt=q%ixyx!k zQU@<3og=?z_~A*O-ZTV(=zb#<#iCZODk!K@Zu*(H2kkUQ*R}FMvHwzc)vTMrtRKOq z8yInKI}bBgCVR|KLh8-L?)BJQ-pTcj1$J`cpm1eDn@K}7sFJeG1{oUqx_fl%=-m%@ z8{nzek_)s)6&Ev8)4y-B{INV5DS`WKLJlRdBPeXr| zMZLU}EDy#noWHOE1+a=0+WdDL|!r5g)Xo-8*>VM#23nJksG?(P1v|T zBi2frdx89h)2aw-w0s75i{-4#v%%j-f1-x?s{;|{A8qnZORWNEAX zX|d<2P=E3ca=B6eLagU!axSBAEaEAM6V!c(QU&-DJNx%}wt-KII!5hxex;LINUlRf z>zp2hpoJS3#A+ao$Lb?R+9r+J)SJS_-`@xLE6;%n)OM~3 zlgSY4u4`&edhQkGAMd9*m@t!_X5yHRQF~=5&-#7R8hGX8WeZ=`=b(c(fQpKNPr~^f zW&MCyfdZ{QF}1)QS6C#A2D`RYQ341gXr(cgv>)k}Cwac-<$@@bUJ0BCPtOre$pa6d zcUJ>YQ_8Ji5b!FHhNs8x1-d1sq32>$p9_E(CFwH)>V=6^^h|oVWugi|bCcaaP|2L@ zo-#0^^HmF|VjtMwn5d9TE)J-)43$FAQvub3twiQXz zwR=zdQRl$ZDNV4Ae`dMV6}JdlVEZ$#6=`#jG`oj}h?(BoErCMWNpEfAXHg(VMVfbY z7b(*Zphyn+6%cXq349VGaZnP94fu?p)$NaH)<%EY7txtUyuI=KUxk3Tr_|gF0b!Z- zoj0V6im_|quEHS zR70An#Z|uEjMz&`VE^F=iBB5)@J|f#7QcetEpy#tUY)M1g5%Rvr|etP+_esc$i8lZ zA708YSnb}}&2@x~@_wb{epi_Xdg;Kfo*w7`>?u%wov8wGa=;Tf`G|z8_M`}NH2V=n z7qz6CXC6P^Tdxcm$>;nEKSQy~K=%ZMf1*~=uo;!LcE(KL zceJkzu+9=ck-h-ye7Kh`zhayKhDS8+6GOWsY?^@Z|^ikqG>Ab)Tg(t)`fi^Ho~M^8yLv>JUJs0 zjIhA`GL(EUdvYy@)1bAzFo|CRI=dzApA&$oG(b$i2xYs+a+UVfh?(Kdz#TweG?T zq`eqbo_|Mo8FS>G0q)a4I@Af3DR!)YIO-XIQnRSLJ##I{-UGx6siX@k&HPHkg=|%z zW&6Nl>L!0Bq^gzms80;&zg?_bKayTQeuD<6(qa6c_Rc&W>b3pj_J~62bW*m6C{bi; z?CTK532nA0m0ih}$zI74M?|uf!Ib?(wqeZ4lCrO3pDEcHWEsZN@BUiqobx>A#h z^_>6knlbk>-}}1m>v~_G%T~_=GxPZG0oCx1A(22&-X`GmNHJC&UAlHME+anjJ93KM z!#q`Z^5S}pA^#$aw9Q3DD>OIETI`ValhA?*pu6zpOm@O_`&f{|3RA}cHByzH;-w=I zBD!oQ_C&oH8Tnz4>`a|GT3&ezswBpAX&$%j4E6a4iW9lSQRJd`o#!C74|1R= z3X=GiQWWe`o@5qc0d;%my(gu2D18j)izywLRk;u>#iLgD?2RSqxiXGgd~>)_S(Xt| z6%w=DKC>C9Kb8e?O;=JIownTse(5#Dg0@Z#1g$1+LaVvhYzIJJv}YQ9;Fc15ls`it zdPT6G&zPE+Ji1%jAZW+F6RpS#qZ!)cW8ktQRBSwRGFQ4fO79oMfb z?A4r7x?+GMOkC#(3w)Ta@Or(>F>TAS$a7vgQrq7O$_<|&f-2_h#=0`GY6wKPZauRc zRfArRyZM$zSCegMG^$hx#Oe!NhHCFLxH`XftPM-<32XFRnwvP1TU#>Q^+Hot^Xx_- zgnyB>aMHrcmTP8N=y-!*PlLtmohtL&Wl9)oYf{Tqoy67rl2MOTWuJG=Q|lK9U&|ws zR~95Y?csAwQMCSMKyI51eFNOA2pT|tN2JVt6F?=xhw9!(hAr#9}mDihnV;{_~O0OquFGb~t~S?S@C#b#|^Y*#~TLzWr( z0UNKJ8bkx!$-&d-Ct!J|T`d_m4^I8R608zHr0P*&JW!~3jPs;+P`j|W%VmXQ!4KI< zX+LZEU_9n*oT@_(5AQCmN;52a4Tx$a1midTiDN~!J-mE!yk~qdHD9ER*So6L&CVR& zLp$sHd1bL*rD26i0cpxKp1k7F3^aO=6@b3eLD5iCcHd{M2+NL~7uTg4twP=qXcNTu$9t>ht##$# zciC=Lc>VLkk4_&#j!i}uU(ZiE^CbP0V}B`Gnz9U}8AbQj<~=iVX84#t0k8IFWUWdc z%UCe4bdvHT%GMhfiQm3H()e3MOeue`V|?^peeiA2yAEnTKgFw3R3+Rb>@eHhT2dRd zMQ;1v7{m1Qx)0bM#g?_PFpl`A7Xc-)-g5y#X(poN2QjjEw8g>R%Qef4 ztScJiTRW3yDu8!bKL5C-&&jA~rhd$j9gq_g-Ryz!ci&8=(ZJKSos%G zK!7+gKj7KCDC|cZq{OrxYF| zRi%NN=v?&S(Zk!|UCJuJL?oQPtud-#fB<;jc&tY)iwq}8OY`XjxL_%buH~8L&`C%A zR*|3y`cwAgvuN@%C-wQq+pLg#X zHu?x9aNZ=AYmU!ERLF(hOn5V_Sp>i0kn;(*QvlDU^KgS%5t5+gQFp`^-IxZ={?3HP z71dVDvIN)wZRQ)Dxu}AbgNu`8ie^Jy=ENxw9?4goZX=k8-)B{fO!~t9N&$~%&&zXj zH@^f)_Bj&Tv-)sXF_zW8Dtk$_DDOVDJ9^$@t@cdNK`jpNIak`Uzn-W4)rHB^vY6!v zJ^4eRw3<0TmZ&NWrUTN&CoJABJYQnSDgmXc(5)2P)qDH$g<<+^rXBl1SFn@2#((2= zH|g7m*D5t{kdFssU2A9kvdtB5;(dG}pNwmH%2D@qssFx8b=3+ZP_8BjuZ!f`t~Es? zAf^UrDBgt;m+E)6ih*wNFPOOffDfkrLit-u?wF8J0A$6DoQ1p7Nq4Z5L-`8`_gfTwuu&p#f~vjCffj#eWvK$r_|PV(RAlgG0Z7iULIWsAO=-$ zQ+8qKg>g+(rcPLatE;j8Bx1N8+&JCutj7m^L>K^~x@%@xt;{63L2%Y_$dCP}9A0{+ zov&MNZ7c6Axsxj$$pY*o-L`guuK&q-a#s;pwdL_b6Ehv zY`<`XFmg85uSP3!EYheeR{Y|v%Y~rCQ(}Htfc2+}I(0rfMGD-_2SFH{-Qmx)V|Tn+ zji?bqRqgk;+DM*F!iDsSxzN$;e(^l>a`JS4)qB9Ej@#^r9G)cKRQGPM4MXinv+Dg= zj<{EPp2aN#Zy(O!ZWHVC1?P#Ha6SMRN926cQ8#*y_Xn4Nar1M_?Lc8HOy*R-Nx2n` z2)y`g61^6D$1CJlP6vj{CN%N2LGb6Az23=cH>w?YE@?U*eIOQSoxNo7(k$#%AaFzp(b|;tnT0Cr)1teg>*3j#cmRYt(X{mO$yF=htudzh(fiEr4@;F>og3;2qwNUhO9=J_(7Ml@#V; ziIALdvQ~eUD-Uo}xi{QYAEoUel{YhJ>}nH;pV>c7*D6n^NAy}rcAdb`yUmnNQdRZ> zU?4Dlr^WKUK-bdSg0;P}^#$dZS;>7);Dmp9HoK*mEtQQ4Cn$011Ic&Ge$k|db?)_MlB|MRk{SNs$;VP)S|?DjsGD+ zYv*vyX-?I(T^ufFPXSlBFYUv}abka^Q7iHZER2f^hyCu$R271Dy_~JdYiWBW8;+Yq z#hY|*>zUyaC5`8}SLSkRe$%w^8A|L41BsmBnRT%$A2$f#0IRG;g0WHLR1iaLI~$BZ z06?rQ?E-sGFYa=&^Q)4^lw4C`JXhqyCmSL6U*CeTrqDq=dI)C+dO3qiZ_RaAktCl3 zSWyw6$Dgly$u?}n0hE(@%OJC7dWvq>EY0j?h!36sAI0RGEEpNivR(Z{9&1q z_xL~||Ks8YxvP|ni@zn^8$FbGPc@fy!?$4IgqSN7%uO#(8Av_r$(?NhRnDEBy=Ou1 zx_$E3GfZyRo%C{99f^g=`Eq$4(d%=g?EayT(Si*2eR*1Mbwx(=+G0XyQ)3;YVx)*b z`wuw7k;8<)r%j1BPO;0*QSDd#?&IcsSp)5}L!Uh-Sibn6r??_?ygR?Mj*)fG>lW8@ z+2`Y%xKw?AgVKANDRiK4MhrwXBEY4$BYeM4tDV0tUEF{`vz)?i_9tw=THA3;zu4KD z;*i8nN;u9QGC$F6O8}l;SFRcAlkKa$dk)E?aF)^Z-W63UvJ)cTmTT!>oGD++Q zo~cFfO-WurSB_U}z~^2vh50=R?loHp5k!P)f?hjB)P{zHIbYq{;Cu~_<=kC7G14PS zUR8xyQ^P*)!KPQz^xFFLa$i4tRe<)kmQ}MOt{x5I#s!QM)w?^8o5l`5u*O?op_8N9 z`^kqrbm&k5jL4NTyKhXXzCwt(%iaB#zQy+SA42K?Ozu#*X_LS!m!R$O^~#PxBfAZO zSV%Yo_~=X7pD%E@Wj&1-mBovW#Z65f4aqK#0fZU zDe5&*j|EhjJdc1}Q*H_DMkpH(TrI!Su0Vd$MJ(VO9%+q6rn`X4E*D3ah#nynMKi!F zP|f>rJ$wu_Vw%>O;KmL2b090@(luc1$hy-}Cx_9#e3`Nzv#n+O&rxkF|0{`F>88tR zM+6U2apD??Iz1TsrQ_Q+l9UavOhk1z)r#f=?V5+)(^yZ7JxxvT(K=2Px#mYG9EDf{@D zXCorB3jPLAtE0*w`O%WOQsI+$^fqvhXC6|;Jk1~W+%As9;$Q~Xd+ani&>IMe(wE&3 zTc1OtcO?<=4JbXI7;ws)=)E`)J&*J0L)_f{hXOPX&G#VQzH1+;$(xNFaGGl7FGuN# zsDn9$X0?EwpN$9F?5ad+5H_G7tJTvH=CYGV&T7m`^2YMQE!WeMcyuWioiy4hT4KjI zqpFA~mxX6W8^mqXxltb%o)otG*7@#W61c%HIgF3oD6`cd7E$_8o-XT^H=v9lSYnMT z#=s3+&#!szJc9rzE^#2>wFNcYRzNgXrCL=Dpt<1=&t$Ca z{NWywrR{~sK-y&tq+Hf$Q>udYP|4Td<*Hjg^@M;}Y8so2L4dl^{M~#4)0199 z5141yCic1R<=hyd`>aP?z`5eGKFi<}EvJVWN9ZX-1p#a_Wf`d*5FLpk4X%#+-QV6B zeh0`RUqeNX<*|F}>@RTW9z_h%VL`ei1*%ygz%7!HSHn`%A~NRa~EQy#~)mjq)ZS;u2A*9o5BOzd?#3 zpD_FZRyY3z*^I!+Pnk!AYMlx9_(*s%0MTx;8o z@_HOO#0kHO2eEm}YV)s1IgirjqT8TzCdk$a&hDZ?3V&6yV)_<(@HN~=DtemqjEP`z zz-4!w>|{v`Ok7|0zQ26md@ap}!s(3S0oUGe7%1llR2BVyE!F61wI?HZA!Flo6thoId}_lk)#Y z5o{HfkvZ?IMgqE5oy0|WC2SLq*UwS{rxc(U`kXCK3sYg9w8C_jE!k1%H?qjc+5}@#Ede>NA7-WyOJL;1N+N3X&L1BIY5E@%utyGq&nR& zKR*5DA||&!0+tO9V~tUQBR>?zu>V#s%Aq*|7G0mZZ*b{5Kf7VQ{y4(-7x`u8Lt^TU zgbY%3-WNWRUV9!Pf~_zQ+O*_Djw;-u53C4ScGv_7R|F*i3bD{`1cN!w0?vgJP*!ufO|;mwlhou}m;; z?XjAst#WM~%wSL=N7?sp`3?WWn<2DBqo<>z(;;*Ti(v8JiveIB`R=lng4VF zXR?{1e;(hK<`0hS7Vx0Rc~Y(9wm1(T0>po=ur+&aaj9taApUc91U>zi_4wOg%>dQa zKPRQRwAFQz69=?L3nr{;%lxM)0C5Qo{1aO}v;N(PEq$N<-H0F9rS(<(_lfwCIfmBh z-=k?uC+z>+(X?=i{RIsT1D%$dvf-Ky`PW`-2~+>qaR1mxW9BFQfAXM6UqNhV7Yz-~ z1B+GD3kZRo8uL*!v~H@~aV@n|-N **Note:** The `wireguardPeerCIDR` field in the `NebiusNodeClass` is only required when using WireGuard for cross-cloud connectivity. When using Unbounded CNI, this field should not be set. - ### Creating a NodePool The `NodePool` defines scheduling constraints and references the `NebiusNodeClass`: @@ -291,11 +289,11 @@ After a few minutes, the new Nebius node should appear: ```bash $ kubectl get nodes -o wide NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME +aks-gateway-26665104-vmss000000 Ready 3h9m v1.34.2 172.16.2.4 Ubuntu 22.04.5 LTS 5.15.0-1102-azure containerd://1.7.30-1 aks-system-94214615-vmss000000 Ready 3h13m v1.34.2 172.16.1.4 Ubuntu 22.04.5 LTS 5.15.0-1102-azure containerd://1.7.30-1 aks-system-94214615-vmss000001 Ready 3h13m v1.34.2 172.16.1.5 Ubuntu 22.04.5 LTS 5.15.0-1102-azure containerd://1.7.30-1 aks-system-94214615-vmss000002 Ready 3h13m v1.34.2 172.16.1.6 Ubuntu 22.04.5 LTS 5.15.0-1102-azure containerd://1.7.30-1 -aks-wireguard-23306360-vmss000000 Ready 3h9m v1.34.2 172.16.2.4 20.91.194.208 Ubuntu 22.04.5 LTS 5.15.0-1102-azure containerd://1.7.30-1 -computeinstance-e00a4p0rrnms9n24jp Ready 8m30s v1.33.3 100.96.1.237 Ubuntu 24.04.4 LTS 6.11.0-1016-nvidia containerd://2.0.4 +computeinstance-e00a4p0rrnms9n24jp Ready 8m30s v1.33.3 172.20.0.10 Ubuntu 24.04.4 LTS 6.11.0-1016-nvidia containerd://2.0.4 ``` The pod is now running on the Nebius node: @@ -362,10 +360,10 @@ After the GPU node is provisioned, both nodes and pods should be running: ```bash $ kubectl get nodes NAME STATUS ROLES AGE VERSION +aks-gateway-26665104-vmss000000 Ready 3h14m v1.34.2 aks-system-94214615-vmss000000 Ready 3h19m v1.34.2 aks-system-94214615-vmss000001 Ready 3h19m v1.34.2 aks-system-94214615-vmss000002 Ready 3h18m v1.34.2 -aks-wireguard-23306360-vmss000000 Ready 3h14m v1.34.2 computeinstance-e00a4p0rrnms9n24jp Ready 13m v1.33.3 computeinstance-e00zjdx1e50bxcfekk Ready 107s v1.33.3 ``` @@ -436,9 +434,9 @@ After the disruption grace period, Karpenter will terminate the idle Nebius node ```bash $ kubectl get nodes NAME STATUS ROLES AGE VERSION +aks-gateway-26665104-vmss000000 Ready 18h v1.33.6 aks-system-32742974-vmss000000 Ready 18h v1.33.6 aks-system-32742974-vmss000001 Ready 18h v1.33.6 -aks-wireguard-12237243-vmss000000 Ready 18h v1.33.6 ``` ![](./images/karpenter/node-deletion.png)