feat: implement inter-site routing support for wg

plugin/pkg/services/agentpools/nebius/instance/agentpools.go

@@ -92,6 +92,7 @@ func (srv *agentPoolsServer) CreateOrUpdate(
 		ud.RunCmd = append(ud.RunCmd, strings.Join([]string{
 			"export ANNOTATION_PREFIX='stretch.azure.com/wireguard-'",
 			fmt.Sprintf("export WG_ADDRESS='%s/32'", wireguardIP),
+			fmt.Sprintf("export WG_SITE='%s'", apSpec.GetSubnetId()),
 			"export WG_DAEMONIZE='true'",
 			"/root/wg-spoke.sh",
 		}, "\n"))

plugin/pkg/services/agentpools/nebius/instance/assets/wg-spoke.sh

@@ -7,6 +7,11 @@
 # hub node's public key and endpoint. Configures the local WireGuard
 # interface and restarts it whenever the hub configuration changes.
 #
+# When WG_SITE is set, the script also discovers other spoke nodes in the
+# same site (e.g., same VPC/subnet) and establishes direct WireGuard
+# peerings using VPC-internal IPs as endpoints, so intra-site traffic
+# bypasses the hub.
+#
 # Environment variables:
 #   KUBECONFIG        — path to kubeconfig (default: /etc/kubernetes/kubelet.conf)
 #   NODE_NAME         — this node's name (default: $(hostname))
@@ -20,6 +25,11 @@
 #   WG_DAEMONIZE      — if set to "true", fork the poll loop into the background
 #                        so cloud-init or other callers don't block (default: false)
 #   WG_LOG_FILE       — log file path when daemonized (default: /var/log/wg-spoke.log)
+#   WG_SITE           — site identifier for intra-site peering (default: empty,
+#                        which disables site peer discovery)
+#   WG_VPC_IP         — this node's VPC/private IP, published to peers for
+#                        direct intra-site connectivity (default: auto-detected
+#                        from the default-route network interface)
 #
 
 set -euo pipefail
@@ -34,6 +44,14 @@ ANNOTATION_PREFIX="${ANNOTATION_PREFIX:-wireguard.kube/}"
 POLL_INTERVAL="${POLL_INTERVAL:-10}"
 WG_DAEMONIZE="${WG_DAEMONIZE:-false}"
 WG_LOG_FILE="${WG_LOG_FILE:-/var/log/wg-spoke.log}"
+WG_SITE="${WG_SITE:-}"
+
+# Auto-detect VPC IP from the default-route interface if not explicitly set.
+if [[ -z "${WG_VPC_IP:-}" ]]; then
+    _default_iface=$(ip route show default | awk '{print $5; exit}')
+    WG_VPC_IP=$(ip -4 addr show dev "${_default_iface}" | grep -oP 'inet \K[^/]+' | head -1)
+    unset _default_iface
+fi
 
 KUBECTL="kubectl --kubeconfig=${KUBECONFIG}"
 WG_CONFIG_DIR="/etc/wireguard"
@@ -44,11 +62,14 @@ ENDPOINT_ANNOTATION="${ANNOTATION_PREFIX}endpoint"
 ALLOWED_IPS_ANNOTATION="${ANNOTATION_PREFIX}allowed-ips"
 PEER_LABEL="${ANNOTATION_PREFIX}peer"
 HUB_LABEL="${ANNOTATION_PREFIX}hub"
+SITE_LABEL="${ANNOTATION_PREFIX}site"
+VPC_IP_ANNOTATION="${ANNOTATION_PREFIX}vpc-ip"
 
 # State tracking for change detection
 CURRENT_HUB_KEY=""
 CURRENT_HUB_ENDPOINT=""
 CURRENT_HUB_ALLOWED_IPS=""
+CURRENT_SITE_PEERS=""
 
 log() { echo "[$(date -u '+%Y-%m-%dT%H:%M:%SZ')] $*"; }
 
@@ -80,6 +101,16 @@ register_peer() {
         "${KEY_ANNOTATION}=${pubkey}" \
         "${ALLOWED_IPS_ANNOTATION}=${allowed_ips}" \
         --overwrite
+
+    # Publish site membership and VPC IP for intra-site peering.
+    if [[ -n "${WG_SITE}" ]]; then
+        ${KUBECTL} label node "${NODE_NAME}" "${SITE_LABEL}=${WG_SITE}" --overwrite
+        ${KUBECTL} annotate node "${NODE_NAME}" \
+            "${VPC_IP_ANNOTATION}=${WG_VPC_IP}" \
+            --overwrite
+        log "Site peer registered: site=${WG_SITE} vpc-ip=${WG_VPC_IP}"
+    fi
+
     log "Peer registered"
 }
 
@@ -103,8 +134,42 @@ get_node_annotation() {
         2>/dev/null || true
 }
 
+# --- Site peer discovery ---
+
+# discover_site_peers lists spoke nodes in the same site (excluding self).
+# Outputs one line per peer, sorted by node name for stable change detection:
+#   <node-name> <public-key> <vpc-ip> <allowed-ips>
+# Peers missing any required annotation are silently skipped.
+discover_site_peers() {
+    [[ -z "${WG_SITE}" ]] && return
+
+    local nodes
+    nodes=$(${KUBECTL} get nodes \
+        -l "${PEER_LABEL}=true,${SITE_LABEL}=${WG_SITE}" \
+        -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' \
+        2>/dev/null) || return
+
+    local node key vpc_ip allowed_ips
+    while IFS= read -r node; do
+        [[ -z "${node}" || "${node}" == "${NODE_NAME}" ]] && continue
+
+        key=$(get_node_annotation "${node}" "${KEY_ANNOTATION}")
+        vpc_ip=$(get_node_annotation "${node}" "${VPC_IP_ANNOTATION}")
+        allowed_ips=$(get_node_annotation "${node}" "${ALLOWED_IPS_ANNOTATION}")
+
+        # Skip peers that haven't fully registered yet.
+        [[ -z "${key}" || -z "${vpc_ip}" || -z "${allowed_ips}" ]] && continue
+
+        echo "${node} ${key} ${vpc_ip} ${allowed_ips}"
+    done <<< "${nodes}" | sort
+}
+
 # --- WireGuard config ---
 
+# write_config generates the WireGuard configuration file.
+# Args: hub_key hub_endpoint hub_allowed_ips
+# Site peers are read from the SITE_PEERS variable (one peer per line,
+# format: "<name> <public-key> <vpc-ip> <allowed-ips>").
 write_config() {
     local hub_key="$1"
     local hub_endpoint="$2"
@@ -125,6 +190,24 @@ Endpoint = ${hub_endpoint}
 AllowedIPs = ${hub_allowed_ips}
 PersistentKeepalive = 25
 EOF
+
+    # Append site peers (if any).
+    if [[ -n "${SITE_PEERS:-}" ]]; then
+        local name key vpc_ip allowed_ips
+        while IFS=' ' read -r name key vpc_ip allowed_ips; do
+            [[ -z "${name}" ]] && continue
+            cat >> "${WG_CONFIG}" <<EOF
+
+[Peer]
+# Site peer: ${name}
+PublicKey = ${key}
+Endpoint = ${vpc_ip}:${WG_LISTEN_PORT}
+AllowedIPs = ${allowed_ips}
+PersistentKeepalive = 25
+EOF
+        done <<< "${SITE_PEERS}"
+    fi
+
     chmod 600 "${WG_CONFIG}"
 }
 
@@ -135,10 +218,25 @@ restart_wg() {
     log "WireGuard interface ${WG_INTERFACE} is up"
 }
 
+# reload_wg applies peer changes without tearing down the interface.
+# Falls back to a full restart if syncconf is not available.
+reload_wg() {
+    log "Reloading WireGuard peers on ${WG_INTERFACE}..."
+    if wg syncconf "${WG_INTERFACE}" <(wg-quick strip "${WG_INTERFACE}") 2>/dev/null; then
+        log "WireGuard peers reloaded"
+    else
+        log "syncconf failed, falling back to full restart"
+        restart_wg
+    fi
+}
+
 # --- Main loop ---
 
 poll_loop() {
     log "Watching for hub node (label ${HUB_LABEL}=true), polling every ${POLL_INTERVAL}s..."
+    if [[ -n "${WG_SITE}" ]]; then
+        log "Site peering enabled: site=${WG_SITE} vpc-ip=${WG_VPC_IP}"
+    fi
 
     while true; do
         hub_node=$(discover_hub)
@@ -171,13 +269,43 @@ poll_loop() {
             fi
         fi
 
+        # Discover site peers (empty string if WG_SITE is unset).
+        SITE_PEERS=$(discover_site_peers)
+
+        local hub_changed=false
+        local peers_changed=false
+
         if [[ "${hub_key}" != "${CURRENT_HUB_KEY}" || "${hub_endpoint}" != "${CURRENT_HUB_ENDPOINT}" || "${hub_allowed_ips}" != "${CURRENT_HUB_ALLOWED_IPS}" ]]; then
             log "Hub config changed: node=${hub_node} key=${hub_key} endpoint=${hub_endpoint} allowed-ips=${hub_allowed_ips}"
+            hub_changed=true
+        fi
+
+        if [[ "${SITE_PEERS}" != "${CURRENT_SITE_PEERS}" ]]; then
+            if [[ -n "${SITE_PEERS}" ]]; then
+                local peer_count
+                peer_count=$(echo "${SITE_PEERS}" | wc -l | tr -d ' ')
+                log "Site peers changed: ${peer_count} peer(s) in site ${WG_SITE}"
+            else
+                log "Site peers changed: no peers in site ${WG_SITE}"
+            fi
+            peers_changed=true
+        fi
+
+        if [[ "${hub_changed}" == "true" || "${peers_changed}" == "true" ]]; then
             write_config "${hub_key}" "${hub_endpoint}" "${hub_allowed_ips}"
-            restart_wg
+
+            if [[ "${hub_changed}" == "true" ]]; then
+                # Interface config changed — full restart required.
+                restart_wg
+            else
+                # Only peers changed — live reload is sufficient.
+                reload_wg
+            fi
+
             CURRENT_HUB_KEY="${hub_key}"
             CURRENT_HUB_ENDPOINT="${hub_endpoint}"
             CURRENT_HUB_ALLOWED_IPS="${hub_allowed_ips}"
+            CURRENT_SITE_PEERS="${SITE_PEERS}"
         fi
 
         sleep "${POLL_INTERVAL}"