feat: windows cse ensure oras and reserve some exit code for network isolated cluster

parts/windows/kuberneteswindowssetup.ps1

@@ -221,8 +221,13 @@ $global:WindowsCiliumNetworkingPath = Join-Path -Path $global:cacheDir -ChildPat
 $global:WindowsCiliumInstallPath = Join-Path -Path $global:WindowsCiliumNetworkingPath -ChildPath 'install'
 
 # Network isolated cluster
-$global:BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER="{{GetBootstrapProfileContainerRegistryServer}}"
-$global:MCR_REPOSITORY_BASE="{{GetMCRRepositoryBase}}"
+$global:BootstrapProfileContainerRegistryServer="{{GetBootstrapProfileContainerRegistryServer}}"
+$global:MCRRepositoryBase="{{GetMCRRepositoryBase}}"
+
+$global:OrasCacheDir="c:\aks-tools\oras\" # refer to components.json
+$global:OrasPath="c:\aks-tools\oras\oras.exe"
+$global:OrasOutput="c:\aks-tools\oras\oras_verbose.out"
+$global:OrasRegistryConfigFile="c:\aks-tools\oras\config.yaml" # oras registry auth config file, not used, but have to define to avoid error "Error: failed to get user home directory: $HOME is not defined"
 
 # Extract cse helper script from ZIP
 [io.file]::WriteAllBytes("scripts.zip", [System.Convert]::FromBase64String($zippedFiles))
@@ -291,6 +296,12 @@ if (Test-Path -Path 'c:\AzureData\windows\windowsciliumnetworkingfunc.ps1') {
     Write-Log "Windows Cilium Networking function script not found, skipping dot-source"
 }
 
+if (Test-Path -Path 'c:\AzureData\windows\networkisolatedclusterfunc.ps1') {
+    . c:\AzureData\windows\networkisolatedclusterfunc.ps1
+} else {
+    Write-Log "Network Isolated Cluster function script not found, skipping dot-source"
+}
+
 # ====== BASE PREP: BASE IMAGE PREPARATION ======
 # All operations that prepare the base VHD image
 function BasePrep {
@@ -333,6 +344,18 @@ function BasePrep {
 
     Write-KubeClusterConfig -MasterIP $MasterIP -KubeDnsServiceIp $KubeDnsServiceIp
 
+    # oras initialization, including install and login, must be in front of Install-CredentialProvider, Get-KubePackage and Install-Containerd-Based-On-Kubernetes-Version
+    if ((Test-Path variable:global:BootstrapProfileContainerRegistryServer) -and
+    -not [string]::IsNullOrWhiteSpace($global:BootstrapProfileContainerRegistryServer)) {
+    # variable exists and is not empty/whitespace
+        if (Get-Command -Name Initialize-Oras -ErrorAction SilentlyContinue) {
+            Logs-To-Event -TaskName "AKS.WindowsCSE.InitializeOras" -TaskMessage "Ensure oras is initialized for network isolated cluster"
+            Initialize-Oras
+        } else {
+            Write-Log "Initialize-Oras is not a recognized function, will skip oras initialization for network isolated cluster"
+        }
+    }
+
     # to ensure we don't introduce any incompatibility between base CSE + CSE package versions
     if (Get-Command -Name Install-SecureTLSBootstrapClient -ErrorAction SilentlyContinue) {
         Install-SecureTLSBootstrapClient -KubeDir $global:KubeDir -CustomSecureTLSBootstrapClientDownloadUrl $global:CustomSecureTLSBootstrappingClientDownloadURL

parts/windows/windowscsehelper.ps1

@@ -79,9 +79,16 @@ $global:WINDOWS_CSE_ERROR_WINDOWS_CILIUM_NETWORKING_INSTALL_FAILED=72
 $global:WINDOWS_CSE_ERROR_EXTRACT_ZIP=73
 $global:WINDOWS_CSE_ERROR_LOAD_METADATA=74
 $global:WINDOWS_CSE_ERROR_PARSE_METADATA=75
+$global:WINDOWS_CSE_ERROR_ORAS_NOT_FOUND=76 # exit code for not finding oras in the expected path, which is a prerequisite for pulling packages from registry for network isolated cluster
+$global:WINDOWS_CSE_ERROR_ORAS_IMDS_TIMEOUT=77 # exit code for timeout waiting for IMDS response
+$global:WINDOWS_CSE_ERROR_ORAS_PULL_NETWORK_TIMEOUT=78 # exit code for error pulling oras when login
+$global:WINDOWS_CSE_ERROR_ORAS_PULL_UNAUTHORIZED=79 # exit code for error pulling artifact with oras from registry with authorization issue
+$global:WINDOWS_CSE_ERROR_ORAS_PULL_WINDOWSZIP_FAIL=80 # exit code for error pulling kubelet kubectl artifact with oras from registry
+$global:WINDOWS_CSE_ERROR_ORAS_PULL_CREDENTIAL_PROVIDER=81 # exit code for error pulling credential provider artifact with oras from registry
+$global:WINDOWS_CSE_ERROR_ORAS_PULL_POD_INFRA_CONTAINER=82 # exit code for error pulling pause image with oras from registry
 # WINDOWS_CSE_ERROR_MAX_CODE is only used in unit tests to verify whether new error code name is added in $global:ErrorCodeNames
 # Please use the current value of WINDOWS_CSE_ERROR_MAX_CODE as the value of the new error code and increment it by 1
-$global:WINDOWS_CSE_ERROR_MAX_CODE=76
+$global:WINDOWS_CSE_ERROR_MAX_CODE=83
 
 # Please add new error code for downloading new packages in RP code too
 $global:ErrorCodeNames = @(
@@ -160,7 +167,14 @@ $global:ErrorCodeNames = @(
     "WINDOWS_CSE_ERROR_WINDOWS_CILIUM_NETWORKING_INSTALL_FAILED",
     "WINDOWS_CSE_ERROR_EXTRACT_ZIP",
     "WINDOWS_CSE_ERROR_LOAD_METADATA",
-    "WINDOWS_CSE_ERROR_PARSE_METADATA"
+    "WINDOWS_CSE_ERROR_PARSE_METADATA",
+    "WINDOWS_CSE_ERROR_ORAS_NOT_FOUND",
+    "WINDOWS_CSE_ERROR_ORAS_IMDS_TIMEOUT",
+    "WINDOWS_CSE_ERROR_ORAS_PULL_NETWORK_TIMEOUT",
+    "WINDOWS_CSE_ERROR_ORAS_PULL_UNAUTHORIZED",
+    "WINDOWS_CSE_ERROR_ORAS_PULL_WINDOWSZIP_FAIL",
+    "WINDOWS_CSE_ERROR_ORAS_PULL_CREDENTIAL_PROVIDER",
+    "WINDOWS_CSE_ERROR_ORAS_PULL_POD_INFRA_CONTAINER"
 )
 
 # The package domain to be used
@@ -635,3 +649,4 @@ function Resolve-Error ($ErrorRecord=$Error[0])
        $Exception |Format-List * -Force
    }
 }
+

staging/cse/windows/containerdfunc.ps1

@@ -161,7 +161,7 @@ function ProcessAndWriteContainerdConfig {
 
   # Set up registry mirrors
   Set-ContainerdRegistryConfig -Registry "docker.io" -RegistryHost "registry-1.docker.io"
-  if ((Test-Path variable:global:BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER) -and -not [string]::IsNullOrEmpty($global:BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER)) {
+  if ((Test-Path variable:global:BootstrapProfileContainerRegistryServer) -and -not [string]::IsNullOrEmpty($global:BootstrapProfileContainerRegistryServer)) {
     Set-BootstrapProfileRegistryContainerdHost
   } else {
     Set-ContainerdRegistryConfig -Registry "mcr.azk8s.cn" -RegistryHost "mcr.azure.cn"
@@ -235,9 +235,9 @@ server = "https://$Registry"
 }
 
 function Set-BootstrapProfileRegistryContainerdHost {
-  $mcrRegistry = if ((Test-Path variable:global:MCR_REPOSITORY_BASE) -and
-      -not [string]::IsNullOrEmpty($global:MCR_REPOSITORY_BASE)) {
-    [string]$global:MCR_REPOSITORY_BASE
+  $mcrRegistry = if ((Test-Path variable:global:MCRRepositoryBase) -and
+      -not [string]::IsNullOrEmpty($global:MCRRepositoryBase)) {
+    [string]$global:MCRRepositoryBase
   }
   else {
     "mcr.microsoft.com"
@@ -246,7 +246,7 @@ function Set-BootstrapProfileRegistryContainerdHost {
   $mcrRegistryPath = Join-Path $rootRegistryPath $mcrRegistry
   $hostsTomlPath = Join-Path $mcrRegistryPath "hosts.toml"
 
-  $registryHost = [string]$global:BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER
+  $registryHost = [string]$global:BootstrapProfileContainerRegistryServer
   $registryHost = ($registryHost -replace '^https?://', '').TrimEnd('/')
 
   $registryHostParts = $registryHost.Split('/', 2)

staging/cse/windows/containerdfunc.tests.ps1

@@ -309,10 +309,10 @@ Describe "Set-BootstrapProfileRegistryContainerdHost" {
     }
   }
 
-  It "Should write hosts.toml for default mcr.microsoft.com when MCR_REPOSITORY_BASE is not set" {
-    $global:BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER = "myacr.azurecr.io"
-    if (Test-Path variable:global:MCR_REPOSITORY_BASE) {
-      Remove-Variable -Name MCR_REPOSITORY_BASE -Scope Global
+  It "Should write hosts.toml for default mcr.microsoft.com when MCRRepositoryBase is not set" {
+    $global:BootstrapProfileContainerRegistryServer = "myacr.azurecr.io"
+    if (Test-Path variable:global:MCRRepositoryBase) {
+      Remove-Variable -Name MCRRepositoryBase -Scope Global
     }
 
     Set-BootstrapProfileRegistryContainerdHost
@@ -329,8 +329,8 @@ Describe "Set-BootstrapProfileRegistryContainerdHost" {
   }
 
   It "Should sanitize bootstrap profile host and use custom mcr repository base" {
-    $global:MCR_REPOSITORY_BASE = "my.mcr.mirror"
-    $global:BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER = "https://myacr.azurecr.io/some/path/"
+    $global:MCRRepositoryBase = "my.mcr.mirror"
+    $global:BootstrapProfileContainerRegistryServer = "https://myacr.azurecr.io/some/path/"
 
     Set-BootstrapProfileRegistryContainerdHost
 
@@ -343,8 +343,8 @@ Describe "Set-BootstrapProfileRegistryContainerdHost" {
   }
 
   It "Should map host with repository prefix to v2 path" {
-    $global:MCR_REPOSITORY_BASE = "mcr.microsoft.com"
-    $global:BOOTSTRAP_PROFILE_CONTAINER_REGISTRY_SERVER = "myacr.azurecr.io/aaa"
+    $global:MCRRepositoryBase = "mcr.microsoft.com"
+    $global:BootstrapProfileContainerRegistryServer = "myacr.azurecr.io/aaa"
 
     Set-BootstrapProfileRegistryContainerdHost
 

staging/cse/windows/networkisolatedclusterfunc.ps1

@@ -0,0 +1,67 @@
+# functions for network isolated cluster
+
+# Initialize-Oras will install oras and login the registry if anonymous access is disabled. This is required for network isolated cluster to pull windowszip from private container registry.
+function Initialize-Oras {
+    Install-Oras
+    # reserve for Invoke-OrasLogin to avoid frequent code changes in parts/windows/
+}
+
+# unpackage and install oras from cache
+# Oras is used for pulling windows binaries, e.g. windowszip, from private container registry when it is network isolated cluster.
+function Install-Oras {
+    # Check if OrasPath variable exists to avoid latest cached cse in vhd with possible old ab svc
+    $orasPathVarExists = Test-Path variable:global:OrasPath
+    if (-not $orasPathVarExists) {
+        Write-Log "OrasPath variable does not exist. Setting OrasPath to default value C:\aks-tools\oras\oras.exe"
+        $global:OrasPath = "C:\aks-tools\oras\oras.exe"
+    }
+
+    if (Test-Path -Path $global:OrasPath) {
+        # oras already installed, skip
+        Write-Log "Oras already installed at $($global:OrasPath)"
+        return
+    }
+    # Ensure cache directory exists before checking for archives or downloading
+    if (-Not (Test-Path $global:OrasCacheDir)) {
+        New-Item -ItemType Directory -Path $global:OrasCacheDir -Force | Out-Null
+    }
+
+    if (-Not (Test-Path $global:OrasCacheDir)) {
+        Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_ORAS_NOT_FOUND -ErrorMessage "Oras cache directory not found at $($global:OrasCacheDir)"
+    }
+
+    # Look for a cached oras archive (.tar.gz or .zip) in the oras cache directory
+    $orasArchive = Get-ChildItem -Path $global:OrasCacheDir -File |
+        Where-Object { $_.Name -like "*.tar.gz" -or $_.Name -like "*.zip" } |
+        Sort-Object LastWriteTime -Descending |
+        Select-Object -First 1
+    if (-Not $orasArchive) {
+        Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_ORAS_NOT_FOUND -ErrorMessage "No oras archive (.tar.gz or .zip) found in $($global:OrasCacheDir)"
+    }
+
+    # Extract the archive to the oras install directory
+    $orasInstallDir = [IO.Path]::GetDirectoryName($global:OrasPath)
+    if (-Not (Test-Path $orasInstallDir)) {
+        New-Item -ItemType Directory -Path $orasInstallDir -Force | Out-Null
+    }
+
+    Write-Log "Extracting oras from $($orasArchive.FullName) to $orasInstallDir"
+    if ($orasArchive.Name -like "*.zip") {
+        AKS-Expand-Archive -Path $orasArchive.FullName -DestinationPath $orasInstallDir
+    } elseif ($orasArchive.Name -like "*.tar.gz") {
+        try {
+            tar -xzf $orasArchive.FullName -C $orasInstallDir
+            if ($LASTEXITCODE -ne 0) {
+                Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_ORAS_NOT_FOUND -ErrorMessage "Failed to extract oras archive $($orasArchive.FullName) (tar exit code $LASTEXITCODE)"
+            }
+        } catch {
+            Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_ORAS_NOT_FOUND -ErrorMessage "Exception while extracting oras archive $($orasArchive.FullName): $($_.Exception.Message)"
+        }
+    }
+
+    if (-Not (Test-Path $global:OrasPath)) {
+        Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_ORAS_NOT_FOUND -ErrorMessage "Oras executable not found at $($global:OrasPath) after extraction"
+    }
+
+    Write-Log "Oras installed successfully at $($global:OrasPath)"
+}

staging/cse/windows/networkisolatedclusterfunc.tests.ps1

@@ -0,0 +1,107 @@
+
+BeforeAll {
+  . $PSScriptRoot\..\..\..\parts\windows\windowscsehelper.ps1
+  . $PSCommandPath.Replace('.tests.ps1', '.ps1')
+
+}
+
+Describe "Install-Oras" {
+  BeforeEach {
+    $global:OrasPath = "C:\aks-tools\oras\oras.exe"
+    $global:OrasCacheDir = "C:\akse-cache\oras"
+    $script:archiveExtractCalls = 0
+
+    Mock New-Item -MockWith {}
+    Mock Expand-Archive -MockWith { $script:archiveExtractCalls++ }
+    Mock AKS-Expand-Archive -MockWith {
+      param($Path, $DestinationPath, $Force)
+      $script:archiveExtractCalls++
+    }
+    Mock tar -MockWith {}
+    Mock Set-ExitCode -MockWith {
+      Param(
+        [Parameter(Mandatory = $true)][int]$ExitCode,
+        [Parameter(Mandatory = $true)][string]$ErrorMessage
+      )
+      throw "Set-ExitCode:${ExitCode}:${ErrorMessage}"
+    }
+  }
+
+  It "should return early when oras executable already exists" {
+    Mock Test-Path -MockWith {
+      Param($Path)
+      return $Path -eq $global:OrasPath
+    }
+
+    { Install-Oras } | Should -Not -Throw
+    Assert-MockCalled -CommandName 'New-Item' -Times 0
+    Assert-MockCalled -CommandName 'Expand-Archive' -Times 0
+    Assert-MockCalled -CommandName 'AKS-Expand-Archive' -Times 0
+  }
+
+  It "should extract cached zip archive and install oras" {
+    $script:orasInstalled = $false
+
+    Mock Test-Path -MockWith {
+      Param($Path)
+      switch ($Path) {
+        { $_ -eq $global:OrasPath } { return $script:orasInstalled }
+        { $_ -eq $global:OrasCacheDir } { return $true }
+        { $_ -eq "C:\aks-tools\oras" } { return $false }
+        default { return $true }
+      }
+    }
+
+    Mock Get-ChildItem -MockWith {
+      return [pscustomobject]@{ Name = "oras_1.3.0_windows_amd64.zip"; FullName = "C:\akse-cache\oras\oras_1.3.0_windows_amd64.zip" }
+    }
+
+    Mock Expand-Archive -MockWith {
+      $script:archiveExtractCalls++
+      $script:orasInstalled = $true
+    }
+    Mock AKS-Expand-Archive -MockWith {
+      param($Path, $DestinationPath, $Force)
+      $script:archiveExtractCalls++
+      $script:orasInstalled = $true
+    }
+
+    { Install-Oras } | Should -Not -Throw
+    $script:archiveExtractCalls | Should -Be 1
+  }
+
+  It "should fail when no cached oras archive exists" {
+    Mock Test-Path -MockWith {
+      Param($Path)
+      return $Path -ne $global:OrasPath
+    }
+
+    Mock Get-ChildItem -MockWith { @() }
+
+    {
+      Install-Oras
+    } | Should -Throw "*Set-ExitCode:$($global:WINDOWS_CSE_ERROR_ORAS_NOT_FOUND):No oras archive*"
+  }
+
+  It "should fail when tar extraction returns non-zero exit code" {
+    Mock Test-Path -MockWith {
+      Param($Path)
+      switch ($Path) {
+        { $_ -eq $global:OrasPath } { return $false }
+        { $_ -eq $global:OrasCacheDir } { return $true }
+        { $_ -eq "C:\aks-tools\oras" } { return $true }
+        default { return $true }
+      }
+    }
+
+    Mock Get-ChildItem -MockWith {
+      return [pscustomobject]@{ Name = "oras_1.3.0_windows_amd64.tar.gz"; FullName = "C:\akse-cache\oras\oras_1.3.0_windows_amd64.tar.gz" }
+    }
+
+    Mock tar -MockWith { $global:LASTEXITCODE = 1 }
+
+    {
+      Install-Oras
+    } | Should -Throw "*Set-ExitCode:$($global:WINDOWS_CSE_ERROR_ORAS_NOT_FOUND):Failed to extract oras archive*"
+  }
+}