Implement Runtime NVMe Instance Storage Discovery Using AWS EBS Symlinks

agent/bootstrap_test.go

@@ -1132,6 +1132,16 @@ var _ = Describe("bootstrap", func() {
 				monitRetryStrategy := boshretry.NewAttemptRetryStrategy(10, 1*time.Second, monitRetryable, logger)
 
 				devicePathResolver := fakedevicepathresolver.NewFakeDevicePathResolver()
+				instanceStorageResolver := fakedevicepathresolver.NewFakeInstanceStorageResolver()
+
+				// Default stub: instance storage resolver returns device paths as-is
+				instanceStorageResolver.DiscoverInstanceStorageStub = func(devices []boshsettings.DiskSettings) ([]string, error) {
+					paths := make([]string, len(devices))
+					for i, device := range devices {
+						paths[i] = device.Path
+					}
+					return paths, nil
+				}
 
 				fakeUUIDGenerator := boshuuid.NewGenerator()
 				routesSearcher := boshnet.NewRoutesSearcher(logger, runner, nil)
@@ -1153,6 +1163,7 @@ var _ = Describe("bootstrap", func() {
 					ubuntuCertManager,
 					monitRetryStrategy,
 					devicePathResolver,
+					instanceStorageResolver,
 					state,
 					linuxOptions,
 					logger,

infrastructure/devicepathresolver/auto_detecting_instance_storage_resolver.go

@@ -0,0 +1,99 @@
+package devicepathresolver
+
+import (
+	"strings"
+
+	boshlog "github.com/cloudfoundry/bosh-utils/logger"
+	boshsys "github.com/cloudfoundry/bosh-utils/system"
+
+	boshsettings "github.com/cloudfoundry/bosh-agent/v2/settings"
+)
+
+// autoDetectingInstanceStorageResolver automatically detects whether to use
+// NVMe-specific logic or identity resolution based on device paths from the CPI.
+// If any device path starts with "/dev/nvme", it uses AWS NVMe discovery logic.
+// Otherwise, it uses the CPI-provided paths directly (identity resolution).
+type autoDetectingInstanceStorageResolver struct {
+	fs                  boshsys.FileSystem
+	devicePathResolver  DevicePathResolver
+	logger              boshlog.Logger
+	ebsSymlinkPattern   string
+	nvmeDevicePattern   string
+	awsNVMeResolver     InstanceStorageResolver
+	identityResolver    InstanceStorageResolver
+	resolverInitialized bool
+	useNVMeResolver     bool
+}
+
+// NewAutoDetectingInstanceStorageResolver creates a resolver that automatically
+// detects NVMe instances based on device paths from the CPI.
+func NewAutoDetectingInstanceStorageResolver(
+	fs boshsys.FileSystem,
+	devicePathResolver DevicePathResolver,
+	logger boshlog.Logger,
+	ebsSymlinkPattern string,
+	nvmeDevicePattern string,
+) InstanceStorageResolver {
+	if ebsSymlinkPattern == "" {
+		ebsSymlinkPattern = "/dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_*"
+	}
+	if nvmeDevicePattern == "" {
+		nvmeDevicePattern = "/dev/nvme*n1"
+	}
+
+	return &autoDetectingInstanceStorageResolver{
+		fs:                  fs,
+		devicePathResolver:  devicePathResolver,
+		logger:              logger,
+		ebsSymlinkPattern:   ebsSymlinkPattern,
+		nvmeDevicePattern:   nvmeDevicePattern,
+		resolverInitialized: false,
+	}
+}
+
+func (r *autoDetectingInstanceStorageResolver) DiscoverInstanceStorage(devices []boshsettings.DiskSettings) ([]string, error) {
+	if len(devices) == 0 {
+		return []string{}, nil
+	}
+
+	// Auto-detect on first call by checking if any device path starts with /dev/nvme
+	if !r.resolverInitialized {
+		r.useNVMeResolver = r.detectNVMeDevices(devices)
+
+		if r.useNVMeResolver {
+			r.logger.Info("AutoDetectingInstanceStorageResolver",
+				"Detected NVMe device paths from CPI - using AWS NVMe instance storage discovery")
+			r.awsNVMeResolver = NewAWSNVMeInstanceStorageResolver(
+				r.fs,
+				r.devicePathResolver,
+				r.logger,
+				r.ebsSymlinkPattern,
+				r.nvmeDevicePattern,
+			)
+		} else {
+			r.logger.Info("AutoDetectingInstanceStorageResolver",
+				"Detected non-NVMe device paths from CPI - using identity resolution")
+			r.identityResolver = NewIdentityInstanceStorageResolver(r.devicePathResolver)
+		}
+
+		r.resolverInitialized = true
+	}
+
+	if r.useNVMeResolver {
+		return r.awsNVMeResolver.DiscoverInstanceStorage(devices)
+	}
+	return r.identityResolver.DiscoverInstanceStorage(devices)
+}
+
+// detectNVMeDevices checks if any device path from the CPI starts with /dev/nvme
+// This matches the CPI's logic: if current_disk =~ /^\/dev\/nvme/
+func (r *autoDetectingInstanceStorageResolver) detectNVMeDevices(devices []boshsettings.DiskSettings) bool {
+	for _, device := range devices {
+		if strings.HasPrefix(device.Path, "/dev/nvme") {
+			r.logger.Debug("AutoDetectingInstanceStorageResolver",
+				"Detected NVMe from CPI-provided path: %s", device.Path)
+			return true
+		}
+	}
+	return false
+}

infrastructure/devicepathresolver/auto_detecting_instance_storage_resolver_test.go

@@ -0,0 +1,115 @@
+package devicepathresolver_test
+
+import (
+	"runtime"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	boshlog "github.com/cloudfoundry/bosh-utils/logger"
+	fakesys "github.com/cloudfoundry/bosh-utils/system/fakes"
+
+	. "github.com/cloudfoundry/bosh-agent/v2/infrastructure/devicepathresolver"
+	fakedpresolv "github.com/cloudfoundry/bosh-agent/v2/infrastructure/devicepathresolver/fakes"
+	boshsettings "github.com/cloudfoundry/bosh-agent/v2/settings"
+)
+
+var _ = Describe("AutoDetectingInstanceStorageResolver", func() {
+	var (
+		resolver               InstanceStorageResolver
+		fakeFS                 *fakesys.FakeFileSystem
+		fakeDevicePathResolver *fakedpresolv.FakeDevicePathResolver
+		logger                 boshlog.Logger
+	)
+
+	BeforeEach(func() {
+		if runtime.GOOS != "linux" {
+			Skip("Only supported on Linux")
+		}
+		fakeFS = fakesys.NewFakeFileSystem()
+		fakeDevicePathResolver = fakedpresolv.NewFakeDevicePathResolver()
+		logger = boshlog.NewLogger(boshlog.LevelNone)
+	})
+
+	Context("when CPI provides NVMe device paths", func() {
+		It("automatically uses AWS NVMe instance storage discovery", func() {
+			resolver = NewAutoDetectingInstanceStorageResolver(
+				fakeFS,
+				fakeDevicePathResolver,
+				logger,
+				"/dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_*",
+				"/dev/nvme*n1",
+			)
+
+			devices := []boshsettings.DiskSettings{
+				{Path: "/dev/nvme0n1"},
+				{Path: "/dev/nvme1n1"},
+			}
+
+			err := fakeFS.WriteFileString("/dev/nvme0n1", "")
+			Expect(err).NotTo(HaveOccurred())
+			err = fakeFS.WriteFileString("/dev/nvme1n1", "")
+			Expect(err).NotTo(HaveOccurred())
+			err = fakeFS.WriteFileString("/dev/nvme2n1", "")
+			Expect(err).NotTo(HaveOccurred())
+
+			fakeFS.GlobStub = func(pattern string) ([]string, error) {
+				if pattern == "/dev/nvme*n1" {
+					return []string{"/dev/nvme0n1", "/dev/nvme1n1", "/dev/nvme2n1"}, nil
+				}
+				if pattern == "/dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_*" {
+					return []string{"/dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_vol-root"}, nil
+				}
+				return []string{}, nil
+			}
+
+			err = fakeFS.Symlink("/dev/nvme0n1", "/dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_vol-root")
+			Expect(err).NotTo(HaveOccurred())
+
+			paths, err := resolver.DiscoverInstanceStorage(devices)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(paths).To(Equal([]string{"/dev/nvme1n1", "/dev/nvme2n1"}))
+		})
+	})
+
+	Context("when CPI provides non-NVMe device paths", func() {
+		It("automatically uses identity resolution", func() {
+			resolver = NewAutoDetectingInstanceStorageResolver(
+				fakeFS,
+				fakeDevicePathResolver,
+				logger,
+				"",
+				"",
+			)
+
+			devices := []boshsettings.DiskSettings{
+				{Path: "/dev/xvdba"},
+				{Path: "/dev/xvdbb"},
+			}
+
+			fakeDevicePathResolver.GetRealDevicePathStub = func(diskSettings boshsettings.DiskSettings) (string, bool, error) {
+				return diskSettings.Path, false, nil
+			}
+
+			paths, err := resolver.DiscoverInstanceStorage(devices)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(paths).To(Equal([]string{"/dev/xvdba", "/dev/xvdbb"}))
+		})
+	})
+
+	Context("when device list is empty", func() {
+		It("returns empty list", func() {
+			resolver = NewAutoDetectingInstanceStorageResolver(
+				fakeFS,
+				fakeDevicePathResolver,
+				logger,
+				"",
+				"",
+			)
+
+			paths, err := resolver.DiscoverInstanceStorage([]boshsettings.DiskSettings{})
+			Expect(err).NotTo(HaveOccurred())
+			Expect(paths).To(BeEmpty())
+		})
+	})
+})

infrastructure/devicepathresolver/aws_nvme_instance_storage_resolver.go

@@ -0,0 +1,93 @@
+package devicepathresolver
+
+import (
+	"sort"
+
+	bosherr "github.com/cloudfoundry/bosh-utils/errors"
+	boshlog "github.com/cloudfoundry/bosh-utils/logger"
+	boshsys "github.com/cloudfoundry/bosh-utils/system"
+
+	boshsettings "github.com/cloudfoundry/bosh-agent/v2/settings"
+)
+
+type awsNVMeInstanceStorageResolver struct {
+	fs                 boshsys.FileSystem
+	devicePathResolver DevicePathResolver
+	logger             boshlog.Logger
+	logTag             string
+	ebsSymlinkPattern  string
+	nvmeDevicePattern  string
+}
+
+func NewAWSNVMeInstanceStorageResolver(
+	fs boshsys.FileSystem,
+	devicePathResolver DevicePathResolver,
+	logger boshlog.Logger,
+	ebsSymlinkPattern string,
+	nvmeDevicePattern string,
+) InstanceStorageResolver {
+	if ebsSymlinkPattern == "" {
+		ebsSymlinkPattern = "/dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_*"
+	}
+	if nvmeDevicePattern == "" {
+		nvmeDevicePattern = "/dev/nvme*n1"
+	}
+
+	return &awsNVMeInstanceStorageResolver{
+		fs:                 fs,
+		devicePathResolver: devicePathResolver,
+		logger:             logger,
+		logTag:             "AWSNVMeInstanceStorageResolver",
+		ebsSymlinkPattern:  ebsSymlinkPattern,
+		nvmeDevicePattern:  nvmeDevicePattern,
+	}
+}
+
+func (r *awsNVMeInstanceStorageResolver) DiscoverInstanceStorage(devices []boshsettings.DiskSettings) ([]string, error) {
+	if len(devices) == 0 {
+		return []string{}, nil
+	}
+
+	allNvmeDevices, err := r.fs.Glob(r.nvmeDevicePattern)
+	if err != nil {
+		return nil, bosherr.WrapError(err, "Globbing NVMe devices")
+	}
+
+	r.logger.Debug(r.logTag, "Found NVMe devices: %v", allNvmeDevices)
+
+	ebsSymlinks, err := r.fs.Glob(r.ebsSymlinkPattern)
+	if err != nil {
+		return nil, bosherr.WrapError(err, "Globbing EBS symlinks")
+	}
+
+	ebsDevices := make(map[string]bool)
+	for _, symlink := range ebsSymlinks {
+		absPath, err := r.fs.ReadAndFollowLink(symlink)
+		if err != nil {
+			r.logger.Debug(r.logTag, "Could not resolve symlink %s: %s", symlink, err.Error())
+			continue
+		}
+
+		r.logger.Debug(r.logTag, "EBS volume: %s -> %s", symlink, absPath)
+		ebsDevices[absPath] = true
+	}
+
+	var instanceStorage []string
+	for _, devicePath := range allNvmeDevices {
+		if !ebsDevices[devicePath] {
+			instanceStorage = append(instanceStorage, devicePath)
+			r.logger.Info(r.logTag, "Discovered instance storage: %s", devicePath)
+		} else {
+			r.logger.Debug(r.logTag, "Excluding EBS volume: %s", devicePath)
+		}
+	}
+
+	sort.Strings(instanceStorage)
+
+	if len(instanceStorage) != len(devices) {
+		return nil, bosherr.Errorf("Expected %d instance storage devices but discovered %d: %v",
+			len(devices), len(instanceStorage), instanceStorage)
+	}
+
+	return instanceStorage, nil
+}