diff --git a/agent/bootstrap_test.go b/agent/bootstrap_test.go index e075fd788..e526b9959 100644 --- a/agent/bootstrap_test.go +++ b/agent/bootstrap_test.go @@ -1132,6 +1132,16 @@ var _ = Describe("bootstrap", func() { monitRetryStrategy := boshretry.NewAttemptRetryStrategy(10, 1*time.Second, monitRetryable, logger) devicePathResolver := fakedevicepathresolver.NewFakeDevicePathResolver() + instanceStorageResolver := fakedevicepathresolver.NewFakeInstanceStorageResolver() + + // Default stub: instance storage resolver returns device paths as-is + instanceStorageResolver.DiscoverInstanceStorageStub = func(devices []boshsettings.DiskSettings) ([]string, error) { + paths := make([]string, len(devices)) + for i, device := range devices { + paths[i] = device.Path + } + return paths, nil + } fakeUUIDGenerator := boshuuid.NewGenerator() routesSearcher := boshnet.NewRoutesSearcher(logger, runner, nil) @@ -1153,6 +1163,7 @@ var _ = Describe("bootstrap", func() { ubuntuCertManager, monitRetryStrategy, devicePathResolver, + instanceStorageResolver, state, linuxOptions, logger, diff --git a/infrastructure/devicepathresolver/auto_detecting_instance_storage_resolver.go b/infrastructure/devicepathresolver/auto_detecting_instance_storage_resolver.go new file mode 100644 index 000000000..0c366990e --- /dev/null +++ b/infrastructure/devicepathresolver/auto_detecting_instance_storage_resolver.go @@ -0,0 +1,99 @@ +package devicepathresolver + +import ( + "strings" + + boshlog "github.com/cloudfoundry/bosh-utils/logger" + boshsys "github.com/cloudfoundry/bosh-utils/system" + + boshsettings "github.com/cloudfoundry/bosh-agent/v2/settings" +) + +// autoDetectingInstanceStorageResolver automatically detects whether to use +// NVMe-specific logic or identity resolution based on device paths from the CPI. +// If any device path starts with "/dev/nvme", it uses AWS NVMe discovery logic. +// Otherwise, it uses the CPI-provided paths directly (identity resolution). +type autoDetectingInstanceStorageResolver struct { + fs boshsys.FileSystem + devicePathResolver DevicePathResolver + logger boshlog.Logger + ebsSymlinkPattern string + nvmeDevicePattern string + awsNVMeResolver InstanceStorageResolver + identityResolver InstanceStorageResolver + resolverInitialized bool + useNVMeResolver bool +} + +// NewAutoDetectingInstanceStorageResolver creates a resolver that automatically +// detects NVMe instances based on device paths from the CPI. +func NewAutoDetectingInstanceStorageResolver( + fs boshsys.FileSystem, + devicePathResolver DevicePathResolver, + logger boshlog.Logger, + ebsSymlinkPattern string, + nvmeDevicePattern string, +) InstanceStorageResolver { + if ebsSymlinkPattern == "" { + ebsSymlinkPattern = "/dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_*" + } + if nvmeDevicePattern == "" { + nvmeDevicePattern = "/dev/nvme*n1" + } + + return &autoDetectingInstanceStorageResolver{ + fs: fs, + devicePathResolver: devicePathResolver, + logger: logger, + ebsSymlinkPattern: ebsSymlinkPattern, + nvmeDevicePattern: nvmeDevicePattern, + resolverInitialized: false, + } +} + +func (r *autoDetectingInstanceStorageResolver) DiscoverInstanceStorage(devices []boshsettings.DiskSettings) ([]string, error) { + if len(devices) == 0 { + return []string{}, nil + } + + // Auto-detect on first call by checking if any device path starts with /dev/nvme + if !r.resolverInitialized { + r.useNVMeResolver = r.detectNVMeDevices(devices) + + if r.useNVMeResolver { + r.logger.Info("AutoDetectingInstanceStorageResolver", + "Detected NVMe device paths from CPI - using AWS NVMe instance storage discovery") + r.awsNVMeResolver = NewAWSNVMeInstanceStorageResolver( + r.fs, + r.devicePathResolver, + r.logger, + r.ebsSymlinkPattern, + r.nvmeDevicePattern, + ) + } else { + r.logger.Info("AutoDetectingInstanceStorageResolver", + "Detected non-NVMe device paths from CPI - using identity resolution") + r.identityResolver = NewIdentityInstanceStorageResolver(r.devicePathResolver) + } + + r.resolverInitialized = true + } + + if r.useNVMeResolver { + return r.awsNVMeResolver.DiscoverInstanceStorage(devices) + } + return r.identityResolver.DiscoverInstanceStorage(devices) +} + +// detectNVMeDevices checks if any device path from the CPI starts with /dev/nvme +// This matches the CPI's logic: if current_disk =~ /^\/dev\/nvme/ +func (r *autoDetectingInstanceStorageResolver) detectNVMeDevices(devices []boshsettings.DiskSettings) bool { + for _, device := range devices { + if strings.HasPrefix(device.Path, "/dev/nvme") { + r.logger.Debug("AutoDetectingInstanceStorageResolver", + "Detected NVMe from CPI-provided path: %s", device.Path) + return true + } + } + return false +} diff --git a/infrastructure/devicepathresolver/auto_detecting_instance_storage_resolver_test.go b/infrastructure/devicepathresolver/auto_detecting_instance_storage_resolver_test.go new file mode 100644 index 000000000..ea59fdef0 --- /dev/null +++ b/infrastructure/devicepathresolver/auto_detecting_instance_storage_resolver_test.go @@ -0,0 +1,115 @@ +package devicepathresolver_test + +import ( + "runtime" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + boshlog "github.com/cloudfoundry/bosh-utils/logger" + fakesys "github.com/cloudfoundry/bosh-utils/system/fakes" + + . "github.com/cloudfoundry/bosh-agent/v2/infrastructure/devicepathresolver" + fakedpresolv "github.com/cloudfoundry/bosh-agent/v2/infrastructure/devicepathresolver/fakes" + boshsettings "github.com/cloudfoundry/bosh-agent/v2/settings" +) + +var _ = Describe("AutoDetectingInstanceStorageResolver", func() { + var ( + resolver InstanceStorageResolver + fakeFS *fakesys.FakeFileSystem + fakeDevicePathResolver *fakedpresolv.FakeDevicePathResolver + logger boshlog.Logger + ) + + BeforeEach(func() { + if runtime.GOOS != "linux" { + Skip("Only supported on Linux") + } + fakeFS = fakesys.NewFakeFileSystem() + fakeDevicePathResolver = fakedpresolv.NewFakeDevicePathResolver() + logger = boshlog.NewLogger(boshlog.LevelNone) + }) + + Context("when CPI provides NVMe device paths", func() { + It("automatically uses AWS NVMe instance storage discovery", func() { + resolver = NewAutoDetectingInstanceStorageResolver( + fakeFS, + fakeDevicePathResolver, + logger, + "/dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_*", + "/dev/nvme*n1", + ) + + devices := []boshsettings.DiskSettings{ + {Path: "/dev/nvme0n1"}, + {Path: "/dev/nvme1n1"}, + } + + err := fakeFS.WriteFileString("/dev/nvme0n1", "") + Expect(err).NotTo(HaveOccurred()) + err = fakeFS.WriteFileString("/dev/nvme1n1", "") + Expect(err).NotTo(HaveOccurred()) + err = fakeFS.WriteFileString("/dev/nvme2n1", "") + Expect(err).NotTo(HaveOccurred()) + + fakeFS.GlobStub = func(pattern string) ([]string, error) { + if pattern == "/dev/nvme*n1" { + return []string{"/dev/nvme0n1", "/dev/nvme1n1", "/dev/nvme2n1"}, nil + } + if pattern == "/dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_*" { + return []string{"/dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_vol-root"}, nil + } + return []string{}, nil + } + + err = fakeFS.Symlink("/dev/nvme0n1", "/dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_vol-root") + Expect(err).NotTo(HaveOccurred()) + + paths, err := resolver.DiscoverInstanceStorage(devices) + Expect(err).NotTo(HaveOccurred()) + Expect(paths).To(Equal([]string{"/dev/nvme1n1", "/dev/nvme2n1"})) + }) + }) + + Context("when CPI provides non-NVMe device paths", func() { + It("automatically uses identity resolution", func() { + resolver = NewAutoDetectingInstanceStorageResolver( + fakeFS, + fakeDevicePathResolver, + logger, + "", + "", + ) + + devices := []boshsettings.DiskSettings{ + {Path: "/dev/xvdba"}, + {Path: "/dev/xvdbb"}, + } + + fakeDevicePathResolver.GetRealDevicePathStub = func(diskSettings boshsettings.DiskSettings) (string, bool, error) { + return diskSettings.Path, false, nil + } + + paths, err := resolver.DiscoverInstanceStorage(devices) + Expect(err).NotTo(HaveOccurred()) + Expect(paths).To(Equal([]string{"/dev/xvdba", "/dev/xvdbb"})) + }) + }) + + Context("when device list is empty", func() { + It("returns empty list", func() { + resolver = NewAutoDetectingInstanceStorageResolver( + fakeFS, + fakeDevicePathResolver, + logger, + "", + "", + ) + + paths, err := resolver.DiscoverInstanceStorage([]boshsettings.DiskSettings{}) + Expect(err).NotTo(HaveOccurred()) + Expect(paths).To(BeEmpty()) + }) + }) +}) diff --git a/infrastructure/devicepathresolver/aws_nvme_instance_storage_resolver.go b/infrastructure/devicepathresolver/aws_nvme_instance_storage_resolver.go new file mode 100644 index 000000000..fd73a5610 --- /dev/null +++ b/infrastructure/devicepathresolver/aws_nvme_instance_storage_resolver.go @@ -0,0 +1,93 @@ +package devicepathresolver + +import ( + "sort" + + bosherr "github.com/cloudfoundry/bosh-utils/errors" + boshlog "github.com/cloudfoundry/bosh-utils/logger" + boshsys "github.com/cloudfoundry/bosh-utils/system" + + boshsettings "github.com/cloudfoundry/bosh-agent/v2/settings" +) + +type awsNVMeInstanceStorageResolver struct { + fs boshsys.FileSystem + devicePathResolver DevicePathResolver + logger boshlog.Logger + logTag string + ebsSymlinkPattern string + nvmeDevicePattern string +} + +func NewAWSNVMeInstanceStorageResolver( + fs boshsys.FileSystem, + devicePathResolver DevicePathResolver, + logger boshlog.Logger, + ebsSymlinkPattern string, + nvmeDevicePattern string, +) InstanceStorageResolver { + if ebsSymlinkPattern == "" { + ebsSymlinkPattern = "/dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_*" + } + if nvmeDevicePattern == "" { + nvmeDevicePattern = "/dev/nvme*n1" + } + + return &awsNVMeInstanceStorageResolver{ + fs: fs, + devicePathResolver: devicePathResolver, + logger: logger, + logTag: "AWSNVMeInstanceStorageResolver", + ebsSymlinkPattern: ebsSymlinkPattern, + nvmeDevicePattern: nvmeDevicePattern, + } +} + +func (r *awsNVMeInstanceStorageResolver) DiscoverInstanceStorage(devices []boshsettings.DiskSettings) ([]string, error) { + if len(devices) == 0 { + return []string{}, nil + } + + allNvmeDevices, err := r.fs.Glob(r.nvmeDevicePattern) + if err != nil { + return nil, bosherr.WrapError(err, "Globbing NVMe devices") + } + + r.logger.Debug(r.logTag, "Found NVMe devices: %v", allNvmeDevices) + + ebsSymlinks, err := r.fs.Glob(r.ebsSymlinkPattern) + if err != nil { + return nil, bosherr.WrapError(err, "Globbing EBS symlinks") + } + + ebsDevices := make(map[string]bool) + for _, symlink := range ebsSymlinks { + absPath, err := r.fs.ReadAndFollowLink(symlink) + if err != nil { + r.logger.Debug(r.logTag, "Could not resolve symlink %s: %s", symlink, err.Error()) + continue + } + + r.logger.Debug(r.logTag, "EBS volume: %s -> %s", symlink, absPath) + ebsDevices[absPath] = true + } + + var instanceStorage []string + for _, devicePath := range allNvmeDevices { + if !ebsDevices[devicePath] { + instanceStorage = append(instanceStorage, devicePath) + r.logger.Info(r.logTag, "Discovered instance storage: %s", devicePath) + } else { + r.logger.Debug(r.logTag, "Excluding EBS volume: %s", devicePath) + } + } + + sort.Strings(instanceStorage) + + if len(instanceStorage) != len(devices) { + return nil, bosherr.Errorf("Expected %d instance storage devices but discovered %d: %v", + len(devices), len(instanceStorage), instanceStorage) + } + + return instanceStorage, nil +} diff --git a/infrastructure/devicepathresolver/aws_nvme_instance_storage_resolver_test.go b/infrastructure/devicepathresolver/aws_nvme_instance_storage_resolver_test.go new file mode 100644 index 000000000..047ab0f4f --- /dev/null +++ b/infrastructure/devicepathresolver/aws_nvme_instance_storage_resolver_test.go @@ -0,0 +1,124 @@ +package devicepathresolver_test + +import ( + "runtime" + + boshlog "github.com/cloudfoundry/bosh-utils/logger" + fakesys "github.com/cloudfoundry/bosh-utils/system/fakes" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + . "github.com/cloudfoundry/bosh-agent/v2/infrastructure/devicepathresolver" + fakedpresolv "github.com/cloudfoundry/bosh-agent/v2/infrastructure/devicepathresolver/fakes" + boshsettings "github.com/cloudfoundry/bosh-agent/v2/settings" +) + +var _ = Describe("AWSNVMeInstanceStorageResolver", func() { + var ( + resolver InstanceStorageResolver + fakeFS *fakesys.FakeFileSystem + fakeDevicePathResolver *fakedpresolv.FakeDevicePathResolver + logger boshlog.Logger + ) + BeforeEach(func() { + if runtime.GOOS != "linux" { + Skip("Only supported on Linux") + } + fakeFS = fakesys.NewFakeFileSystem() + fakeDevicePathResolver = fakedpresolv.NewFakeDevicePathResolver() + logger = boshlog.NewLogger(boshlog.LevelNone) + resolver = NewAWSNVMeInstanceStorageResolver(fakeFS, fakeDevicePathResolver, logger, "", "") + }) + Describe("DiscoverInstanceStorage", func() { + Context("when devices are NVMe", func() { + It("discovers instance storage by filtering out EBS volumes", func() { + devices := []boshsettings.DiskSettings{ + {Path: "/dev/nvme0n1"}, + {Path: "/dev/nvme1n1"}, + } + + // Create device files + err := fakeFS.WriteFileString("/dev/nvme0n1", "") + Expect(err).NotTo(HaveOccurred()) + err = fakeFS.WriteFileString("/dev/nvme1n1", "") + Expect(err).NotTo(HaveOccurred()) + err = fakeFS.WriteFileString("/dev/nvme2n1", "") + Expect(err).NotTo(HaveOccurred()) + + fakeFS.GlobStub = func(pattern string) ([]string, error) { + if pattern == "/dev/nvme*n1" { + return []string{"/dev/nvme0n1", "/dev/nvme1n1", "/dev/nvme2n1"}, nil + } + if pattern == "/dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_*" { + return []string{"/dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_vol-root"}, nil + } + return nil, nil + } + + err = fakeFS.Symlink("/dev/nvme0n1", "/dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_vol-root") + Expect(err).NotTo(HaveOccurred()) + + paths, err := resolver.DiscoverInstanceStorage(devices) + Expect(err).NotTo(HaveOccurred()) + Expect(paths).To(Equal([]string{"/dev/nvme1n1", "/dev/nvme2n1"})) + }) + It("returns error if not enough instance storage devices found", func() { + devices := []boshsettings.DiskSettings{ + {Path: "/dev/nvme0n1"}, + {Path: "/dev/nvme1n1"}, + {Path: "/dev/nvme2n1"}, + } + + // Create device files + err := fakeFS.WriteFileString("/dev/nvme0n1", "") + Expect(err).NotTo(HaveOccurred()) + err = fakeFS.WriteFileString("/dev/nvme1n1", "") + Expect(err).NotTo(HaveOccurred()) + + fakeFS.GlobStub = func(pattern string) ([]string, error) { + if pattern == "/dev/nvme*n1" { + return []string{"/dev/nvme0n1", "/dev/nvme1n1"}, nil + } + if pattern == "/dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_*" { + return []string{"/dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_vol-root"}, nil + } + return nil, nil + } + + err = fakeFS.Symlink("/dev/nvme0n1", "/dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_vol-root") + Expect(err).NotTo(HaveOccurred()) + + _, err = resolver.DiscoverInstanceStorage(devices) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("Expected 3 instance storage devices but discovered 1")) + }) + It("returns error if too many instance storage devices found", func() { + devices := []boshsettings.DiskSettings{ + {Path: "/dev/nvme0n1"}, + {Path: "/dev/nvme1n1"}, + } + + // Create device files + err := fakeFS.WriteFileString("/dev/nvme0n1", "") + Expect(err).NotTo(HaveOccurred()) + err = fakeFS.WriteFileString("/dev/nvme1n1", "") + Expect(err).NotTo(HaveOccurred()) + err = fakeFS.WriteFileString("/dev/nvme2n1", "") + Expect(err).NotTo(HaveOccurred()) + + fakeFS.GlobStub = func(pattern string) ([]string, error) { + if pattern == "/dev/nvme*n1" { + return []string{"/dev/nvme0n1", "/dev/nvme1n1", "/dev/nvme2n1"}, nil + } + // No EBS symlinks - all devices are instance storage + return nil, nil + } + + // No symlinks to filter out - all 3 devices will be returned as instance storage + _, err = resolver.DiscoverInstanceStorage(devices) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("Expected 2 instance storage devices but discovered 3")) + }) + }) + }) +}) diff --git a/infrastructure/devicepathresolver/fakes/fake_device_path_resolver.go b/infrastructure/devicepathresolver/fakes/fake_device_path_resolver.go index 5b0426e69..7ed054d26 100644 --- a/infrastructure/devicepathresolver/fakes/fake_device_path_resolver.go +++ b/infrastructure/devicepathresolver/fakes/fake_device_path_resolver.go @@ -5,11 +5,15 @@ import ( ) type FakeDevicePathResolver struct { - GetRealDevicePathDiskSettings boshsettings.DiskSettings + GetRealDevicePathDiskSettings []boshsettings.DiskSettings RealDevicePath string GetRealDevicePathStub func(boshsettings.DiskSettings) (string, bool, error) GetRealDevicePathTimedOut bool GetRealDevicePathErr error + GetRealDevicePathCallCount_ int + GetRealDevicePathReturnsPath string + GetRealDevicePathReturnsTO bool + GetRealDevicePathReturnsErr error } func NewFakeDevicePathResolver() *FakeDevicePathResolver { @@ -17,15 +21,30 @@ func NewFakeDevicePathResolver() *FakeDevicePathResolver { } func (r *FakeDevicePathResolver) GetRealDevicePath(diskSettings boshsettings.DiskSettings) (string, bool, error) { - r.GetRealDevicePathDiskSettings = diskSettings - - if r.GetRealDevicePathErr != nil { - return "", r.GetRealDevicePathTimedOut, r.GetRealDevicePathErr - } + r.GetRealDevicePathDiskSettings = append(r.GetRealDevicePathDiskSettings, diskSettings) + r.GetRealDevicePathCallCount_++ if r.GetRealDevicePathStub != nil { return r.GetRealDevicePathStub(diskSettings) } + if r.GetRealDevicePathReturnsErr != nil { + return r.GetRealDevicePathReturnsPath, r.GetRealDevicePathReturnsTO, r.GetRealDevicePathReturnsErr + } + + if r.GetRealDevicePathErr != nil { + return "", r.GetRealDevicePathTimedOut, r.GetRealDevicePathErr + } + return r.RealDevicePath, false, nil } + +func (r *FakeDevicePathResolver) GetRealDevicePathCallCount() int { + return r.GetRealDevicePathCallCount_ +} + +func (r *FakeDevicePathResolver) GetRealDevicePathReturns(path string, timedOut bool, err error) { + r.GetRealDevicePathReturnsPath = path + r.GetRealDevicePathReturnsTO = timedOut + r.GetRealDevicePathReturnsErr = err +} diff --git a/infrastructure/devicepathresolver/fakes/fake_instance_storage_resolver.go b/infrastructure/devicepathresolver/fakes/fake_instance_storage_resolver.go new file mode 100644 index 000000000..a816e0bdd --- /dev/null +++ b/infrastructure/devicepathresolver/fakes/fake_instance_storage_resolver.go @@ -0,0 +1,32 @@ +package fakes + +import ( + boshsettings "github.com/cloudfoundry/bosh-agent/v2/settings" +) + +type FakeInstanceStorageResolver struct { + DiscoverInstanceStorageDevices []boshsettings.DiskSettings + DiscoverInstanceStoragePaths []string + DiscoverInstanceStorageErr error + DiscoverInstanceStorageCallCount int + DiscoverInstanceStorageStub func([]boshsettings.DiskSettings) ([]string, error) +} + +func NewFakeInstanceStorageResolver() *FakeInstanceStorageResolver { + return &FakeInstanceStorageResolver{} +} + +func (r *FakeInstanceStorageResolver) DiscoverInstanceStorage(devices []boshsettings.DiskSettings) ([]string, error) { + r.DiscoverInstanceStorageDevices = devices + r.DiscoverInstanceStorageCallCount++ + + if r.DiscoverInstanceStorageStub != nil { + return r.DiscoverInstanceStorageStub(devices) + } + + if r.DiscoverInstanceStorageErr != nil { + return nil, r.DiscoverInstanceStorageErr + } + + return r.DiscoverInstanceStoragePaths, nil +} diff --git a/infrastructure/devicepathresolver/identity_instance_storage_resolver.go b/infrastructure/devicepathresolver/identity_instance_storage_resolver.go new file mode 100644 index 000000000..15bc1f4a5 --- /dev/null +++ b/infrastructure/devicepathresolver/identity_instance_storage_resolver.go @@ -0,0 +1,31 @@ +package devicepathresolver + +import ( + bosherr "github.com/cloudfoundry/bosh-utils/errors" + + boshsettings "github.com/cloudfoundry/bosh-agent/v2/settings" +) + +// identityInstanceStorageResolver returns device paths as-is from the CPI +type identityInstanceStorageResolver struct { + devicePathResolver DevicePathResolver +} + +// NewIdentityInstanceStorageResolver creates a resolver that uses CPI-provided paths directly +func NewIdentityInstanceStorageResolver(devicePathResolver DevicePathResolver) InstanceStorageResolver { + return &identityInstanceStorageResolver{ + devicePathResolver: devicePathResolver, + } +} + +func (r *identityInstanceStorageResolver) DiscoverInstanceStorage(devices []boshsettings.DiskSettings) ([]string, error) { + paths := make([]string, len(devices)) + for i, device := range devices { + realPath, _, err := r.devicePathResolver.GetRealDevicePath(device) + if err != nil { + return nil, bosherr.WrapErrorf(err, "Getting device %s path", device) + } + paths[i] = realPath + } + return paths, nil +} diff --git a/infrastructure/devicepathresolver/identity_instance_storage_resolver_test.go b/infrastructure/devicepathresolver/identity_instance_storage_resolver_test.go new file mode 100644 index 000000000..ba050543c --- /dev/null +++ b/infrastructure/devicepathresolver/identity_instance_storage_resolver_test.go @@ -0,0 +1,62 @@ +package devicepathresolver_test + +import ( + "errors" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + . "github.com/cloudfoundry/bosh-agent/v2/infrastructure/devicepathresolver" + fakedpresolv "github.com/cloudfoundry/bosh-agent/v2/infrastructure/devicepathresolver/fakes" + boshsettings "github.com/cloudfoundry/bosh-agent/v2/settings" +) + +var _ = Describe("IdentityInstanceStorageResolver", func() { + var ( + resolver InstanceStorageResolver + fakeDevicePathResolver *fakedpresolv.FakeDevicePathResolver + ) + + BeforeEach(func() { + fakeDevicePathResolver = fakedpresolv.NewFakeDevicePathResolver() + resolver = NewIdentityInstanceStorageResolver(fakeDevicePathResolver) + }) + + Describe("DiscoverInstanceStorage", func() { + It("returns device paths resolved by the underlying device path resolver", func() { + devices := []boshsettings.DiskSettings{ + {Path: "/dev/xvdb"}, + {Path: "/dev/xvdc"}, + } + + fakeDevicePathResolver.GetRealDevicePathStub = func(diskSettings boshsettings.DiskSettings) (string, bool, error) { + return diskSettings.Path, false, nil + } + + paths, err := resolver.DiscoverInstanceStorage(devices) + Expect(err).NotTo(HaveOccurred()) + Expect(paths).To(Equal([]string{"/dev/xvdb", "/dev/xvdc"})) + Expect(fakeDevicePathResolver.GetRealDevicePathCallCount()).To(Equal(2)) + }) + + It("returns error if device path resolver fails", func() { + devices := []boshsettings.DiskSettings{ + {Path: "/dev/xvdb"}, + } + + fakeDevicePathResolver.GetRealDevicePathReturns("", false, errors.New("fake-error")) + + _, err := resolver.DiscoverInstanceStorage(devices) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("fake-error")) + }) + + It("returns empty slice for empty input", func() { + devices := []boshsettings.DiskSettings{} + + paths, err := resolver.DiscoverInstanceStorage(devices) + Expect(err).NotTo(HaveOccurred()) + Expect(paths).To(Equal([]string{})) + }) + }) +}) diff --git a/infrastructure/devicepathresolver/instance_storage_resolver.go b/infrastructure/devicepathresolver/instance_storage_resolver.go new file mode 100644 index 000000000..b33888df5 --- /dev/null +++ b/infrastructure/devicepathresolver/instance_storage_resolver.go @@ -0,0 +1,13 @@ +package devicepathresolver + +import ( + boshsettings "github.com/cloudfoundry/bosh-agent/v2/settings" +) + +// InstanceStorageResolver discovers instance storage devices, filtering out +// IaaS-managed volumes like EBS, persistent disks, etc. +type InstanceStorageResolver interface { + // DiscoverInstanceStorage takes a list of expected ephemeral disks and returns + // the actual device paths for instance storage, excluding IaaS-managed volumes + DiscoverInstanceStorage(devices []boshsettings.DiskSettings) ([]string, error) +} diff --git a/infrastructure/devicepathresolver/scsi_device_path_resolver_test.go b/infrastructure/devicepathresolver/scsi_device_path_resolver_test.go index b23a89610..3746e3788 100644 --- a/infrastructure/devicepathresolver/scsi_device_path_resolver_test.go +++ b/infrastructure/devicepathresolver/scsi_device_path_resolver_test.go @@ -42,7 +42,7 @@ var _ = Describe("scsiDevicePathResolver", func() { Expect(timeout).To(BeFalse()) Expect(realPath).To(Equal("fake-id-resolved-device-path")) - Expect(scsiIDDevicePathResolver.GetRealDevicePathDiskSettings).To(Equal(diskSettings)) + Expect(scsiIDDevicePathResolver.GetRealDevicePathDiskSettings).To(ContainElement(diskSettings)) }) }) @@ -60,7 +60,7 @@ var _ = Describe("scsiDevicePathResolver", func() { Expect(timeout).To(BeFalse()) Expect(realPath).To(Equal("fake-volume-id-resolved-device-path")) - Expect(scsiVolumeIDDevicePathResolver.GetRealDevicePathDiskSettings).To(Equal(diskSettings)) + Expect(scsiVolumeIDDevicePathResolver.GetRealDevicePathDiskSettings).To(ContainElement(diskSettings)) }) }) @@ -80,7 +80,7 @@ var _ = Describe("scsiDevicePathResolver", func() { Expect(timeout).To(BeFalse()) Expect(realPath).To(Equal("fake-lun-resolved-device-path")) - Expect(scsiLunDevicePathResolver.GetRealDevicePathDiskSettings).To(Equal(diskSettings)) + Expect(scsiLunDevicePathResolver.GetRealDevicePathDiskSettings).To(ContainElement(diskSettings)) }) }) diff --git a/infrastructure/devicepathresolver/virtio_device_path_resolver_test.go b/infrastructure/devicepathresolver/virtio_device_path_resolver_test.go index b7873c916..142a695a8 100644 --- a/infrastructure/devicepathresolver/virtio_device_path_resolver_test.go +++ b/infrastructure/devicepathresolver/virtio_device_path_resolver_test.go @@ -63,7 +63,7 @@ var _ = Describe("VirtioDevicePathResolver", func() { Expect(timeout).To(BeFalse()) Expect(realPath).To(Equal("fake-mapped-resolved-device-path")) - Expect(mappedDevicePathResolver.GetRealDevicePathDiskSettings).To(Equal(diskSettings)) + Expect(mappedDevicePathResolver.GetRealDevicePathDiskSettings).To(ContainElement(diskSettings)) }) Context("when mappedDevicePathResolver times out", func() { diff --git a/platform/linux_platform.go b/platform/linux_platform.go index 2bb056173..6f0ff917c 100644 --- a/platform/linux_platform.go +++ b/platform/linux_platform.go @@ -77,6 +77,18 @@ type LinuxOptions struct { // possible values: virtio, scsi, iscsi, "" DevicePathResolutionType string + // Strategy for discovering instance storage devices; + // possible values: aws-nvme, "" + InstanceStorageResolutionType string + + // Pattern for identifying IaaS-managed volumes (e.g., EBS on AWS) + // Used with InstanceStorageResolutionType to filter out non-instance storage + InstanceStorageManagedVolumePattern string + + // Pattern for discovering all potential instance storage devices + // Used with InstanceStorageResolutionType for device enumeration + InstanceStorageDevicePattern string + // Strategy for resolving ephemeral & persistent disk partitioners; // possible values: parted, "" (default is sfdisk if disk < 2TB, parted otherwise) PartitionerType string @@ -95,27 +107,28 @@ type LinuxOptions struct { } type linux struct { - fs boshsys.FileSystem - cmdRunner boshsys.CmdRunner - collector boshstats.Collector - compressor boshcmd.Compressor - copier boshcmd.Copier - dirProvider boshdirs.Provider - vitalsService boshvitals.Service - cdutil cdrom.CDUtil - diskManager boshdisk.Manager - netManager boshnet.Manager - certManager boshcert.Manager - monitRetryStrategy boshretry.RetryStrategy - devicePathResolver boshdpresolv.DevicePathResolver - options LinuxOptions - state *BootstrapState - logger boshlog.Logger - defaultNetworkResolver boshsettings.DefaultNetworkResolver - uuidGenerator boshuuid.Generator - auditLogger AuditLogger - logsTarProvider boshlogstarprovider.LogsTarProvider - serviceManager servicemanager.ServiceManager + fs boshsys.FileSystem + cmdRunner boshsys.CmdRunner + collector boshstats.Collector + compressor boshcmd.Compressor + copier boshcmd.Copier + dirProvider boshdirs.Provider + vitalsService boshvitals.Service + cdutil cdrom.CDUtil + diskManager boshdisk.Manager + netManager boshnet.Manager + certManager boshcert.Manager + monitRetryStrategy boshretry.RetryStrategy + devicePathResolver boshdpresolv.DevicePathResolver + instanceStorageResolver boshdpresolv.InstanceStorageResolver + options LinuxOptions + state *BootstrapState + logger boshlog.Logger + defaultNetworkResolver boshsettings.DefaultNetworkResolver + uuidGenerator boshuuid.Generator + auditLogger AuditLogger + logsTarProvider boshlogstarprovider.LogsTarProvider + serviceManager servicemanager.ServiceManager } func NewLinuxPlatform( @@ -132,6 +145,7 @@ func NewLinuxPlatform( certManager boshcert.Manager, monitRetryStrategy boshretry.RetryStrategy, devicePathResolver boshdpresolv.DevicePathResolver, + instanceStorageResolver boshdpresolv.InstanceStorageResolver, state *BootstrapState, options LinuxOptions, logger boshlog.Logger, @@ -142,27 +156,28 @@ func NewLinuxPlatform( serviceManager servicemanager.ServiceManager, ) Platform { return &linux{ - fs: fs, - cmdRunner: cmdRunner, - collector: collector, - compressor: compressor, - copier: copier, - dirProvider: dirProvider, - vitalsService: vitalsService, - cdutil: cdutil, - diskManager: diskManager, - netManager: netManager, - certManager: certManager, - monitRetryStrategy: monitRetryStrategy, - devicePathResolver: devicePathResolver, - state: state, - options: options, - logger: logger, - defaultNetworkResolver: defaultNetworkResolver, - uuidGenerator: uuidGenerator, - auditLogger: auditLogger, - logsTarProvider: logsTarProvider, - serviceManager: serviceManager, + fs: fs, + cmdRunner: cmdRunner, + collector: collector, + compressor: compressor, + copier: copier, + dirProvider: dirProvider, + vitalsService: vitalsService, + cdutil: cdutil, + diskManager: diskManager, + netManager: netManager, + certManager: certManager, + monitRetryStrategy: monitRetryStrategy, + devicePathResolver: devicePathResolver, + instanceStorageResolver: instanceStorageResolver, + state: state, + options: options, + logger: logger, + defaultNetworkResolver: defaultNetworkResolver, + uuidGenerator: uuidGenerator, + auditLogger: auditLogger, + logsTarProvider: logsTarProvider, + serviceManager: serviceManager, } } @@ -728,19 +743,24 @@ func (p linux) SetupRawEphemeralDisks(devices []boshsettings.DiskSettings) (err return nil } - p.logger.Info(logTag, "Setting up raw ephemeral disks") + if len(devices) == 0 { + return nil + } + + p.logger.Info(logTag, "Setting up %d raw ephemeral disk(s)", len(devices)) - for i, device := range devices { - realPath, _, err := p.devicePathResolver.GetRealDevicePath(device) - if err != nil { - return bosherr.WrapError(err, "Getting real device path") - } + instanceStorageDevices, err := p.instanceStorageResolver.DiscoverInstanceStorage(devices) + if err != nil { + return bosherr.WrapError(err, "Discovering instance storage devices") + } + // Partition each discovered device + for i, devicePath := range instanceStorageDevices { // check if device is already partitioned correctly stdout, stderr, _, err := p.cmdRunner.RunCommand( "parted", "-s", - realPath, + devicePath, "p", ) @@ -748,21 +768,22 @@ func (p linux) SetupRawEphemeralDisks(devices []boshsettings.DiskSettings) (err // "unrecognised disk label" is acceptable, since the disk may not have been partitioned if !strings.Contains(stdout, "unrecognised disk label") && !strings.Contains(stderr, "unrecognised disk label") { - return bosherr.WrapError(err, "Setting up raw ephemeral disks") + return bosherr.WrapErrorf(err, "Checking partition on %s", devicePath) } } if strings.Contains(stdout, "Partition Table: gpt") && strings.Contains(stdout, "raw-ephemeral-") { + p.logger.Info(logTag, "Device %s already partitioned, skipping", devicePath) continue } // change to gpt partition type, change units to percentage, make partition with name and span from 0-100% - p.logger.Info(logTag, "Creating partition on `%s'", realPath) + p.logger.Info(logTag, "Creating partition on %s as raw-ephemeral-%d", devicePath, i) _, _, _, err = p.cmdRunner.RunCommand( "parted", "-s", - realPath, + devicePath, "mklabel", "gpt", "unit", @@ -774,7 +795,7 @@ func (p linux) SetupRawEphemeralDisks(devices []boshsettings.DiskSettings) (err ) if err != nil { - return bosherr.WrapError(err, "Setting up raw ephemeral disks") + return bosherr.WrapErrorf(err, "Creating partition on %s", devicePath) } } diff --git a/platform/linux_platform_test.go b/platform/linux_platform_test.go index 892493a39..b72cb0268 100644 --- a/platform/linux_platform_test.go +++ b/platform/linux_platform_test.go @@ -47,6 +47,7 @@ var _ = Describe("LinuxPlatform", func() { diskManager *diskfakes.FakeManager dirProvider boshdirs.Provider devicePathResolver *fakedpresolv.FakeDevicePathResolver + instanceStorageResolver *fakedpresolv.FakeInstanceStorageResolver platform Platform cdutil *fakecdrom.FakeCDUtil compressor boshcmd.Compressor @@ -89,6 +90,17 @@ var _ = Describe("LinuxPlatform", func() { certManager = new(certfakes.FakeManager) monitRetryStrategy = fakeretry.NewFakeRetryStrategy() devicePathResolver = fakedpresolv.NewFakeDevicePathResolver() + instanceStorageResolver = fakedpresolv.NewFakeInstanceStorageResolver() + + // Default: instance storage resolver returns device paths as-is (identity resolution) + instanceStorageResolver.DiscoverInstanceStorageStub = func(devices []boshsettings.DiskSettings) ([]string, error) { + paths := make([]string, len(devices)) + for i, device := range devices { + paths[i] = device.Path + } + return paths, nil + } + fakeDefaultNetworkResolver = &fakenet.FakeDefaultNetworkResolver{} serviceManager = &servicemanagerfakes.FakeServiceManager{} @@ -148,6 +160,7 @@ var _ = Describe("LinuxPlatform", func() { certManager, monitRetryStrategy, devicePathResolver, + instanceStorageResolver, state, options, logger, @@ -466,6 +479,7 @@ bosh_foobar:...` certManager, monitRetryStrategy, devicePathResolver, + instanceStorageResolver, state, options, logger, @@ -703,6 +717,7 @@ bosh_foobar:...` certManager, monitRetryStrategy, devicePathResolver, + instanceStorageResolver, state, options, logger, @@ -1849,6 +1864,148 @@ Number Start End Size File system Name Flags Expect(len(cmdRunner.RunCommands)).To(Equal(0)) }) }) + + Context("NVMe instance storage discovery", func() { + BeforeEach(func() { + devicePathResolver.GetRealDevicePathStub = func(diskSettings boshsettings.DiskSettings) (string, bool, error) { + return diskSettings.Path, false, nil + } + }) + + It("discovers instance storage by excluding EBS volumes via symlinks", func() { + // Setup: 3 NVMe devices, 2 are EBS (nvme0n1, nvme1n1), 1 is instance storage (nvme2n1) + fs.GlobStub = func(pattern string) ([]string, error) { + switch pattern { + case "/dev/nvme*n1": + return []string{"/dev/nvme0n1", "/dev/nvme1n1", "/dev/nvme2n1"}, nil + case "/dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_*": + return []string{ + "/dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_vol123", + "/dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_vol456", + }, nil + default: + return nil, nil + } + } + + // Create the NVMe device files + err := fs.WriteFileString("/dev/nvme0n1", "") + Expect(err).ToNot(HaveOccurred()) + err = fs.WriteFileString("/dev/nvme1n1", "") + Expect(err).ToNot(HaveOccurred()) + err = fs.WriteFileString("/dev/nvme2n1", "") + Expect(err).ToNot(HaveOccurred()) + + // Create symlinks for EBS volumes + err = fs.Symlink("/dev/nvme0n1", "/dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_vol123") + Expect(err).ToNot(HaveOccurred()) + err = fs.Symlink("/dev/nvme1n1", "/dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_vol456") + Expect(err).ToNot(HaveOccurred()) + + // Configure instance storage resolver to simulate NVMe filtering + // Returns only nvme2n1 (the instance storage device, after filtering EBS) + instanceStorageResolver.DiscoverInstanceStorageStub = func(devices []boshsettings.DiskSettings) ([]string, error) { + return []string{"/dev/nvme2n1"}, nil + } + + // Mock parted output for nvme2n1 (instance storage - needs partitioning) + cmdRunner.AddCmdResult("parted -s /dev/nvme2n1 p", fakesys.FakeCmdResult{ + Error: errors.New("unrecognised disk label"), + Stdout: "Error: /dev/nvme2n1: unrecognised disk label", + }) + + err = platform.SetupRawEphemeralDisks([]boshsettings.DiskSettings{{Path: "/dev/nvme0n1"}}) + + Expect(err).ToNot(HaveOccurred()) + Expect(len(cmdRunner.RunCommands)).To(Equal(2)) + Expect(cmdRunner.RunCommands[0]).To(Equal([]string{"parted", "-s", "/dev/nvme2n1", "p"})) + Expect(cmdRunner.RunCommands[1]).To(Equal([]string{"parted", "-s", "/dev/nvme2n1", "mklabel", "gpt", "unit", "%", "mkpart", "raw-ephemeral-0", "0", "100"})) + }) + + It("returns error when no instance storage devices found but CPI expects some", func() { + // Configure instance storage resolver to return error + instanceStorageResolver.DiscoverInstanceStorageStub = func(devices []boshsettings.DiskSettings) ([]string, error) { + return nil, errors.New("Expected 1 instance storage devices but discovered 0") + } + + err := platform.SetupRawEphemeralDisks([]boshsettings.DiskSettings{{Path: "/dev/nvme2n1"}}) + + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("Expected 1 instance storage devices but discovered 0")) + }) + + It("returns error when globbing NVMe devices fails", func() { + // Configure instance storage resolver to return error + instanceStorageResolver.DiscoverInstanceStorageStub = func(devices []boshsettings.DiskSettings) ([]string, error) { + return nil, errors.New("Globbing NVMe devices: permission denied reading /dev") + } + + err := platform.SetupRawEphemeralDisks([]boshsettings.DiskSettings{{Path: "/dev/nvme1n1"}}) + + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("Globbing NVMe devices")) + }) + + It("returns error when globbing EBS symlinks fails", func() { + // Configure instance storage resolver to return error + instanceStorageResolver.DiscoverInstanceStorageStub = func(devices []boshsettings.DiskSettings) ([]string, error) { + return nil, errors.New("Globbing EBS symlinks: permission denied") + } + + err := platform.SetupRawEphemeralDisks([]boshsettings.DiskSettings{{Path: "/dev/nvme1n1"}}) + + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("Globbing EBS symlinks")) + }) + + It("skips symlinks that fail to resolve and continues", func() { + // Configure instance storage resolver to return nvme1n1 and nvme2n1 + // (simulating that broken symlinks are skipped and only nvme0n1 is EBS) + instanceStorageResolver.DiscoverInstanceStorageStub = func(devices []boshsettings.DiskSettings) ([]string, error) { + return []string{"/dev/nvme1n1", "/dev/nvme2n1"}, nil + } + + // Mock parted for nvme1n1 and nvme2n1 (instance storage) + cmdRunner.AddCmdResult("parted -s /dev/nvme1n1 p", fakesys.FakeCmdResult{ + Error: errors.New("unrecognised disk label"), + Stdout: "Error: /dev/nvme1n1: unrecognised disk label", + }) + cmdRunner.AddCmdResult("parted -s /dev/nvme2n1 p", fakesys.FakeCmdResult{ + Error: errors.New("unrecognised disk label"), + Stdout: "Error: /dev/nvme2n1: unrecognised disk label", + }) + + err := platform.SetupRawEphemeralDisks([]boshsettings.DiskSettings{{Path: "/dev/nvme0n1"}, {Path: "/dev/nvme1n1"}}) + + Expect(err).ToNot(HaveOccurred()) + // Should partition nvme1n1 and nvme2n1 (only nvme0n1 was identified as EBS) + Expect(len(cmdRunner.RunCommands)).To(Equal(4)) + }) + + It("uses CPI paths directly for non-NVMe devices", func() { + // For non-NVMe devices, the instance storage resolver returns paths as-is + // (this is the default stub behavior, but be explicit) + instanceStorageResolver.DiscoverInstanceStorageStub = func(devices []boshsettings.DiskSettings) ([]string, error) { + paths := make([]string, len(devices)) + for i, d := range devices { + paths[i] = d.Path + } + return paths, nil + } + + cmdRunner.AddCmdResult("parted -s /dev/xvdb p", fakesys.FakeCmdResult{ + Error: errors.New("unrecognised disk label"), + Stdout: "Error: /dev/xvdb: unrecognised disk label", + }) + + err := platform.SetupRawEphemeralDisks([]boshsettings.DiskSettings{{Path: "/dev/xvdb"}}) + + Expect(err).ToNot(HaveOccurred()) + Expect(len(cmdRunner.RunCommands)).To(Equal(2)) + Expect(cmdRunner.RunCommands[0]).To(Equal([]string{"parted", "-s", "/dev/xvdb", "p"})) + Expect(cmdRunner.RunCommands[1]).To(Equal([]string{"parted", "-s", "/dev/xvdb", "mklabel", "gpt", "unit", "%", "mkpart", "raw-ephemeral-0", "0", "100"})) + }) + }) }) Describe("SetupDataDir", func() { @@ -3731,6 +3888,7 @@ from-device-path dm-0 NETAPP ,LUN C-Mode certManager, monitRetryStrategy, devicePathResolver, + instanceStorageResolver, state, options, logger, diff --git a/platform/provider.go b/platform/provider.go index 7349c0110..aa1fe1363 100644 --- a/platform/provider.go +++ b/platform/provider.go @@ -158,6 +158,16 @@ func NewProvider(logger boshlog.Logger, dirProvider boshdirs.Provider, statsColl devicePathResolver = devicepathresolver.NewFallbackDevicePathResolver(symlinkLunResolver, devicePathResolver, logger) } + // Use auto-detecting instance storage resolver that determines NVMe vs non-NVMe + // based on device paths from the CPI (e.g., /dev/nvme* vs /dev/xvd* or /dev/sd*) + instanceStorageResolver := devicepathresolver.NewAutoDetectingInstanceStorageResolver( + fs, + devicePathResolver, + logger, + options.Linux.InstanceStorageManagedVolumePattern, + options.Linux.InstanceStorageDevicePattern, + ) + uuidGenerator := boshuuid.NewGenerator() logsTarProvider := boshlogstarprovider.NewLogsTarProvider(compressor, copier, dirProvider) @@ -176,6 +186,7 @@ func NewProvider(logger boshlog.Logger, dirProvider boshdirs.Provider, statsColl centosCertManager, monitRetryStrategy, devicePathResolver, + instanceStorageResolver, bootstrapState, options.Linux, logger, @@ -202,6 +213,7 @@ func NewProvider(logger boshlog.Logger, dirProvider boshdirs.Provider, statsColl ubuntuCertManager, monitRetryStrategy, devicePathResolver, + instanceStorageResolver, bootstrapState, options.Linux, logger,