From 7187f10a56d37cf84ac6d7f59b363ade9f9ffe3d Mon Sep 17 00:00:00 2001 From: Angel Marin Date: Wed, 11 Mar 2026 17:31:02 +0100 Subject: [PATCH] HYPERFLEET-706 - fix: lastUpdateTime for conditions --- docs/api-resources.md | 96 +- pkg/config/loader.go | 60 +- pkg/services/CLAUDE.md | 4 +- pkg/services/cluster.go | 110 ++- pkg/services/cluster_test.go | 107 ++- pkg/services/node_pool.go | 109 ++- pkg/services/node_pool_test.go | 128 ++- pkg/services/status_aggregation.go | 586 +++++++++--- pkg/services/status_aggregation_test.go | 847 ++++++++++++++++++ test/e2e-curl/01-initial-state/test.sh | 68 ++ test/e2e-curl/02-partial-adapters/test.sh | 82 ++ test/e2e-curl/03-all-adapters-ready/test.sh | 92 ++ test/e2e-curl/04-generation-bump/test.sh | 94 ++ test/e2e-curl/05-mixed-generations/test.sh | 95 ++ test/e2e-curl/06-stale-report/test.sh | 89 ++ test/e2e-curl/07-all-adapters-new-gen/test.sh | 101 +++ test/e2e-curl/08-adapter-goes-false/test.sh | 99 ++ test/e2e-curl/09-stable-true/test.sh | 106 +++ test/e2e-curl/10-stable-false/test.sh | 91 ++ test/e2e-curl/11-unknown-subsequent/test.sh | 96 ++ test/e2e-curl/12-unknown-first/test.sh | 101 +++ test/e2e-curl/README.md | 84 ++ test/e2e-curl/common.sh | 184 ++++ test/e2e-curl/run_all.sh | 96 ++ test/integration/adapter_status_test.go | 77 +- 25 files changed, 3261 insertions(+), 341 deletions(-) create mode 100755 test/e2e-curl/01-initial-state/test.sh create mode 100755 test/e2e-curl/02-partial-adapters/test.sh create mode 100755 test/e2e-curl/03-all-adapters-ready/test.sh create mode 100755 test/e2e-curl/04-generation-bump/test.sh create mode 100755 test/e2e-curl/05-mixed-generations/test.sh create mode 100755 test/e2e-curl/06-stale-report/test.sh create mode 100755 test/e2e-curl/07-all-adapters-new-gen/test.sh create mode 100755 test/e2e-curl/08-adapter-goes-false/test.sh create mode 100755 test/e2e-curl/09-stable-true/test.sh create mode 100755 test/e2e-curl/10-stable-false/test.sh create mode 100755 test/e2e-curl/11-unknown-subsequent/test.sh create mode 100755 test/e2e-curl/12-unknown-first/test.sh create mode 100644 test/e2e-curl/README.md create mode 100755 test/e2e-curl/common.sh create mode 100755 test/e2e-curl/run_all.sh diff --git a/docs/api-resources.md b/docs/api-resources.md index fa83ebf..70dc108 100644 --- a/docs/api-resources.md +++ b/docs/api-resources.md @@ -249,6 +249,100 @@ The status uses Kubernetes-style conditions instead of a single phase field: - One adapter reports `Available=False` for `observed_generation=1` `Available` transitions to `False` - One adapter reports `Available=False` for `observed_generation=2` `Available` keeps its `True` status +### Aggregation logic + +Description of the aggregation logic for the resource status conditions + +- An API that stores resources entities (clusters, nodepools) +- A sentinel that polls the API for changes and triggers messages +- Instances of "adapters": + - Read the messages + - Reconcile the state with the world + - Report back to the API, using statuses "conditions" + +Resources keep track of its status, which is affected by the reports from adapters + +- Each resource keeps a `generation` property that gets increased on every change +- Adapters associated with a resource, report their state as an array of adapter conditions + - Three of these conditions are always mandatory : `Available`, `Applied`, `Health` + - If one of the mandatory conditions is missing, the report is discarded + - A `observed_generation` field indicating the generation associated with the report + - `observed_time` for when the adapter work was done + - If the reported `observed_generation` is lower than the already stored `observed_generation` for that adapter, the report is discarded +- Each resource has a list of associated "adapters" used to compute the aggregated status.conditions +- Each resource "status.conditions" is array property composed of: + - The `Available` condition of each adapter, named as `Successful` + - 2 aggregated conditions: `Ready` and `Available` computed from the array of `Available` resource statuses conditions + - Only `Available` condition from adapters is used to compute aggregated conditions + +The whole API spec is at: + +The aggregation logic for a resource (cluster/nodepool) works as follows. + +**Notation:** + +- `X` = report's `observed_generation` +- `G` = resource's current `generation` +- `statuses[]` = all stored adapter condition reports +- `lut` = `last_update_time` +- `ltt` = `last_transition_time` +- `obs_gen` = `observed_generation` +- `obs_time` = report's `observed_time` +- `—` = no change + +--- + +#### Discard / Reject Rules + +Checked before any aggregation. A discarded or rejected report causes no state change. + +| Rule | Condition | Outcome | +|---|---|---| +| `obs_gen` too high | report `observed_generation` > resource `generation` | Discarded | +| Stale adapter report | report `observed_generation` < adapter's stored `observed_generation` | Discarded | +| Missing mandatory conditions | Missing any of `Available`, `Applied`, `Health`, or value not in `{True, False, Unknown}` | Discarded | +| Available=Unknown | Report is valid but `Available=Unknown` | Discarded | + +--- + +#### Lifecycle Events + +| Event | Condition | Target | → status | → obs_gen | → lut | → ltt | +|---|---|---|---|---|---|---| +| Creation | — | `Ready` | `False` | `1` | `now` | `now` | +| Creation | — | `Available` | `False` | `1` | `now` | `now` | +| Change (→G) | Was `Ready=True` | `Ready` | `False` | `G` | `now` | `now` | +| Change (→G) | Was `Ready=False` | `Ready` | `False` | `G` | `now` | `—` | +| Change (→G) | — | `Available` | unchanged | unchanged | `—` | `—` | + +--- + +#### Adapter Report Aggregation Matrix + +The **Ready** check and **Available** check are independent — both can apply to the same incoming report. + +##### Report `Available=True` (obs_gen = X) + +| Target | Current State | Required Condition | → status | → lut | → ltt | → obs_gen | +|---|---|---|---|---|---|---| +| `Ready` | `Ready=True` | `X==G` AND all `statuses[].obs_gen==G` AND all `statuses[].status==True` | unchanged | `min(statuses[].lut)` | `—` | `—` | +| `Ready` | `Ready=False` | `X==G` AND all `statuses[].obs_gen==G` AND all `statuses[].status==True` | **`True`** | `min(statuses[].lut)` | `obs_time` | `—` | +| `Ready` | any | Conditions above not met | `—` | `—` | `—` | `—` | +| `Available` | `Available=False` | all `statuses[].obs_gen==X` | **`True`** | `min(statuses[].lut)` | `obs_time` | `X` | +| `Available` | `Available=True` | all `statuses[].obs_gen==X` | unchanged | `min(statuses[].lut)` | `—` | `X` | +| `Available` | any | Conditions above not met | `—` | `—` | `—` | `—` | + +##### Report `Available=False` (obs_gen = X) + +| Target | Current State | Required Condition | → status | → lut | → ltt | → obs_gen | +|---|---|---|---|---|---|---| +| `Ready` | `Ready=False` | `X==G` | unchanged | `min(statuses[].lut)` | `—` | `—` | +| `Ready` | `Ready=True` | `X==G` | **`False`** | `obs_time` | `obs_time` | `—` | +| `Ready` | any | Conditions above not met | `—` | `—` | `—` | `—` | +| `Available` | `Available=False` | all `statuses[].obs_gen==X` | unchanged | `min(statuses[].lut)` | `—` | `X` | +| `Available` | `Available=True` | all `statuses[].obs_gen==X` | **`False`** | `obs_time` | `obs_time` | `X` | +| `Available` | any | Conditions above not met | `—` | `—` | `—` | `—` | + ## NodePool Management ### Endpoints @@ -456,7 +550,7 @@ The status object contains synthesized conditions computed from adapter reports: - All above fields plus: - `observed_generation` - Generation this condition reflects - `created_time` - When condition was first created (API-managed) -- `last_updated_time` - When adapter last reported (API-managed, from AdapterStatus.last_report_time) +- `last_updated_time` - When this condition was last refreshed (API-managed). For **Available**, always the evaluation time. For **Ready**: when Ready=True, the minimum of `last_report_time` across all required adapters that report Available=True at the current generation; when Ready=False, the evaluation time (so consumers can detect staleness). - `last_transition_time` - When status last changed (API-managed) ## Parameter Restrictions diff --git a/pkg/config/loader.go b/pkg/config/loader.go index d0da880..4c77f21 100644 --- a/pkg/config/loader.go +++ b/pkg/config/loader.go @@ -227,41 +227,43 @@ func (l *ConfigLoader) validateConfig(config *ApplicationConfig) error { } // handleJSONArrayEnvVars processes environment variables containing JSON arrays -// Viper doesn't automatically parse JSON from env vars, so we handle this explicitly -// Used for: HYPERFLEET_ADAPTERS_CLUSTER_ADAPTERS and HYPERFLEET_ADAPTERS_NODEPOOL_ADAPTERS +// Viper doesn't automatically parse JSON from env vars, so we handle this explicitly. +// Each viper key is filled from the first non-empty env var in the list (canonical name first, then aliases). func (l *ConfigLoader) handleJSONArrayEnvVars(ctx context.Context) error { - // Map of env var name -> viper key - jsonArrayMappings := map[string]string{ - EnvPrefix + "_ADAPTERS_REQUIRED_CLUSTER": "adapters.required.cluster", - EnvPrefix + "_ADAPTERS_REQUIRED_NODEPOOL": "adapters.required.nodepool", + // viper key -> ordered list of env var names (first one set wins) + clusterEnvVars := []string{ + EnvPrefix + "_ADAPTERS_REQUIRED_CLUSTER", + EnvPrefix + "_CLUSTER_ADAPTERS", // alias for user convenience + } + nodepoolEnvVars := []string{ + EnvPrefix + "_ADAPTERS_REQUIRED_NODEPOOL", + EnvPrefix + "_NODEPOOL_ADAPTERS", // alias for user convenience } - for envVar, viperKey := range jsonArrayMappings { - jsonValue := os.Getenv(envVar) - if jsonValue == "" { - continue - } - - // Parse JSON array - var arrayValue []string - if err := json.Unmarshal([]byte(jsonValue), &arrayValue); err != nil { - return fmt.Errorf("failed to parse %s as JSON array: %w (value: %s)", envVar, err, jsonValue) + setFromEnvVars := func(viperKey string, envVars []string) error { + for _, envVar := range envVars { + jsonValue := os.Getenv(envVar) + if jsonValue == "" { + continue + } + var arrayValue []string + if err := json.Unmarshal([]byte(jsonValue), &arrayValue); err != nil { + return fmt.Errorf("failed to parse %s as JSON array: %w (value: %s)", envVar, err, jsonValue) + } + // Set() overrides Viper's auto-env CSV parsing so JSON arrays are correct. + l.viper.Set(viperKey, arrayValue) + logger.With(ctx, "env_var", envVar, "count", len(arrayValue)).Debug("Parsed JSON array from environment") + return nil } - - // Always set the parsed JSON array value to override Viper's auto-env CSV parsing. - // Viper's AutomaticEnv treats comma-separated strings as arrays, incorrectly parsing - // JSON arrays like '["a","b"]' as ["[\"a\"", "\"b\"]"] instead of ["a", "b"]. - // - // We use Set() to ensure proper JSON parsing overrides Viper's CSV parsing. - // This maintains ENV > Config > Default precedence for adapters. - // - // NOTE: Adapters currently have no CLI flags (see bindFlags line 494). - // If CLI flags are added in the future, this code needs updating to check - // if the value came from a flag before calling Set(). - l.viper.Set(viperKey, arrayValue) - logger.With(ctx, "env_var", envVar, "count", len(arrayValue)).Debug("Parsed JSON array from environment") + return nil } + if err := setFromEnvVars("adapters.required.cluster", clusterEnvVars); err != nil { + return err + } + if err := setFromEnvVars("adapters.required.nodepool", nodepoolEnvVars); err != nil { + return err + } return nil } diff --git a/pkg/services/CLAUDE.md b/pkg/services/CLAUDE.md index 4d52c13..61cb3bd 100644 --- a/pkg/services/CLAUDE.md +++ b/pkg/services/CLAUDE.md @@ -22,9 +22,11 @@ func NewClusterService(dao, adapterStatusDao, config) ClusterService ## Status Aggregation `UpdateClusterStatusFromAdapters()` in `cluster.go` synthesizes two top-level conditions: -- **Available**: True if all required adapters report `Available=True` (any generation) +- **Available**: True if all required adapters report `Available=True` at ANY generation (last-known-good semantics). `ObservedGeneration` = minimum observed generation across qualifying adapters. When False, `ObservedGeneration` = current resource generation. - **Ready**: True if all adapters report `Available=True` AND `observed_generation` matches current generation +Ready's `LastUpdatedTime` is computed in `status_aggregation.computeReadyLastUpdated`: when Ready=False it is the minimum of `LastReportTime` across all required adapters (falls back to `now` if any required adapter has no stored status yet); when Ready=True it is the minimum of `LastReportTime` across required adapters that have Available=True at the current generation. True→False transitions override this with the triggering adapter's `observedTime`. + `ProcessAdapterStatus()` validates mandatory conditions (`Available`, `Applied`, `Health`) before persisting. Rejects `Available=Unknown` on subsequent reports (only allowed on first report). ## GenericService diff --git a/pkg/services/cluster.go b/pkg/services/cluster.go index cd82873..7247aae 100644 --- a/pkg/services/cluster.go +++ b/pkg/services/cluster.go @@ -96,8 +96,12 @@ func (s *sqlClusterService) Replace(ctx context.Context, cluster *api.Cluster) ( return nil, handleUpdateError("Cluster", err) } - // REMOVED: Event creation - no event-driven components - return cluster, nil + updatedCluster, svcErr := s.UpdateClusterStatusFromAdapters(ctx, cluster.ID) + if svcErr != nil { + return nil, svcErr + } + + return updatedCluster, nil } func (s *sqlClusterService) Delete(ctx context.Context, id string) *errors.ServiceError { @@ -143,9 +147,22 @@ func (s *sqlClusterService) OnDelete(ctx context.Context, id string) error { return nil } -// UpdateClusterStatusFromAdapters aggregates adapter statuses into cluster status +// UpdateClusterStatusFromAdapters aggregates adapter statuses into cluster status. +// Uses time.Now() as the observed time (for generation-change recomputations). +// Called from Create/Replace, so isLifecycleChange=true (Available frozen, Ready resets). func (s *sqlClusterService) UpdateClusterStatusFromAdapters( ctx context.Context, clusterID string, +) (*api.Cluster, *errors.ServiceError) { + return s.updateClusterStatusFromAdapters(ctx, clusterID, time.Now(), true) +} + +// updateClusterStatusFromAdapters is the internal implementation. +// observedTime is the triggering adapter's observed_time (its LastReportTime) and is used +// for transition timestamps in the synthetic conditions. +// isLifecycleChange=true freezes Available and resets Ready.lut=now (Create/Replace path). +// isLifecycleChange=false uses the normal adapter-report aggregation path. +func (s *sqlClusterService) updateClusterStatusFromAdapters( + ctx context.Context, clusterID string, observedTime time.Time, isLifecycleChange bool, ) (*api.Cluster, *errors.ServiceError) { // Get the cluster cluster, err := s.clusterDao.Get(ctx, clusterID) @@ -210,11 +227,14 @@ func (s *sqlClusterService) UpdateClusterStatusFromAdapters( // Compute synthetic Available and Ready conditions availableCondition, readyCondition := BuildSyntheticConditions( + ctx, cluster.StatusConditions, adapterStatuses, s.adapterConfig.RequiredClusterAdapters(), cluster.Generation, now, + observedTime, + isLifecycleChange, ) // Combine synthetic conditions with adapter conditions @@ -238,13 +258,15 @@ func (s *sqlClusterService) UpdateClusterStatusFromAdapters( return cluster, nil } -// ProcessAdapterStatus handles the business logic for adapter status: -// - Validates that all mandatory conditions (Available, Applied, Health) are present -// - Rejects duplicate condition types -// - For first status report: accepts Unknown Available condition to avoid data loss -// - For subsequent reports: rejects Unknown Available condition to preserve existing valid state -// - Uses complete replacement semantics: each update replaces all conditions for this adapter -// - Returns (nil, nil) for discarded updates +// ProcessAdapterStatus handles the business logic for adapter status. +// Pre-processing rules applied in order (spec §2): +// - Stale: discards if observed_generation < existing adapter generation +// - P1: discards if observed_generation > resource generation (report ahead of resource) +// - P2: rejects if mandatory conditions (Available, Applied, Health) are missing or have invalid status +// - P3: discards if Available == Unknown (not processed per spec) +// +// Otherwise: upserts the status and triggers aggregation. +// Returns (nil, nil) for discarded/rejected updates. func (s *sqlClusterService) ProcessAdapterStatus( ctx context.Context, clusterID string, adapterStatus *api.AdapterStatus, ) (*api.AdapterStatus, *errors.ServiceError) { @@ -256,12 +278,12 @@ func (s *sqlClusterService) ProcessAdapterStatus( return nil, errors.GeneralError("Failed to get adapter status: %s", findErr) } } + // Stale check: discard if older than the adapter's last recorded generation. if existingStatus != nil && adapterStatus.ObservedGeneration < existingStatus.ObservedGeneration { - // Discard stale status updates (older observed_generation). return nil, nil } - // Parse conditions from the adapter status + // Parse conditions from the adapter status (needed for P2 and P3 before resource fetch). var conditions []api.AdapterCondition if len(adapterStatus.Conditions) > 0 { if err := json.Unmarshal(adapterStatus.Conditions, &conditions); err != nil { @@ -269,7 +291,7 @@ func (s *sqlClusterService) ProcessAdapterStatus( } } - // Validate mandatory conditions and check for duplicates + // P2: validate mandatory conditions (presence and valid status values). if errorType, conditionName := ValidateMandatoryConditions(conditions); errorType != "" { ctx = logger.WithClusterID(ctx, clusterID) logger.Info(ctx, fmt.Sprintf("Discarding adapter status update from %s: %s condition %s", @@ -277,44 +299,44 @@ func (s *sqlClusterService) ProcessAdapterStatus( return nil, nil } - // Check Available condition for Unknown status - triggerAggregation := false + // P3: discard if Available == Unknown (spec §2, all reports). for _, cond := range conditions { - if cond.Type != api.ConditionTypeAvailable { - continue - } - - triggerAggregation = true - if cond.Status == api.AdapterConditionUnknown { - if existingStatus != nil { - // Non-first report && Available=Unknown → reject - ctx = logger.WithClusterID(ctx, clusterID) - logger.Info(ctx, fmt.Sprintf("Discarding adapter status update from %s: subsequent Unknown Available", - adapterStatus.Adapter)) - return nil, nil - } - // First report from this adapter: allow storing even with Available=Unknown - // but skip aggregation since Unknown should not affect cluster-level conditions - triggerAggregation = false + if cond.Type == api.ConditionTypeAvailable && cond.Status == api.AdapterConditionUnknown { + ctx = logger.WithClusterID(ctx, clusterID) + logger.Info(ctx, fmt.Sprintf("Discarding adapter status update from %s: Available=Unknown reports are not processed", + adapterStatus.Adapter)) + return nil, nil } - break } - // Upsert the adapter status (complete replacement) - upsertedStatus, err := s.adapterStatusDao.Upsert(ctx, adapterStatus) + // P1: discard if observed_generation is ahead of the current resource generation. + // Checked after P2/P3 to avoid unnecessary resource fetches for invalid/Unknown reports. + cluster, err := s.clusterDao.Get(ctx, clusterID) if err != nil { - return nil, handleCreateError("AdapterStatus", err) + return nil, handleGetError("Cluster", "id", clusterID, err) + } + if adapterStatus.ObservedGeneration > cluster.Generation { + ctx = logger.WithClusterID(ctx, clusterID) + logger.Info(ctx, fmt.Sprintf( + "Discarding adapter status update from %s: observed_generation %d > resource generation %d", + adapterStatus.Adapter, adapterStatus.ObservedGeneration, cluster.Generation)) + return nil, nil } - // Only trigger aggregation when triggerAggregation is true - if triggerAggregation { - if _, aggregateErr := s.UpdateClusterStatusFromAdapters( - ctx, clusterID, - ); aggregateErr != nil { - // Log error but don't fail the request - the status will be computed on next update - ctx = logger.WithClusterID(ctx, clusterID) - logger.WithError(ctx, aggregateErr).Warn("Failed to aggregate cluster status") - } + // Upsert the adapter status (complete replacement). + upsertedStatus, upsertErr := s.adapterStatusDao.Upsert(ctx, adapterStatus) + if upsertErr != nil { + return nil, handleCreateError("AdapterStatus", upsertErr) + } + + // Trigger aggregation using the adapter's observed_time for transition timestamps. + observedTime := time.Now() + if upsertedStatus.LastReportTime != nil { + observedTime = *upsertedStatus.LastReportTime + } + if _, aggregateErr := s.updateClusterStatusFromAdapters(ctx, clusterID, observedTime, false); aggregateErr != nil { + ctx = logger.WithClusterID(ctx, clusterID) + logger.WithError(ctx, aggregateErr).Warn("Failed to aggregate cluster status") } return upsertedStatus, nil diff --git a/pkg/services/cluster_test.go b/pkg/services/cluster_test.go index f4ea939..05ead0f 100644 --- a/pkg/services/cluster_test.go +++ b/pkg/services/cluster_test.go @@ -165,7 +165,8 @@ func (d *mockAdapterStatusDao) All(ctx context.Context) (api.AdapterStatusList, var _ dao.AdapterStatusDao = &mockAdapterStatusDao{} -// TestProcessAdapterStatus_FirstUnknownCondition tests that the first Unknown Available condition is stored +// TestProcessAdapterStatus_FirstUnknownCondition tests that Available=Unknown is discarded +// per spec §2 P3, regardless of whether it is the first or a subsequent report. func TestProcessAdapterStatus_FirstUnknownCondition(t *testing.T) { RegisterTestingT(t) @@ -202,14 +203,60 @@ func TestProcessAdapterStatus_FirstUnknownCondition(t *testing.T) { result, err := service.ProcessAdapterStatus(ctx, clusterID, adapterStatus) - // First report with Unknown should be accepted + // Per spec §2 P3: Available=Unknown is always discarded. Expect(err).To(BeNil()) - Expect(result).ToNot(BeNil(), "First report with Available=Unknown should be stored") - Expect(result.Adapter).To(Equal("test-adapter")) + Expect(result).To(BeNil(), "Available=Unknown must be discarded (spec §2 P3)") - // Verify the status was stored + // Verify the status was NOT stored + storedStatuses, _ := adapterStatusDao.FindByResource(ctx, "Cluster", clusterID) + Expect(len(storedStatuses)).To(Equal(0), "Unknown status must not be stored") +} + +// TestProcessAdapterStatus_FutureObservedGeneration tests that a report with +// observed_generation > resource.Generation is discarded per spec §2 P1. +func TestProcessAdapterStatus_FutureObservedGeneration(t *testing.T) { + RegisterTestingT(t) + + clusterDao := newMockClusterDao() + adapterStatusDao := newMockAdapterStatusDao() + config := testAdapterConfig() + service := NewClusterService(clusterDao, adapterStatusDao, config) + + ctx := context.Background() + clusterID := testClusterID + + // Create cluster at generation 1 + cluster := &api.Cluster{Generation: 1} + cluster.ID = clusterID + _, svcErr := service.Create(ctx, cluster) + Expect(svcErr).To(BeNil()) + + // Send a report claiming observed_generation=2, but resource is at gen=1 + conditions := []api.AdapterCondition{ + {Type: api.ConditionTypeAvailable, Status: api.AdapterConditionTrue, LastTransitionTime: time.Now()}, + {Type: api.ConditionTypeApplied, Status: api.AdapterConditionTrue, LastTransitionTime: time.Now()}, + {Type: api.ConditionTypeHealth, Status: api.AdapterConditionTrue, LastTransitionTime: time.Now()}, + } + conditionsJSON, _ := json.Marshal(conditions) + now := time.Now() + adapterStatus := &api.AdapterStatus{ + ResourceType: "Cluster", + ResourceID: clusterID, + Adapter: "test-adapter", + Conditions: conditionsJSON, + ObservedGeneration: 2, // ahead of resource gen=1 + CreatedTime: &now, + } + + result, err := service.ProcessAdapterStatus(ctx, clusterID, adapterStatus) + + // Per spec §2 P1: observed_generation > G → Discard. + Expect(err).To(BeNil()) + Expect(result).To(BeNil(), "Report with observed_generation > resource.Generation must be discarded (spec §2 P1)") + + // Verify the status was NOT stored storedStatuses, _ := adapterStatusDao.FindByResource(ctx, "Cluster", clusterID) - Expect(len(storedStatuses)).To(Equal(1), "First Unknown status should be stored") + Expect(len(storedStatuses)).To(Equal(0), "Future-generation report must not be stored") } // TestProcessAdapterStatus_SubsequentUnknownCondition tests that subsequent Unknown Available conditions are discarded @@ -379,8 +426,8 @@ func TestProcessAdapterStatus_FalseCondition(t *testing.T) { Expect(len(storedStatuses)).To(Equal(1), "Status should be stored for False condition") } -// TestProcessAdapterStatus_FirstMultipleConditions_AvailableUnknown tests that first reports with -// Available=Unknown are accepted even when other conditions are present +// TestProcessAdapterStatus_FirstMultipleConditions_AvailableUnknown tests that Available=Unknown +// is discarded per spec §2 P3, even when other conditions are present and it is the first report. func TestProcessAdapterStatus_FirstMultipleConditions_AvailableUnknown(t *testing.T) { RegisterTestingT(t) @@ -435,12 +482,13 @@ func TestProcessAdapterStatus_FirstMultipleConditions_AvailableUnknown(t *testin result, err := service.ProcessAdapterStatus(ctx, clusterID, adapterStatus) + // Per spec §2 P3: Available=Unknown is always discarded, even on first report. Expect(err).To(BeNil()) - Expect(result).ToNot(BeNil(), "First report with Available=Unknown should be accepted") + Expect(result).To(BeNil(), "Available=Unknown must be discarded (spec §2 P3)") - // Verify the status was stored + // Verify the status was NOT stored storedStatuses, _ := adapterStatusDao.FindByResource(ctx, "Cluster", clusterID) - Expect(len(storedStatuses)).To(Equal(1), "First status with Available=Unknown should be stored") + Expect(len(storedStatuses)).To(Equal(0), "Unknown status must not be stored") } // TestProcessAdapterStatus_SubsequentMultipleConditions_AvailableUnknown tests that subsequent reports @@ -596,24 +644,19 @@ func TestClusterAvailableReadyTransitions(t *testing.T) { Expect(ready.Status).To(Equal(api.ConditionFalse)) Expect(ready.ObservedGeneration).To(Equal(int32(2))) - // One adapter updates to gen=2 => Ready still False; Available still True (minObservedGeneration still 1). + // One adapter updates to gen=2 while the other stays at gen=1 => all_at_X=false. + // Per spec §5.2: all_at_X=false → no change → Available stays True@gen1. upsert("validation", api.AdapterConditionTrue, 2) avail, ready = getSynth() Expect(avail.Status).To(Equal(api.ConditionTrue)) Expect(avail.ObservedGeneration).To(Equal(int32(1))) Expect(ready.Status).To(Equal(api.ConditionFalse)) - // One adapter updates to gen=1 => Ready still False; Available still True (minObservedGeneration still 1). - // This is an edge case where an adapter reports a gen=1 status after a gen=2 status. - // Since we don't allow downgrading observed generations, we should not overwrite the cluster conditions. - // And Available should remain True, but in reality it should be False. - // This should be an unexpected edge case, since once a resource changes generation, - // all adapters should report a gen=2 status. - // So, while we are keeping Available True for gen=1, - // there should be soon an update to gen=2, which will overwrite the Available condition. + // Stale gen=1 report from validation is rejected (we don't allow downgrading observed generations). + // DAO state remains: validation@gen2=True, dns@gen1=True => Available still True@gen1. upsert("validation", api.AdapterConditionFalse, 1) avail, ready = getSynth() - Expect(avail.Status).To(Equal(api.ConditionTrue)) // <-- this is the edge case + Expect(avail.Status).To(Equal(api.ConditionTrue)) Expect(avail.ObservedGeneration).To(Equal(int32(1))) Expect(ready.Status).To(Equal(api.ConditionFalse)) @@ -628,7 +671,7 @@ func TestClusterAvailableReadyTransitions(t *testing.T) { upsert("dns", api.AdapterConditionFalse, 2) avail, ready = getSynth() Expect(avail.Status).To(Equal(api.ConditionFalse)) - Expect(avail.ObservedGeneration).To(Equal(int32(0))) + Expect(avail.ObservedGeneration).To(Equal(int32(2))) Expect(ready.Status).To(Equal(api.ConditionFalse)) // Available=Unknown is a no-op (does not store, does not overwrite cluster conditions). @@ -722,7 +765,7 @@ func TestClusterStaleAdapterStatusUpdatePolicy(t *testing.T) { Expect(available.Status).To(Equal(api.ConditionTrue)) Expect(available.ObservedGeneration).To(Equal(int32(2))) - // Stale False is more restrictive and should override. + // Stale False from gen=1 is discarded by the stale check (validation is already at gen=2). upsert("validation", api.AdapterConditionFalse, 1) available = getAvailable() Expect(available.Status).To(Equal(api.ConditionTrue)) @@ -769,7 +812,9 @@ func TestClusterSyntheticTimestampsStableWithoutAdapterStatus(t *testing.T) { StatusConditions: initialConditionsJSON, } cluster.ID = clusterID + beforeCreate := time.Now() created, svcErr := service.Create(ctx, cluster) + afterCreate := time.Now() Expect(svcErr).To(BeNil()) var createdConds []api.ResourceCondition @@ -789,12 +834,17 @@ func TestClusterSyntheticTimestampsStableWithoutAdapterStatus(t *testing.T) { Expect(createdReady).ToNot(BeNil()) Expect(createdAvailable.CreatedTime).To(Equal(fixedNow)) Expect(createdAvailable.LastTransitionTime).To(Equal(fixedNow)) + // No adapters reported → all_at_X=false → spec §5.2 no change → Available preserved from initialConditions. Expect(createdAvailable.LastUpdatedTime).To(Equal(fixedNow)) Expect(createdReady.CreatedTime).To(Equal(fixedNow)) Expect(createdReady.LastTransitionTime).To(Equal(fixedNow)) - Expect(createdReady.LastUpdatedTime).To(Equal(fixedNow)) + // Ready.LastUpdatedTime is refreshed to the evaluation time when isReady=false; assert it lies in the Create() window. + Expect(createdReady.LastUpdatedTime).To(BeTemporally(">=", beforeCreate)) + Expect(createdReady.LastUpdatedTime).To(BeTemporally("<=", afterCreate)) + beforeUpdate := time.Now() updated, err := service.UpdateClusterStatusFromAdapters(ctx, clusterID) + afterUpdate := time.Now() Expect(err).To(BeNil()) var updatedConds []api.ResourceCondition @@ -814,10 +864,14 @@ func TestClusterSyntheticTimestampsStableWithoutAdapterStatus(t *testing.T) { Expect(updatedReady).ToNot(BeNil()) Expect(updatedAvailable.CreatedTime).To(Equal(fixedNow)) Expect(updatedAvailable.LastTransitionTime).To(Equal(fixedNow)) + // No adapters reported → all_at_X=false → Available still preserved from fixedNow. Expect(updatedAvailable.LastUpdatedTime).To(Equal(fixedNow)) Expect(updatedReady.CreatedTime).To(Equal(fixedNow)) Expect(updatedReady.LastTransitionTime).To(Equal(fixedNow)) - Expect(updatedReady.LastUpdatedTime).To(Equal(fixedNow)) + // Ready.LastUpdatedTime is refreshed to the evaluation time when isReady=false; + // assert it lies in the UpdateClusterStatusFromAdapters() window. + Expect(updatedReady.LastUpdatedTime).To(BeTemporally(">=", beforeUpdate)) + Expect(updatedReady.LastUpdatedTime).To(BeTemporally("<=", afterUpdate)) } // TestProcessAdapterStatus_MissingMandatoryCondition_Available tests that updates missing Available are rejected @@ -1004,6 +1058,9 @@ func TestProcessAdapterStatus_CustomConditionRemoval(t *testing.T) { Expect(unmarshalErr).To(BeNil()) Expect(len(storedConditions1)).To(Equal(4)) + // Bump cluster generation to 2 so the second update (ObservedGeneration=2) is not blocked by P1. + clusterDao.clusters[clusterID].Generation = 2 + // Second update: remove custom condition (only send mandatory conditions) conditions2 := []api.AdapterCondition{ {Type: api.ConditionTypeAvailable, Status: api.AdapterConditionTrue, LastTransitionTime: time.Now()}, diff --git a/pkg/services/node_pool.go b/pkg/services/node_pool.go index 5b90568..bf6f73d 100644 --- a/pkg/services/node_pool.go +++ b/pkg/services/node_pool.go @@ -98,8 +98,12 @@ func (s *sqlNodePoolService) Replace( return nil, handleUpdateError("NodePool", err) } - // REMOVED: Event creation - no event-driven components - return nodePool, nil + updatedNodePool, svcErr := s.UpdateNodePoolStatusFromAdapters(ctx, nodePool.ID) + if svcErr != nil { + return nil, svcErr + } + + return updatedNodePool, nil } func (s *sqlNodePoolService) Delete(ctx context.Context, id string) *errors.ServiceError { @@ -144,9 +148,22 @@ func (s *sqlNodePoolService) OnDelete(ctx context.Context, id string) error { return nil } -// UpdateNodePoolStatusFromAdapters aggregates adapter statuses into nodepool status +// UpdateNodePoolStatusFromAdapters aggregates adapter statuses into nodepool status. +// Uses time.Now() as the observed time (for generation-change recomputations). +// Called from Create/Replace, so isLifecycleChange=true (Available frozen, Ready resets). func (s *sqlNodePoolService) UpdateNodePoolStatusFromAdapters( ctx context.Context, nodePoolID string, +) (*api.NodePool, *errors.ServiceError) { + return s.updateNodePoolStatusFromAdapters(ctx, nodePoolID, time.Now(), true) +} + +// updateNodePoolStatusFromAdapters is the internal implementation. +// observedTime is the triggering adapter's observed_time (its LastReportTime) and is used +// for transition timestamps in the synthetic conditions. +// isLifecycleChange=true freezes Available and resets Ready.lut=now (Create/Replace path). +// isLifecycleChange=false uses the normal adapter-report aggregation path. +func (s *sqlNodePoolService) updateNodePoolStatusFromAdapters( + ctx context.Context, nodePoolID string, observedTime time.Time, isLifecycleChange bool, ) (*api.NodePool, *errors.ServiceError) { // Get the nodepool nodePool, err := s.nodePoolDao.Get(ctx, nodePoolID) @@ -211,11 +228,14 @@ func (s *sqlNodePoolService) UpdateNodePoolStatusFromAdapters( // Compute synthetic Available and Ready conditions availableCondition, readyCondition := BuildSyntheticConditions( + ctx, nodePool.StatusConditions, adapterStatuses, s.adapterConfig.RequiredNodePoolAdapters(), nodePool.Generation, now, + observedTime, + isLifecycleChange, ) // Combine synthetic conditions with adapter conditions @@ -239,13 +259,15 @@ func (s *sqlNodePoolService) UpdateNodePoolStatusFromAdapters( return nodePool, nil } -// ProcessAdapterStatus handles the business logic for adapter status: -// - Validates that all mandatory conditions (Available, Applied, Health) are present -// - Rejects duplicate condition types -// - For first status report: accepts Unknown Available condition to avoid data loss -// - For subsequent reports: rejects Unknown Available condition to preserve existing valid state -// - Uses complete replacement semantics: each update replaces all conditions for this adapter -// - Returns (nil, nil) for discarded updates +// ProcessAdapterStatus handles the business logic for adapter status. +// Pre-processing rules applied in order (spec §2): +// - Stale: discards if observed_generation < existing adapter generation +// - P1: discards if observed_generation > resource generation (report ahead of resource) +// - P2: rejects if mandatory conditions (Available, Applied, Health) are missing or have invalid status +// - P3: discards if Available == Unknown (not processed per spec) +// +// Otherwise: upserts the status and triggers aggregation. +// Returns (nil, nil) for discarded/rejected updates. func (s *sqlNodePoolService) ProcessAdapterStatus( ctx context.Context, nodePoolID string, adapterStatus *api.AdapterStatus, ) (*api.AdapterStatus, *errors.ServiceError) { @@ -257,12 +279,12 @@ func (s *sqlNodePoolService) ProcessAdapterStatus( return nil, errors.GeneralError("Failed to get adapter status: %s", findErr) } } + // Stale check: discard if older than the adapter's last recorded generation. if existingStatus != nil && adapterStatus.ObservedGeneration < existingStatus.ObservedGeneration { - // Discard stale status updates (older observed_generation). return nil, nil } - // Parse conditions from the adapter status + // Parse conditions from the adapter status (needed for P2 and P3 before resource fetch). var conditions []api.AdapterCondition if len(adapterStatus.Conditions) > 0 { if err := json.Unmarshal(adapterStatus.Conditions, &conditions); err != nil { @@ -270,7 +292,7 @@ func (s *sqlNodePoolService) ProcessAdapterStatus( } } - // Validate mandatory conditions and check for duplicates + // P2: validate mandatory conditions (presence and valid status values). if errorType, conditionName := ValidateMandatoryConditions(conditions); errorType != "" { logger.With(ctx, logger.FieldNodePoolID, nodePoolID). Info(fmt.Sprintf("Discarding adapter status update from %s: %s condition %s", @@ -278,44 +300,43 @@ func (s *sqlNodePoolService) ProcessAdapterStatus( return nil, nil } - // Check Available condition for Unknown status - triggerAggregation := false + // P3: discard if Available == Unknown (spec §2, all reports). for _, cond := range conditions { - if cond.Type != api.ConditionTypeAvailable { - continue - } - - triggerAggregation = true - if cond.Status == api.AdapterConditionUnknown { - if existingStatus != nil { - // Non-first report && Available=Unknown → reject - logger.With(ctx, logger.FieldNodePoolID, nodePoolID). - Info(fmt.Sprintf("Discarding adapter status update from %s: subsequent Unknown Available", - adapterStatus.Adapter)) - return nil, nil - } - // First report from this adapter: allow storing even with Available=Unknown - // but skip aggregation since Unknown should not affect nodepool-level conditions - triggerAggregation = false + if cond.Type == api.ConditionTypeAvailable && cond.Status == api.AdapterConditionUnknown { + logger.With(ctx, logger.FieldNodePoolID, nodePoolID). + Info(fmt.Sprintf("Discarding adapter status update from %s: Available=Unknown reports are not processed", + adapterStatus.Adapter)) + return nil, nil } - break } - // Upsert the adapter status (complete replacement) - upsertedStatus, err := s.adapterStatusDao.Upsert(ctx, adapterStatus) + // P1: discard if observed_generation is ahead of the current resource generation. + // Checked after P2/P3 to avoid unnecessary resource fetches for invalid/Unknown reports. + nodePool, err := s.nodePoolDao.Get(ctx, nodePoolID) if err != nil { - return nil, handleCreateError("AdapterStatus", err) + return nil, handleGetError("NodePool", "id", nodePoolID, err) + } + if adapterStatus.ObservedGeneration > nodePool.Generation { + logger.With(ctx, logger.FieldNodePoolID, nodePoolID). + Info(fmt.Sprintf("Discarding adapter status update from %s: observed_generation %d > resource generation %d", + adapterStatus.Adapter, adapterStatus.ObservedGeneration, nodePool.Generation)) + return nil, nil } - // Only trigger aggregation when triggerAggregation is true - if triggerAggregation { - if _, aggregateErr := s.UpdateNodePoolStatusFromAdapters( - ctx, nodePoolID, - ); aggregateErr != nil { - // Log error but don't fail the request - the status will be computed on next update - logger.With(ctx, logger.FieldNodePoolID, nodePoolID). - WithError(aggregateErr).Warn("Failed to aggregate nodepool status") - } + // Upsert the adapter status (complete replacement). + upsertedStatus, upsertErr := s.adapterStatusDao.Upsert(ctx, adapterStatus) + if upsertErr != nil { + return nil, handleCreateError("AdapterStatus", upsertErr) + } + + // Trigger aggregation using the adapter's observed_time for transition timestamps. + observedTime := time.Now() + if upsertedStatus.LastReportTime != nil { + observedTime = *upsertedStatus.LastReportTime + } + if _, aggregateErr := s.updateNodePoolStatusFromAdapters(ctx, nodePoolID, observedTime, false); aggregateErr != nil { + logger.With(ctx, logger.FieldNodePoolID, nodePoolID). + WithError(aggregateErr).Warn("Failed to aggregate nodepool status") } return upsertedStatus, nil diff --git a/pkg/services/node_pool_test.go b/pkg/services/node_pool_test.go index c38193b..df15df1 100644 --- a/pkg/services/node_pool_test.go +++ b/pkg/services/node_pool_test.go @@ -82,7 +82,8 @@ func (d *mockNodePoolDao) All(ctx context.Context) (api.NodePoolList, error) { var _ dao.NodePoolDao = &mockNodePoolDao{} -// TestNodePoolProcessAdapterStatus_FirstUnknownCondition tests that the first Unknown Available condition is stored +// TestNodePoolProcessAdapterStatus_FirstUnknownCondition tests that Available=Unknown is discarded +// per spec §2 P3, regardless of whether it is the first or a subsequent report. func TestNodePoolProcessAdapterStatus_FirstUnknownCondition(t *testing.T) { RegisterTestingT(t) @@ -95,7 +96,13 @@ func TestNodePoolProcessAdapterStatus_FirstUnknownCondition(t *testing.T) { ctx := context.Background() nodePoolID := testNodePoolID - // Create first adapter status with all mandatory conditions but Available=Unknown + // Create nodepool first (needed for P1 check) + nodePool := &api.NodePool{Generation: 1} + nodePool.ID = nodePoolID + _, svcErr := service.Create(ctx, nodePool) + Expect(svcErr).To(BeNil()) + + // Send first status with Available=Unknown conditions := []api.AdapterCondition{ { Type: api.ConditionTypeAvailable, @@ -117,22 +124,71 @@ func TestNodePoolProcessAdapterStatus_FirstUnknownCondition(t *testing.T) { now := time.Now() adapterStatus := &api.AdapterStatus{ - ResourceType: "NodePool", - ResourceID: nodePoolID, - Adapter: "test-adapter", - Conditions: conditionsJSON, - CreatedTime: &now, + ResourceType: "NodePool", + ResourceID: nodePoolID, + Adapter: "test-adapter", + Conditions: conditionsJSON, + ObservedGeneration: 1, + CreatedTime: &now, } result, err := service.ProcessAdapterStatus(ctx, nodePoolID, adapterStatus) + // Per spec §2 P3: Available=Unknown is always discarded. Expect(err).To(BeNil()) - Expect(result).ToNot(BeNil(), "First report with Available=Unknown should be accepted") - Expect(result.Adapter).To(Equal("test-adapter")) + Expect(result).To(BeNil(), "Available=Unknown must be discarded (spec §2 P3)") - // Verify the status was stored + // Verify the status was NOT stored storedStatuses, _ := adapterStatusDao.FindByResource(ctx, "NodePool", nodePoolID) - Expect(len(storedStatuses)).To(Equal(1), "First Unknown status should be stored") + Expect(len(storedStatuses)).To(Equal(0), "Unknown status must not be stored") +} + +// TestNodePoolProcessAdapterStatus_FutureObservedGeneration tests that a report with +// observed_generation > resource.Generation is discarded per spec §2 P1. +func TestNodePoolProcessAdapterStatus_FutureObservedGeneration(t *testing.T) { + RegisterTestingT(t) + + nodePoolDao := newMockNodePoolDao() + adapterStatusDao := newMockAdapterStatusDao() + + config := testNodePoolAdapterConfig() + service := NewNodePoolService(nodePoolDao, adapterStatusDao, config) + + ctx := context.Background() + nodePoolID := testNodePoolID + + // Create nodepool at generation 1 + nodePool := &api.NodePool{Generation: 1} + nodePool.ID = nodePoolID + _, svcErr := service.Create(ctx, nodePool) + Expect(svcErr).To(BeNil()) + + // Send a report claiming observed_generation=2, but resource is at gen=1 + conditions := []api.AdapterCondition{ + {Type: api.ConditionTypeAvailable, Status: api.AdapterConditionTrue, LastTransitionTime: time.Now()}, + {Type: api.ConditionTypeApplied, Status: api.AdapterConditionTrue, LastTransitionTime: time.Now()}, + {Type: api.ConditionTypeHealth, Status: api.AdapterConditionTrue, LastTransitionTime: time.Now()}, + } + conditionsJSON, _ := json.Marshal(conditions) + now := time.Now() + adapterStatus := &api.AdapterStatus{ + ResourceType: "NodePool", + ResourceID: nodePoolID, + Adapter: "test-adapter", + Conditions: conditionsJSON, + ObservedGeneration: 2, // ahead of resource gen=1 + CreatedTime: &now, + } + + result, err := service.ProcessAdapterStatus(ctx, nodePoolID, adapterStatus) + + // Per spec §2 P1: observed_generation > G → Discard. + Expect(err).To(BeNil()) + Expect(result).To(BeNil(), "Report with observed_generation > resource.Generation must be discarded (spec §2 P1)") + + // Verify the status was NOT stored + storedStatuses, _ := adapterStatusDao.FindByResource(ctx, "NodePool", nodePoolID) + Expect(len(storedStatuses)).To(Equal(0), "Future-generation report must not be stored") } // TestNodePoolProcessAdapterStatus_SubsequentUnknownCondition tests that subsequent Unknown conditions are discarded @@ -242,8 +298,8 @@ func TestNodePoolProcessAdapterStatus_TrueCondition(t *testing.T) { Expect(len(storedStatuses)).To(Equal(1), "Status should be stored for True condition") } -// TestNodePoolProcessAdapterStatus_FirstMultipleConditions_AvailableUnknown tests that first reports -// with Available=Unknown are accepted even when other conditions are present +// TestNodePoolProcessAdapterStatus_FirstMultipleConditions_AvailableUnknown tests that Available=Unknown +// is discarded per spec §2 P3, even when other conditions are present and it is the first report. func TestNodePoolProcessAdapterStatus_FirstMultipleConditions_AvailableUnknown(t *testing.T) { RegisterTestingT(t) @@ -256,6 +312,12 @@ func TestNodePoolProcessAdapterStatus_FirstMultipleConditions_AvailableUnknown(t ctx := context.Background() nodePoolID := testNodePoolID + // Create nodepool first (needed for P1 check) + nodePool := &api.NodePool{Generation: 1} + nodePool.ID = nodePoolID + _, svcErr := service.Create(ctx, nodePool) + Expect(svcErr).To(BeNil()) + // Create first adapter status with all mandatory conditions but Available=Unknown conditions := []api.AdapterCondition{ { @@ -283,21 +345,23 @@ func TestNodePoolProcessAdapterStatus_FirstMultipleConditions_AvailableUnknown(t now := time.Now() adapterStatus := &api.AdapterStatus{ - ResourceType: "NodePool", - ResourceID: nodePoolID, - Adapter: "test-adapter", - Conditions: conditionsJSON, - CreatedTime: &now, + ResourceType: "NodePool", + ResourceID: nodePoolID, + Adapter: "test-adapter", + Conditions: conditionsJSON, + ObservedGeneration: 1, + CreatedTime: &now, } result, err := service.ProcessAdapterStatus(ctx, nodePoolID, adapterStatus) + // Per spec §2 P3: Available=Unknown is always discarded, even on first report. Expect(err).To(BeNil()) - Expect(result).ToNot(BeNil(), "First report with Available=Unknown should be accepted") + Expect(result).To(BeNil(), "Available=Unknown must be discarded (spec §2 P3)") - // Verify the status was stored + // Verify the status was NOT stored storedStatuses, _ := adapterStatusDao.FindByResource(ctx, "NodePool", nodePoolID) - Expect(len(storedStatuses)).To(Equal(1), "First status with Available=Unknown should be stored") + Expect(len(storedStatuses)).To(Equal(0), "Unknown status must not be stored") } // TestNodePoolProcessAdapterStatus_SubsequentMultipleConditions_AvailableUnknown tests that subsequent @@ -451,7 +515,8 @@ func TestNodePoolAvailableReadyTransitions(t *testing.T) { Expect(ready.Status).To(Equal(api.ConditionFalse)) Expect(ready.ObservedGeneration).To(Equal(int32(2))) - // One adapter updates to gen=2 => Ready still False; Available still True (minObservedGeneration still 1). + // One adapter updates to gen=2 while the other stays at gen=1 => all_at_X=false. + // Per spec §5.2: all_at_X=false → no change → Available stays True@gen1. upsert("validation", api.AdapterConditionTrue, 2) avail, ready = getSynth() Expect(avail.Status).To(Equal(api.ConditionTrue)) @@ -469,7 +534,7 @@ func TestNodePoolAvailableReadyTransitions(t *testing.T) { upsert("hypershift", api.AdapterConditionFalse, 2) avail, ready = getSynth() Expect(avail.Status).To(Equal(api.ConditionFalse)) - Expect(avail.ObservedGeneration).To(Equal(int32(0))) + Expect(avail.ObservedGeneration).To(Equal(int32(2))) Expect(ready.Status).To(Equal(api.ConditionFalse)) // Adapter status missing mandatory conditions should be rejected and not overwrite synthetic conditions. @@ -582,7 +647,7 @@ func TestNodePoolStaleAdapterStatusUpdatePolicy(t *testing.T) { Expect(available.Status).To(Equal(api.ConditionTrue)) Expect(available.ObservedGeneration).To(Equal(int32(2))) - // Stale False is more restrictive and should override but we do not override newer generation responses + // Stale False from gen=1 is discarded by the stale check (validation is already at gen=2). upsert("validation", api.AdapterConditionFalse, 1) available = getAvailable() Expect(available.Status).To(Equal(api.ConditionTrue)) @@ -629,7 +694,9 @@ func TestNodePoolSyntheticTimestampsStableWithoutAdapterStatus(t *testing.T) { StatusConditions: initialConditionsJSON, } nodePool.ID = nodePoolID + beforeCreate := time.Now() created, svcErr := service.Create(ctx, nodePool) + afterCreate := time.Now() Expect(svcErr).To(BeNil()) var createdConds []api.ResourceCondition @@ -649,12 +716,17 @@ func TestNodePoolSyntheticTimestampsStableWithoutAdapterStatus(t *testing.T) { Expect(createdReady).ToNot(BeNil()) Expect(createdAvailable.CreatedTime).To(Equal(fixedNow)) Expect(createdAvailable.LastTransitionTime).To(Equal(fixedNow)) + // No adapters reported → all_at_X=false → spec §5.2 no change → Available preserved from initialConditions. Expect(createdAvailable.LastUpdatedTime).To(Equal(fixedNow)) Expect(createdReady.CreatedTime).To(Equal(fixedNow)) Expect(createdReady.LastTransitionTime).To(Equal(fixedNow)) - Expect(createdReady.LastUpdatedTime).To(Equal(fixedNow)) + // Ready.LastUpdatedTime is refreshed to the evaluation time when isReady=false; assert it lies in the Create() window. + Expect(createdReady.LastUpdatedTime).To(BeTemporally(">=", beforeCreate)) + Expect(createdReady.LastUpdatedTime).To(BeTemporally("<=", afterCreate)) + beforeUpdate := time.Now() updated, err := service.UpdateNodePoolStatusFromAdapters(ctx, nodePoolID) + afterUpdate := time.Now() Expect(err).To(BeNil()) var updatedConds []api.ResourceCondition @@ -674,8 +746,12 @@ func TestNodePoolSyntheticTimestampsStableWithoutAdapterStatus(t *testing.T) { Expect(updatedReady).ToNot(BeNil()) Expect(updatedAvailable.CreatedTime).To(Equal(fixedNow)) Expect(updatedAvailable.LastTransitionTime).To(Equal(fixedNow)) + // No adapters reported → all_at_X=false → Available still preserved from fixedNow. Expect(updatedAvailable.LastUpdatedTime).To(Equal(fixedNow)) Expect(updatedReady.CreatedTime).To(Equal(fixedNow)) Expect(updatedReady.LastTransitionTime).To(Equal(fixedNow)) - Expect(updatedReady.LastUpdatedTime).To(Equal(fixedNow)) + // Ready.LastUpdatedTime is refreshed to the evaluation time when isReady=false; + // assert it lies in the UpdateNodePoolStatusFromAdapters() window. + Expect(updatedReady.LastUpdatedTime).To(BeTemporally(">=", beforeUpdate)) + Expect(updatedReady.LastUpdatedTime).To(BeTemporally("<=", afterUpdate)) } diff --git a/pkg/services/status_aggregation.go b/pkg/services/status_aggregation.go index 9afa275..2352897 100644 --- a/pkg/services/status_aggregation.go +++ b/pkg/services/status_aggregation.go @@ -1,13 +1,15 @@ package services import ( + "context" "encoding/json" - "math" + "fmt" "strings" "time" "unicode" "github.com/openshift-hyperfleet/hyperfleet-api/pkg/api" + "github.com/openshift-hyperfleet/hyperfleet-api/pkg/logger" ) // Mandatory condition types that must be present in all adapter status updates @@ -15,8 +17,9 @@ var mandatoryConditionTypes = []string{api.ConditionTypeAvailable, api.Condition // Condition validation error types const ( - ConditionValidationErrorDuplicate = "duplicate" - ConditionValidationErrorMissing = "missing" + ConditionValidationErrorDuplicate = "duplicate" + ConditionValidationErrorMissing = "missing" + ConditionValidationErrorInvalidStatus = "invalid_status" ) // Required adapter lists configured via pkg/config/adapter.go (see AdapterRequirementsConfig) @@ -30,30 +33,43 @@ var adapterConditionSuffixMap = map[string]string{ // Add custom mappings here when needed } -// ValidateMandatoryConditions checks if all mandatory conditions are present and rejects duplicate condition types. +// validAdapterConditionStatuses holds the set of allowed status values for adapter conditions. +var validAdapterConditionStatuses = map[api.AdapterConditionStatus]bool{ + api.AdapterConditionTrue: true, + api.AdapterConditionFalse: true, + api.AdapterConditionUnknown: true, +} + +// ValidateMandatoryConditions checks that all mandatory conditions are present, have no +// duplicate types, and carry a valid status value (True, False, or Unknown). // Returns (errorType, conditionName) where: // - If duplicate found: (ConditionValidationErrorDuplicate, duplicateConditionType) // - If missing condition: (ConditionValidationErrorMissing, missingConditionType) +// - If invalid status: (ConditionValidationErrorInvalidStatus, conditionType) // - If all valid: ("", "") func ValidateMandatoryConditions(conditions []api.AdapterCondition) (errorType, conditionName string) { - // Check for duplicate condition types - seen := make(map[string]bool) + // Check for duplicate condition types and track status values. + seen := make(map[string]api.AdapterConditionStatus) for _, cond := range conditions { // Reject empty condition types if cond.Type == "" { return ConditionValidationErrorMissing, "" } - if seen[cond.Type] { + if _, exists := seen[cond.Type]; exists { return ConditionValidationErrorDuplicate, cond.Type } - seen[cond.Type] = true + seen[cond.Type] = cond.Status } - // Check that all mandatory conditions are present + // Check that all mandatory conditions are present and have valid status values. for _, mandatoryType := range mandatoryConditionTypes { - if !seen[mandatoryType] { + status, exists := seen[mandatoryType] + if !exists { return ConditionValidationErrorMissing, mandatoryType } + if !validAdapterConditionStatuses[status] { + return ConditionValidationErrorInvalidStatus, mandatoryType + } } return "", "" @@ -96,150 +112,404 @@ func MapAdapterToConditionType(adapterName string) string { return result.String() } -// ComputeAvailableCondition checks if all required adapters have Available=True at ANY generation. -// Returns: (isAvailable bool, minObservedGeneration int32) -// "Available" means the system is running at some known good configuration (last known good config). -// The minObservedGeneration is the lowest generation across all required adapters. -func ComputeAvailableCondition(adapterStatuses api.AdapterStatusList, requiredAdapters []string) (bool, int32) { - if len(adapterStatuses) == 0 || len(requiredAdapters) == 0 { - return false, 1 +// adapterAvailabilitySnapshot is the result of scanning all required adapters to determine +// whether they are at a consistent generation and whether they all report Available=True. +type adapterAvailabilitySnapshot struct { + // consistent is true when every required adapter has reported at the same observed generation. + // When false, the existing Available condition should be left unchanged. + consistent bool + // available is true when consistent=true and every required adapter has Available=True. + available bool + // generation is the common observed generation across required adapters (only valid when consistent=true). + generation int32 + // minLastReportTime is the earliest LastReportTime across all required adapters (nil when none available). + minLastReportTime *time.Time +} + +// scanAdapterAvailability inspects the stored adapter statuses for the required adapters and +// returns a snapshot describing whether they are consistent and available. +// +// Available=True requires all required adapters to have reported at the same observed generation +// and all to have Available=True. If any required adapter is missing or at a different generation, +// consistent=false is returned and the caller should leave the current Available condition unchanged. +func scanAdapterAvailability( + ctx context.Context, + adapterStatuses api.AdapterStatusList, + requiredAdapters []string, +) adapterAvailabilitySnapshot { + if len(requiredAdapters) == 0 { + return adapterAvailabilitySnapshot{} } - // Build a map of adapter name -> (available status, observed generation) - adapterMap := make(map[string]struct { + type adapterInfo struct { available string observedGeneration int32 - }) + lastReportTime *time.Time + } + adapterMap := make(map[string]adapterInfo, len(adapterStatuses)) - for _, adapterStatus := range adapterStatuses { - // Unmarshal conditions to find "Available" + for _, s := range adapterStatuses { var conditions []struct { Type string `json:"type"` Status string `json:"status"` } - if len(adapterStatus.Conditions) > 0 { - if err := json.Unmarshal(adapterStatus.Conditions, &conditions); err == nil { - for _, cond := range conditions { - if cond.Type == api.ConditionTypeAvailable { - adapterMap[adapterStatus.Adapter] = struct { - available string - observedGeneration int32 - }{ - available: cond.Status, - observedGeneration: adapterStatus.ObservedGeneration, - } - break - } + if len(s.Conditions) == 0 { + continue + } + if err := json.Unmarshal(s.Conditions, &conditions); err != nil { + logger.WithError(ctx, err).Warn( + fmt.Sprintf("failed to parse adapter conditions for adapter %s", s.Adapter)) + continue + } + for _, cond := range conditions { + if cond.Type == api.ConditionTypeAvailable { + adapterMap[s.Adapter] = adapterInfo{ + available: cond.Status, + observedGeneration: s.ObservedGeneration, + lastReportTime: s.LastReportTime, } + break } } } - // Count available adapters and track min observed generation - numAvailable := 0 - minObservedGeneration := int32(math.MaxInt32) - - for _, adapterName := range requiredAdapters { - adapterInfo, exists := adapterMap[adapterName] - + // All required adapters must have reported at the same observed generation. + var commonGen *int32 + for _, name := range requiredAdapters { + info, exists := adapterMap[name] if !exists { - // Required adapter not found - not available - continue + return adapterAvailabilitySnapshot{} + } + if commonGen == nil { + g := info.observedGeneration + commonGen = &g + } else if info.observedGeneration != *commonGen { + return adapterAvailabilitySnapshot{} } + } + + if commonGen == nil { + return adapterAvailabilitySnapshot{} + } - // For Available condition, we don't check generation matching - // We just need Available=True at ANY generation - if adapterInfo.available == "True" { - numAvailable++ - if adapterInfo.observedGeneration < minObservedGeneration { - minObservedGeneration = adapterInfo.observedGeneration + // Consistent generation: determine availability and earliest report time. + allAvailable := true + var minLRT *time.Time + for _, name := range requiredAdapters { + info := adapterMap[name] + if info.available != string(api.AdapterConditionTrue) { + allAvailable = false + } + if info.lastReportTime != nil { + if minLRT == nil || info.lastReportTime.Before(*minLRT) { + t := *info.lastReportTime + minLRT = &t } } } - // Available when all required adapters have Available=True (at any generation) - numRequired := len(requiredAdapters) - if numAvailable == numRequired { - return true, minObservedGeneration + return adapterAvailabilitySnapshot{ + consistent: true, + available: allAvailable, + generation: *commonGen, + minLastReportTime: minLRT, + } +} + +// buildAvailableCondition computes the Available ResourceCondition from the current adapter +// availability snapshot, the previous condition (may be nil), and the evaluation time. +// +// observedTime is the triggering adapter's observed_time (its LastReportTime). It is used for +// LastTransitionTime on status changes and LastUpdatedTime on True→False transitions. For all +// other cases LastUpdatedTime is the earliest report time across required adapters. +// +// When adapters are not at a consistent generation, the existing condition is preserved unchanged. +func buildAvailableCondition( + snapshot adapterAvailabilitySnapshot, + existing *api.ResourceCondition, + resourceGeneration int32, + now time.Time, + observedTime time.Time, +) api.ResourceCondition { + if !snapshot.consistent { + if existing != nil { + return *existing + } + return api.ResourceCondition{ + Type: api.ConditionTypeAvailable, + Status: api.ConditionFalse, + ObservedGeneration: resourceGeneration, + LastTransitionTime: now, + CreatedTime: now, + LastUpdatedTime: now, + } + } + + newStatus := api.ConditionFalse + if snapshot.available { + newStatus = api.ConditionTrue + } + + prevStatus := api.ConditionFalse + if existing != nil { + prevStatus = existing.Status + } + + // True→False: use the triggering adapter's observed time (when the failure was first seen). + // All other cases (stays True, stays False, False→True): use earliest adapter report time. + var lut time.Time + switch { + case prevStatus == api.ConditionTrue && newStatus == api.ConditionFalse: + lut = observedTime + case snapshot.minLastReportTime != nil: + lut = *snapshot.minLastReportTime + default: + lut = now } - // Return 0 for minObservedGeneration when not available - return false, 0 + // LastTransitionTime advances on status change using the triggering adapter's observed time. + ltt := observedTime + if existing != nil && existing.Status == newStatus && !existing.LastTransitionTime.IsZero() { + ltt = existing.LastTransitionTime + } + + createdTime := now + if existing != nil && !existing.CreatedTime.IsZero() { + createdTime = existing.CreatedTime + } + + cond := api.ResourceCondition{ + Type: api.ConditionTypeAvailable, + Status: newStatus, + ObservedGeneration: snapshot.generation, + LastTransitionTime: ltt, + CreatedTime: createdTime, + LastUpdatedTime: lut, + } + + // Carry over Reason/Message when status is unchanged. + if existing != nil && prevStatus == newStatus { + if cond.Reason == nil { + cond.Reason = existing.Reason + } + if cond.Message == nil { + cond.Message = existing.Message + } + } + + return cond } -// ComputeReadyCondition checks if all required adapters have Available=True at the CURRENT generation. -// "Ready" means the system is running at the latest spec generation. +// ComputeReadyCondition reports whether all required adapters have Available=True at the current +// resource generation. Unlike Available, Ready requires adapters to be caught up to the latest +// generation — it does not accept reports from older generations. func ComputeReadyCondition( - adapterStatuses api.AdapterStatusList, requiredAdapters []string, resourceGeneration int32, + ctx context.Context, adapterStatuses api.AdapterStatusList, requiredAdapters []string, resourceGeneration int32, ) bool { if len(adapterStatuses) == 0 || len(requiredAdapters) == 0 { return false } - // Build a map of adapter name -> (available status, observed generation) - adapterMap := make(map[string]struct { + type adapterInfo struct { available string observedGeneration int32 - }) + } + adapterMap := make(map[string]adapterInfo, len(adapterStatuses)) - for _, adapterStatus := range adapterStatuses { - // Unmarshal conditions to find "Available" + for _, s := range adapterStatuses { var conditions []struct { Type string `json:"type"` Status string `json:"status"` } - if len(adapterStatus.Conditions) > 0 { - if err := json.Unmarshal(adapterStatus.Conditions, &conditions); err == nil { - for _, cond := range conditions { - if cond.Type == api.ConditionTypeAvailable { - adapterMap[adapterStatus.Adapter] = struct { - available string - observedGeneration int32 - }{ - available: cond.Status, - observedGeneration: adapterStatus.ObservedGeneration, - } - break - } + if len(s.Conditions) == 0 { + continue + } + if err := json.Unmarshal(s.Conditions, &conditions); err != nil { + logger.WithError(ctx, err).Warn( + fmt.Sprintf("failed to parse adapter conditions for adapter %s", s.Adapter)) + continue + } + for _, cond := range conditions { + if cond.Type == api.ConditionTypeAvailable { + adapterMap[s.Adapter] = adapterInfo{ + available: cond.Status, + observedGeneration: s.ObservedGeneration, } + break } } } - // Count ready adapters (Available=True at current generation) numReady := 0 + for _, name := range requiredAdapters { + info, exists := adapterMap[name] + if !exists || (resourceGeneration > 0 && info.observedGeneration != resourceGeneration) { + continue + } + if info.available == string(api.AdapterConditionTrue) { + numReady++ + } + } - for _, adapterName := range requiredAdapters { - adapterInfo, exists := adapterMap[adapterName] + return numReady == len(requiredAdapters) +} - if !exists { - // Required adapter not found - not ready - continue +// findAdapterStatus returns the first adapter status in the list with the given adapter name, or (nil, false). +func findAdapterStatus(adapterStatuses api.AdapterStatusList, adapterName string) (*api.AdapterStatus, bool) { + for _, s := range adapterStatuses { + if s.Adapter == adapterName { + return s, true } + } + return nil, false +} - // For Ready condition, we require generation matching - if resourceGeneration > 0 && adapterInfo.observedGeneration != resourceGeneration { - // Adapter is processing old generation (stale) - not ready - continue +// adapterConditionsHasAvailableTrue returns true if the adapter conditions JSON +// contains a condition with type Available and status True. +func adapterConditionsHasAvailableTrue(ctx context.Context, adapterName string, conditions []byte) bool { + if len(conditions) == 0 { + return false + } + var conds []struct { + Type string `json:"type"` + Status string `json:"status"` + } + if err := json.Unmarshal(conditions, &conds); err != nil { + logger.WithError(ctx, err).Warn( + fmt.Sprintf("failed to parse adapter conditions for adapter %s", adapterName)) + return false + } + for _, c := range conds { + if c.Type == api.ConditionTypeAvailable && c.Status == string(api.AdapterConditionTrue) { + return true } + } + return false +} - // Check available status - if adapterInfo.available == "True" { - numReady++ +// computeReadyLastUpdated returns the timestamp to use for the Ready condition's LastUpdatedTime. +// +// When isReady is false, it returns the minimum LastReportTime across all required adapters +// (spec: last_update_time=min(resource.statuses[].last_update_time) when Ready stays False). +// Falls back to now if any required adapter has no stored status or no LRT yet. +// +// When isReady is true, it returns the minimum LastReportTime across all required adapters +// that have Available=True at the current generation. Falls back to now if none qualify. +// +// Note: True→False transitions override this value with observedTime in buildReadyCondition. +func computeReadyLastUpdated( + ctx context.Context, + adapterStatuses api.AdapterStatusList, + requiredAdapters []string, + resourceGeneration int32, + now time.Time, + isReady bool, +) time.Time { + if !isReady { + // Use min(LRTs) across all required adapters. + // Fall back to now if a required adapter has no stored status or no LRT. + var minTime *time.Time + for _, adapterName := range requiredAdapters { + status, ok := findAdapterStatus(adapterStatuses, adapterName) + if !ok || status.LastReportTime == nil { + return now // safety: required adapter missing or has no timestamp + } + if minTime == nil || status.LastReportTime.Before(*minTime) { + t := *status.LastReportTime + minTime = &t + } + } + if minTime == nil { + return now + } + return *minTime + } + + var minTime *time.Time + for _, adapterName := range requiredAdapters { + status, ok := findAdapterStatus(adapterStatuses, adapterName) + if !ok { + return now // safety: required adapter missing + } + if status.LastReportTime == nil { + return now // safety: no timestamp + } + if status.ObservedGeneration != resourceGeneration { + continue // not at current gen, skip + } + if !adapterConditionsHasAvailableTrue(ctx, adapterName, status.Conditions) { + continue } + if minTime == nil || status.LastReportTime.Before(*minTime) { + t := *status.LastReportTime + minTime = &t + } + } + + if minTime == nil { + return now // safety fallback + } + return *minTime +} + +// buildReadyCondition computes the Ready ResourceCondition from the current adapter statuses, +// the previous condition (may be nil), and the evaluation time. +// +// observedTime is the triggering adapter's observed_time. It is used for LastTransitionTime +// on status changes and LastUpdatedTime on True→False transitions. +// +// Ready=True requires all required adapters to have Available=True at the current resource +// generation. LastUpdatedTime is the evaluation time when False (so callers can apply a +// freshness threshold), or the earliest adapter report time when True. +func buildReadyCondition( + ctx context.Context, + adapterStatuses api.AdapterStatusList, + requiredAdapters []string, + resourceGeneration int32, + existing *api.ResourceCondition, + now time.Time, + observedTime time.Time, +) api.ResourceCondition { + isReady := ComputeReadyCondition(ctx, adapterStatuses, requiredAdapters, resourceGeneration) + status := api.ConditionFalse + if isReady { + status = api.ConditionTrue + } + + cond := api.ResourceCondition{ + Type: api.ConditionTypeReady, + Status: status, + ObservedGeneration: resourceGeneration, + LastTransitionTime: now, + CreatedTime: now, + LastUpdatedTime: now, + } + + lut := computeReadyLastUpdated(ctx, adapterStatuses, requiredAdapters, resourceGeneration, now, isReady) + + // True→False: use the triggering adapter's observed time (when the failure was first seen). + prevStatus := api.ConditionFalse + if existing != nil { + prevStatus = existing.Status + } + if prevStatus == api.ConditionTrue && status == api.ConditionFalse { + lut = observedTime } - // Ready when all required adapters have Available=True at current generation - numRequired := len(requiredAdapters) - return numReady == numRequired + applyConditionHistory(&cond, existing, observedTime, lut) + + return cond } func BuildSyntheticConditions( + ctx context.Context, existingConditionsJSON []byte, adapterStatuses api.AdapterStatusList, requiredAdapters []string, resourceGeneration int32, now time.Time, + observedTime time.Time, + isLifecycleChange bool, ) (api.ResourceCondition, api.ResourceCondition) { var existingAvailable *api.ResourceCondition var existingReady *api.ResourceCondition @@ -258,66 +528,124 @@ func BuildSyntheticConditions( } } - isAvailable, minObservedGeneration := ComputeAvailableCondition(adapterStatuses, requiredAdapters) - availableStatus := api.ConditionFalse - if isAvailable { - availableStatus = api.ConditionTrue + // Zero adapters: trivially satisfied — both conditions are True. + if len(requiredAdapters) == 0 { + available := buildTrueCondition(api.ConditionTypeAvailable, existingAvailable, resourceGeneration, now) + ready := buildTrueCondition(api.ConditionTypeReady, existingReady, resourceGeneration, now) + return available, ready } - availableCondition := api.ResourceCondition{ - Type: api.ConditionTypeAvailable, - Status: availableStatus, - ObservedGeneration: minObservedGeneration, + + // Lifecycle change (Create / Replace): Available is frozen; Ready resets with lut=now. + if isLifecycleChange { + var available api.ResourceCondition + if existingAvailable != nil { + available = *existingAvailable + } else { + available = api.ResourceCondition{ + Type: api.ConditionTypeAvailable, + Status: api.ConditionFalse, + ObservedGeneration: resourceGeneration, + LastTransitionTime: now, + CreatedTime: now, + LastUpdatedTime: now, + } + } + ready := buildReadyConditionLifecycle(existingReady, resourceGeneration, now, observedTime) + return available, ready + } + + // Normal adapter-report path. + snapshot := scanAdapterAvailability(ctx, adapterStatuses, requiredAdapters) + available := buildAvailableCondition(snapshot, existingAvailable, resourceGeneration, now, observedTime) + ready := buildReadyCondition( + ctx, adapterStatuses, requiredAdapters, resourceGeneration, existingReady, now, observedTime) + + return available, ready +} + +// buildTrueCondition produces a True ResourceCondition, preserving history from existing. +// CreatedTime and LastTransitionTime are preserved when the existing condition was also True. +func buildTrueCondition( + condType string, + existing *api.ResourceCondition, + resourceGeneration int32, + now time.Time, +) api.ResourceCondition { + cond := api.ResourceCondition{ + Type: condType, + Status: api.ConditionTrue, + ObservedGeneration: resourceGeneration, LastTransitionTime: now, CreatedTime: now, LastUpdatedTime: now, } - preserveSyntheticCondition(&availableCondition, existingAvailable, now) - - isReady := ComputeReadyCondition(adapterStatuses, requiredAdapters, resourceGeneration) - readyStatus := api.ConditionFalse - if isReady { - readyStatus = api.ConditionTrue + if existing != nil { + if !existing.CreatedTime.IsZero() { + cond.CreatedTime = existing.CreatedTime + } + if existing.Status == api.ConditionTrue && !existing.LastTransitionTime.IsZero() { + cond.LastTransitionTime = existing.LastTransitionTime + } } - readyCondition := api.ResourceCondition{ + return cond +} + +// buildReadyConditionLifecycle produces Ready=False at the new generation with lut=now. +// History (CreatedTime, LastTransitionTime) is preserved via applyConditionHistory. +func buildReadyConditionLifecycle( + existing *api.ResourceCondition, + resourceGeneration int32, + now time.Time, + observedTime time.Time, +) api.ResourceCondition { + cond := api.ResourceCondition{ Type: api.ConditionTypeReady, - Status: readyStatus, + Status: api.ConditionFalse, ObservedGeneration: resourceGeneration, LastTransitionTime: now, CreatedTime: now, LastUpdatedTime: now, } - preserveSyntheticCondition(&readyCondition, existingReady, now) - - return availableCondition, readyCondition + applyConditionHistory(&cond, existing, observedTime, now) + return cond } -func preserveSyntheticCondition(target *api.ResourceCondition, existing *api.ResourceCondition, now time.Time) { +// applyConditionHistory copies stable timestamps and metadata from an existing condition. +// transitionTime is used for LastTransitionTime on status changes — callers pass the +// triggering adapter's observed_time so the timestamp reflects when the change was observed. +// lastUpdatedTime is used unconditionally for LastUpdatedTime — the caller is responsible +// for computing the correct value (e.g. now, computeReadyLastUpdated(...)). +func applyConditionHistory( + target *api.ResourceCondition, + existing *api.ResourceCondition, + transitionTime time.Time, + lastUpdatedTime time.Time, +) { if existing == nil { + target.LastUpdatedTime = lastUpdatedTime return } + if !existing.CreatedTime.IsZero() { + target.CreatedTime = existing.CreatedTime + } + + // LastTransitionTime only advances when the status value (True/False) changes. + // A change in ObservedGeneration alone does not constitute a transition. + if existing.Status == target.Status && !existing.LastTransitionTime.IsZero() { + target.LastTransitionTime = existing.LastTransitionTime + } else { + target.LastTransitionTime = transitionTime + } + + target.LastUpdatedTime = lastUpdatedTime + if existing.Status == target.Status && existing.ObservedGeneration == target.ObservedGeneration { - if !existing.CreatedTime.IsZero() { - target.CreatedTime = existing.CreatedTime - } - if !existing.LastTransitionTime.IsZero() { - target.LastTransitionTime = existing.LastTransitionTime - } - if !existing.LastUpdatedTime.IsZero() { - target.LastUpdatedTime = existing.LastUpdatedTime - } if target.Reason == nil && existing.Reason != nil { target.Reason = existing.Reason } if target.Message == nil && existing.Message != nil { target.Message = existing.Message } - return - } - - if !existing.CreatedTime.IsZero() { - target.CreatedTime = existing.CreatedTime } - target.LastTransitionTime = now - target.LastUpdatedTime = now } diff --git a/pkg/services/status_aggregation_test.go b/pkg/services/status_aggregation_test.go index 81f8e1b..f5f1185 100644 --- a/pkg/services/status_aggregation_test.go +++ b/pkg/services/status_aggregation_test.go @@ -1,12 +1,464 @@ package services import ( + "context" + "encoding/json" "testing" "time" + "gorm.io/datatypes" + "github.com/openshift-hyperfleet/hyperfleet-api/pkg/api" ) +// makeConditionsJSON marshals a slice of {Type, Status} pairs into datatypes.JSON. +func makeConditionsJSON(t *testing.T, conditions []struct{ Type, Status string }) datatypes.JSON { + t.Helper() + b, err := json.Marshal(conditions) + if err != nil { + t.Fatalf("failed to marshal conditions: %v", err) + } + return datatypes.JSON(b) +} + +// makeAdapterStatus builds an AdapterStatus with the given fields. +func makeAdapterStatus( + adapter string, gen int32, lastReportTime *time.Time, conditionsJSON datatypes.JSON, +) *api.AdapterStatus { + return &api.AdapterStatus{ + Adapter: adapter, + ObservedGeneration: gen, + LastReportTime: lastReportTime, + Conditions: conditionsJSON, + } +} + +func ptr(t time.Time) *time.Time { return &t } + +func TestComputeReadyLastUpdated_NotReady_NoAdapters(t *testing.T) { + now := time.Now() + // When isReady=false and a required adapter has no stored status, fall back to now. + result := computeReadyLastUpdated(context.Background(),nil, []string{"dns"}, 1, now, false) + if !result.Equal(now) { + t.Errorf("expected now (missing adapter fallback), got %v", result) + } +} + +func TestComputeReadyLastUpdated_NotReady_WithAdapters(t *testing.T) { + now := time.Now() + older := now.Add(-30 * time.Second) + newer := now.Add(-10 * time.Second) + + // Both required adapters present; one is False → isReady=false. + // LUT must be min(LRTs) = older, not now. + statuses := api.AdapterStatusList{ + makeAdapterStatus("dns", 1, ptr(older), makeConditionsJSON(t, []struct{ Type, Status string }{ + {api.ConditionTypeAvailable, string(api.AdapterConditionFalse)}, + })), + makeAdapterStatus("validator", 1, ptr(newer), makeConditionsJSON(t, []struct{ Type, Status string }{ + {api.ConditionTypeAvailable, string(api.AdapterConditionTrue)}, + })), + } + + result := computeReadyLastUpdated(context.Background(),statuses, []string{"dns", "validator"}, 1, now, false) + if !result.Equal(older) { + t.Errorf("expected min(LRTs)=%v, got %v", older, result) + } +} + +func TestComputeReadyLastUpdated_MissingAdapter(t *testing.T) { + now := time.Now() + statuses := api.AdapterStatusList{ + makeAdapterStatus("validator", 1, ptr(now.Add(-5*time.Second)), makeConditionsJSON(t, []struct{ Type, Status string }{ + {api.ConditionTypeAvailable, string(api.AdapterConditionTrue)}, + })), + } + // "dns" is required but not in the list → safety fallback to now. + result := computeReadyLastUpdated(context.Background(),statuses, []string{"validator", "dns"}, 1, now, true) + if !result.Equal(now) { + t.Errorf("expected now (missing adapter), got %v", result) + } +} + +func TestComputeReadyLastUpdated_NilLastReportTime(t *testing.T) { + now := time.Now() + statuses := api.AdapterStatusList{ + makeAdapterStatus("dns", 1, nil, makeConditionsJSON(t, []struct{ Type, Status string }{ + {api.ConditionTypeAvailable, string(api.AdapterConditionTrue)}, + })), + } + result := computeReadyLastUpdated(context.Background(),statuses, []string{"dns"}, 1, now, true) + if !result.Equal(now) { + t.Errorf("expected now (nil LastReportTime), got %v", result) + } +} + +func TestComputeReadyLastUpdated_WrongGeneration(t *testing.T) { + now := time.Now() + reportTime := now.Add(-10 * time.Second) + statuses := api.AdapterStatusList{ + // ObservedGeneration=1 but resourceGeneration=2 — skipped. + makeAdapterStatus("dns", 1, ptr(reportTime), makeConditionsJSON(t, []struct{ Type, Status string }{ + {api.ConditionTypeAvailable, string(api.AdapterConditionTrue)}, + })), + } + // All adapters skipped → minTime is nil → fallback to now. + result := computeReadyLastUpdated(context.Background(),statuses, []string{"dns"}, 2, now, true) + if !result.Equal(now) { + t.Errorf("expected now (wrong generation), got %v", result) + } +} + +func TestComputeReadyLastUpdated_AvailableFalse(t *testing.T) { + now := time.Now() + reportTime := now.Add(-10 * time.Second) + statuses := api.AdapterStatusList{ + makeAdapterStatus("dns", 1, ptr(reportTime), makeConditionsJSON(t, []struct{ Type, Status string }{ + {api.ConditionTypeAvailable, string(api.AdapterConditionFalse)}, + })), + } + // Available=False → skipped → fallback to now. + result := computeReadyLastUpdated(context.Background(),statuses, []string{"dns"}, 1, now, true) + if !result.Equal(now) { + t.Errorf("expected now (Available=False), got %v", result) + } +} + +func TestComputeReadyLastUpdated_SingleQualifyingAdapter(t *testing.T) { + now := time.Now() + reportTime := now.Add(-30 * time.Second) + statuses := api.AdapterStatusList{ + makeAdapterStatus("dns", 1, ptr(reportTime), makeConditionsJSON(t, []struct{ Type, Status string }{ + {api.ConditionTypeAvailable, string(api.AdapterConditionTrue)}, + })), + } + result := computeReadyLastUpdated(context.Background(),statuses, []string{"dns"}, 1, now, true) + if !result.Equal(reportTime) { + t.Errorf("expected %v, got %v", reportTime, result) + } +} + +func TestComputeReadyLastUpdated_MultipleAdapters_ReturnsMinimum(t *testing.T) { + now := time.Now() + older := now.Add(-60 * time.Second) + newer := now.Add(-10 * time.Second) + + statuses := api.AdapterStatusList{ + makeAdapterStatus("validator", 2, ptr(newer), makeConditionsJSON(t, []struct{ Type, Status string }{ + {api.ConditionTypeAvailable, string(api.AdapterConditionTrue)}, + })), + makeAdapterStatus("dns", 2, ptr(older), makeConditionsJSON(t, []struct{ Type, Status string }{ + {api.ConditionTypeAvailable, string(api.AdapterConditionTrue)}, + })), + } + result := computeReadyLastUpdated(context.Background(),statuses, []string{"validator", "dns"}, 2, now, true) + if !result.Equal(older) { + t.Errorf("expected minimum timestamp %v, got %v", older, result) + } +} + +// TestBuildSyntheticConditions_ReadyLastUpdatedThreaded verifies the full chain: +// when Ready=True, Ready.LastUpdatedTime equals the adapter's LastReportTime, +// not the evaluation time. +func TestBuildSyntheticConditions_ReadyLastUpdatedThreaded(t *testing.T) { + now := time.Now() + reportTime := now.Add(-30 * time.Second) + adapterStatuses := api.AdapterStatusList{ + makeAdapterStatus("dns", 1, ptr(reportTime), makeConditionsJSON(t, []struct{ Type, Status string }{ + {api.ConditionTypeAvailable, string(api.AdapterConditionTrue)}, + })), + } + requiredAdapters := []string{"dns"} + resourceGeneration := int32(1) + + _, readyCondition := BuildSyntheticConditions( + context.Background(), []byte("[]"), adapterStatuses, requiredAdapters, resourceGeneration, now, now, false, + ) + + if !readyCondition.LastUpdatedTime.Equal(reportTime) { + t.Errorf("Ready.LastUpdatedTime = %v, want reportTime %v", + readyCondition.LastUpdatedTime, reportTime) + } +} + +// TestBuildSyntheticConditions_AvailableLastUpdatedTime_Stable verifies that +// Available's LastUpdatedTime is updated to min_lut (the adapter's LastReportTime) +// when all_at_X=true and the status stays True. Per spec §5.2, lut=min_lut for +// all cases except True→False transitions. +func TestBuildSyntheticConditions_AvailableLastUpdatedTime_Stable(t *testing.T) { + originalLastUpdated := time.Now().Add(-5 * time.Minute) + now := time.Now() + reportTime := now.Add(-10 * time.Second) + + adapterStatuses := api.AdapterStatusList{ + makeAdapterStatus("dns", 1, ptr(reportTime), makeConditionsJSON(t, []struct{ Type, Status string }{ + {api.ConditionTypeAvailable, string(api.AdapterConditionTrue)}, + })), + } + requiredAdapters := []string{"dns"} + resourceGeneration := int32(1) + + // Simulate an existing Available condition. + existingConditions := []api.ResourceCondition{ + { + Type: api.ConditionTypeAvailable, + Status: api.ConditionTrue, + ObservedGeneration: 1, + LastUpdatedTime: originalLastUpdated, + LastTransitionTime: originalLastUpdated, + CreatedTime: originalLastUpdated, + }, + } + existingJSON, err := json.Marshal(existingConditions) + if err != nil { + t.Fatalf("failed to marshal existing conditions: %v", err) + } + + availableCondition, _ := BuildSyntheticConditions( + context.Background(), existingJSON, adapterStatuses, requiredAdapters, resourceGeneration, now, now, false) + + // Per spec §5.2: when all_at_X=true and status stays True, lut=min_lut=adapter's LastReportTime. + if !availableCondition.LastUpdatedTime.Equal(reportTime) { + t.Errorf("Available.LastUpdatedTime = %v, want %v (min_lut from adapter's LastReportTime)", + availableCondition.LastUpdatedTime, reportTime) + } +} + +// TestBuildSyntheticConditions_AvailableLastUpdatedTime_UpdatesOnChange verifies that +// on a True→False transition, Available's LastUpdatedTime is set to the triggering +// adapter's observed_time (not now), per spec. +func TestBuildSyntheticConditions_AvailableLastUpdatedTime_UpdatesOnChange(t *testing.T) { + originalLastUpdated := time.Now().Add(-5 * time.Minute) + now := time.Now() + adapterReportTime := now.Add(-10 * time.Second) + + // Adapter now reports Available=False (changed from True). + adapterStatuses := api.AdapterStatusList{ + makeAdapterStatus("dns", 1, ptr(adapterReportTime), makeConditionsJSON(t, []struct{ Type, Status string }{ + {api.ConditionTypeAvailable, string(api.AdapterConditionFalse)}, + })), + } + requiredAdapters := []string{"dns"} + resourceGeneration := int32(1) + + existingConditions := []api.ResourceCondition{ + { + Type: api.ConditionTypeAvailable, + Status: api.ConditionTrue, // was True, now False + ObservedGeneration: 1, + LastUpdatedTime: originalLastUpdated, + LastTransitionTime: originalLastUpdated, + CreatedTime: originalLastUpdated, + }, + } + existingJSON, err := json.Marshal(existingConditions) + if err != nil { + t.Fatalf("failed to marshal existing conditions: %v", err) + } + + // observedTime = adapter's LastReportTime (the triggering adapter's observed_time). + availableCondition, _ := BuildSyntheticConditions( + context.Background(), existingJSON, adapterStatuses, requiredAdapters, + resourceGeneration, now, adapterReportTime, false) + + // Per spec: True→False transition → lut=observed_time (adapter's report time). + if !availableCondition.LastUpdatedTime.Equal(adapterReportTime) { + t.Errorf("Available.LastUpdatedTime = %v, want %v (observed_time on True→False transition)", + availableCondition.LastUpdatedTime, adapterReportTime) + } +} + +// TestBuildSyntheticConditions_Available_MixedGenerations verifies that Available stays False +// when required adapters report at different observed generations (all_at_X=false per spec §5.2). +// With no existing Available state and mixed adapter generations, the condition defaults to False. +func TestBuildSyntheticConditions_Available_MixedGenerations(t *testing.T) { + now := time.Now() + adapterStatuses := api.AdapterStatusList{ + makeAdapterStatus("dns", 1, ptr(now.Add(-10*time.Second)), makeConditionsJSON(t, []struct{ Type, Status string }{ + {api.ConditionTypeAvailable, string(api.AdapterConditionTrue)}, + })), + makeAdapterStatus("validator", 2, ptr(now.Add(-5*time.Second)), makeConditionsJSON(t, []struct{ Type, Status string }{ + {api.ConditionTypeAvailable, string(api.AdapterConditionTrue)}, + })), + } + + // No existing Available condition; adapters at different generations → all_at_X=false. + availableCondition, _ := BuildSyntheticConditions( + context.Background(), []byte("[]"), adapterStatuses, []string{"dns", "validator"}, 2, now, now, false) + + // Per spec §5.2: all_at_X=false → no change. With no existing state, defaults to False. + if availableCondition.Status != api.ConditionFalse { + t.Errorf("Available.Status = %v, want False (all_at_X=false with mixed generations, no existing state)", + availableCondition.Status) + } + if availableCondition.ObservedGeneration != 2 { + t.Errorf("Available.ObservedGeneration = %v, want 2 (resource generation when all_at_X=false)", + availableCondition.ObservedGeneration) + } +} + +// TestBuildSyntheticConditions_AvailableLastUpdatedTime_UpdatesOnGenerationChange verifies that +// Available's LastUpdatedTime is set to min_lut (the adapter's LastReportTime) when the observed +// generation advances and status stays True. Per spec §5.2, lut=min_lut when all_at_X=true and +// the status stays True. +func TestBuildSyntheticConditions_AvailableLastUpdatedTime_UpdatesOnGenerationChange(t *testing.T) { + originalLastUpdated := time.Now().Add(-5 * time.Minute) + now := time.Now() + reportTime := now.Add(-10 * time.Second) + + // Adapter advances from gen1 to gen2; status stays True. + adapterStatuses := api.AdapterStatusList{ + makeAdapterStatus("dns", 2, ptr(reportTime), makeConditionsJSON(t, []struct{ Type, Status string }{ + {api.ConditionTypeAvailable, string(api.AdapterConditionTrue)}, + })), + } + + existingConditions := []api.ResourceCondition{ + { + Type: api.ConditionTypeAvailable, + Status: api.ConditionTrue, + ObservedGeneration: 1, // was gen1, now gen2 + LastUpdatedTime: originalLastUpdated, + LastTransitionTime: originalLastUpdated, + CreatedTime: originalLastUpdated, + }, + } + existingJSON, err := json.Marshal(existingConditions) + if err != nil { + t.Fatalf("failed to marshal existing conditions: %v", err) + } + + availableCondition, _ := BuildSyntheticConditions( + context.Background(), existingJSON, adapterStatuses, []string{"dns"}, 2, now, now, false) + + if availableCondition.Status != api.ConditionTrue { + t.Errorf("Available.Status = %v, want True", availableCondition.Status) + } + // Per spec §5.2: all_at_X=true, stays True → lut=min_lut=adapter's LastReportTime. + if !availableCondition.LastUpdatedTime.Equal(reportTime) { + t.Errorf("Available.LastUpdatedTime = %v, want %v (min_lut from adapter's LastReportTime when generation advances)", + availableCondition.LastUpdatedTime, reportTime) + } +} + +func TestValidateMandatoryConditions_InvalidStatus(t *testing.T) { + // P2: Available status not in {True, False, Unknown} → reject. + conditions := []api.AdapterCondition{ + {Type: api.ConditionTypeAvailable, Status: "InvalidValue", LastTransitionTime: time.Now()}, + {Type: api.ConditionTypeApplied, Status: api.AdapterConditionTrue, LastTransitionTime: time.Now()}, + {Type: api.ConditionTypeHealth, Status: api.AdapterConditionTrue, LastTransitionTime: time.Now()}, + } + + errorType, conditionName := ValidateMandatoryConditions(conditions) + + if errorType != ConditionValidationErrorInvalidStatus { + t.Errorf("Expected errorType ConditionValidationErrorInvalidStatus, got: %s", errorType) + } + if conditionName != api.ConditionTypeAvailable { + t.Errorf("Expected conditionName %s, got: %s", api.ConditionTypeAvailable, conditionName) + } +} + +func TestValidateMandatoryConditions_InvalidStatusApplied(t *testing.T) { + // P2: Applied status not in {True, False, Unknown} → reject. + conditions := []api.AdapterCondition{ + {Type: api.ConditionTypeAvailable, Status: api.AdapterConditionTrue, LastTransitionTime: time.Now()}, + {Type: api.ConditionTypeApplied, Status: "bad-value", LastTransitionTime: time.Now()}, + {Type: api.ConditionTypeHealth, Status: api.AdapterConditionTrue, LastTransitionTime: time.Now()}, + } + + errorType, conditionName := ValidateMandatoryConditions(conditions) + + if errorType != ConditionValidationErrorInvalidStatus { + t.Errorf("Expected errorType ConditionValidationErrorInvalidStatus, got: %s", errorType) + } + if conditionName != api.ConditionTypeApplied { + t.Errorf("Expected conditionName %s, got: %s", api.ConditionTypeApplied, conditionName) + } +} + +// TestBuildSyntheticConditions_Available_AllAtX_True verifies that Available becomes True +// when all required adapters are at the same generation X and all report True (spec §5.2 T6). +func TestBuildSyntheticConditions_Available_AllAtX_True(t *testing.T) { + now := time.Now() + reportTime1 := now.Add(-20 * time.Second) + reportTime2 := now.Add(-10 * time.Second) + + // Both adapters at gen=1 (same X), both True. + adapterStatuses := api.AdapterStatusList{ + makeAdapterStatus("dns", 1, ptr(reportTime1), makeConditionsJSON(t, []struct{ Type, Status string }{ + {api.ConditionTypeAvailable, string(api.AdapterConditionTrue)}, + })), + makeAdapterStatus("validator", 1, ptr(reportTime2), makeConditionsJSON(t, []struct{ Type, Status string }{ + {api.ConditionTypeAvailable, string(api.AdapterConditionTrue)}, + })), + } + + availableCondition, _ := BuildSyntheticConditions( + context.Background(), []byte("[]"), adapterStatuses, []string{"dns", "validator"}, 2, now, now, false) + + // Per spec §5.2: all_at_X=true for X=1, all True → Available=True@1, lut=min_lut. + if availableCondition.Status != api.ConditionTrue { + t.Errorf("Available.Status = %v, want True (all_at_X=true, all True)", availableCondition.Status) + } + if availableCondition.ObservedGeneration != 1 { + t.Errorf("Available.ObservedGeneration = %v, want 1 (the common generation X)", availableCondition.ObservedGeneration) + } + // min_lut = min(reportTime1, reportTime2) = reportTime1 (earlier). + if !availableCondition.LastUpdatedTime.Equal(reportTime1) { + t.Errorf("Available.LastUpdatedTime = %v, want %v (min_lut)", availableCondition.LastUpdatedTime, reportTime1) + } +} + +// TestBuildSyntheticConditions_Available_PreservesExistingTrueOnMixedGens verifies that +// when adapters are at different generations (all_at_X=false), an existing Available=True +// is preserved unchanged (spec §5.2 "True | false → no change"). +func TestBuildSyntheticConditions_Available_PreservesExistingTrueOnMixedGens(t *testing.T) { + originalLastUpdated := time.Now().Add(-2 * time.Minute) + now := time.Now() + + // dns at gen=1, validator at gen=2 → all_at_X=false. + adapterStatuses := api.AdapterStatusList{ + makeAdapterStatus("dns", 1, ptr(now.Add(-30*time.Second)), makeConditionsJSON(t, []struct{ Type, Status string }{ + {api.ConditionTypeAvailable, string(api.AdapterConditionTrue)}, + })), + makeAdapterStatus("validator", 2, ptr(now.Add(-10*time.Second)), + makeConditionsJSON(t, []struct{ Type, Status string }{ + {api.ConditionTypeAvailable, string(api.AdapterConditionTrue)}, + })), + } + + existingConditions := []api.ResourceCondition{ + { + Type: api.ConditionTypeAvailable, + Status: api.ConditionTrue, + ObservedGeneration: 1, + LastUpdatedTime: originalLastUpdated, + LastTransitionTime: originalLastUpdated, + CreatedTime: originalLastUpdated, + }, + } + existingJSON, err := json.Marshal(existingConditions) + if err != nil { + t.Fatalf("failed to marshal existing conditions: %v", err) + } + + availableCondition, _ := BuildSyntheticConditions( + context.Background(), existingJSON, adapterStatuses, []string{"dns", "validator"}, 2, now, now, false) + + // Per spec §5.2: all_at_X=false → no change → preserve existing True@gen1. + if availableCondition.Status != api.ConditionTrue { + t.Errorf("Available.Status = %v, want True (preserved, all_at_X=false)", availableCondition.Status) + } + if availableCondition.ObservedGeneration != 1 { + t.Errorf("Available.ObservedGeneration = %v, want 1 (preserved)", availableCondition.ObservedGeneration) + } + if !availableCondition.LastUpdatedTime.Equal(originalLastUpdated) { + t.Errorf("Available.LastUpdatedTime = %v, want %v (preserved when all_at_X=false)", + availableCondition.LastUpdatedTime, originalLastUpdated) + } +} + func TestMapAdapterToConditionType(t *testing.T) { tests := []struct { adapter string @@ -202,3 +654,398 @@ func TestValidateMandatoryConditions_EmptyConditionType(t *testing.T) { t.Errorf("Expected conditionName '', got: %s", conditionName) } } + +// TestBuildSyntheticConditions_ZeroAdapters_BothTrue verifies that when no required adapters +// are configured, both Available and Ready are trivially True (spec: zero required → satisfied). +func TestBuildSyntheticConditions_ZeroAdapters_BothTrue(t *testing.T) { + now := time.Now() + + available, ready := BuildSyntheticConditions( + context.Background(), nil, nil, nil, 1, now, now, false) + + if available.Status != api.ConditionTrue { + t.Errorf("Available.Status = %v, want True (zero required adapters → trivially satisfied)", available.Status) + } + if ready.Status != api.ConditionTrue { + t.Errorf("Ready.Status = %v, want True (zero required adapters → trivially satisfied)", ready.Status) + } + if available.ObservedGeneration != 1 { + t.Errorf("Available.ObservedGeneration = %v, want 1", available.ObservedGeneration) + } + if ready.ObservedGeneration != 1 { + t.Errorf("Ready.ObservedGeneration = %v, want 1", ready.ObservedGeneration) + } +} + +// TestBuildSyntheticConditions_LifecycleChange_AvailableFrozen verifies that on a lifecycle +// change (Create/Replace), Available is completely frozen and Ready resets with lut=now. +// False→False transition: Ready.ltt is preserved from existing. +func TestBuildSyntheticConditions_LifecycleChange_AvailableFrozen(t *testing.T) { + fixedTime := time.Now().Add(-5 * time.Minute) + now := time.Now() + + existingConditions := []api.ResourceCondition{ + { + Type: api.ConditionTypeAvailable, + Status: api.ConditionFalse, + ObservedGeneration: 1, + LastUpdatedTime: fixedTime, + LastTransitionTime: fixedTime, + CreatedTime: fixedTime, + }, + { + Type: api.ConditionTypeReady, + Status: api.ConditionFalse, + ObservedGeneration: 1, + LastUpdatedTime: fixedTime, + LastTransitionTime: fixedTime, + CreatedTime: fixedTime, + }, + } + existingJSON, err := json.Marshal(existingConditions) + if err != nil { + t.Fatalf("failed to marshal existing conditions: %v", err) + } + + // Adapter reports (should be ignored for Available on lifecycle change). + adapterStatuses := api.AdapterStatusList{ + makeAdapterStatus("dns", 2, ptr(now.Add(-1*time.Second)), makeConditionsJSON(t, []struct{ Type, Status string }{ + {api.ConditionTypeAvailable, string(api.AdapterConditionTrue)}, + })), + } + + available, ready := BuildSyntheticConditions( + context.Background(), existingJSON, adapterStatuses, []string{"dns"}, 2, now, now, true) + + // Available must be completely frozen (unchanged from existing). + if available.Status != api.ConditionFalse { + t.Errorf("Available.Status = %v, want False (frozen on lifecycle change)", available.Status) + } + if available.ObservedGeneration != 1 { + t.Errorf("Available.ObservedGeneration = %v, want 1 (frozen)", available.ObservedGeneration) + } + if !available.LastUpdatedTime.Equal(fixedTime) { + t.Errorf("Available.LastUpdatedTime = %v, want %v (frozen)", available.LastUpdatedTime, fixedTime) + } + if !available.LastTransitionTime.Equal(fixedTime) { + t.Errorf("Available.LastTransitionTime = %v, want %v (frozen)", available.LastTransitionTime, fixedTime) + } + + // Ready must reset: status=False at new generation, lut=now. + if ready.Status != api.ConditionFalse { + t.Errorf("Ready.Status = %v, want False", ready.Status) + } + if ready.ObservedGeneration != 2 { + t.Errorf("Ready.ObservedGeneration = %v, want 2 (new generation)", ready.ObservedGeneration) + } + if !ready.LastUpdatedTime.Equal(now) { + t.Errorf("Ready.LastUpdatedTime = %v, want now=%v (lifecycle reset)", ready.LastUpdatedTime, now) + } + // False→False: ltt preserved from existing. + if !ready.LastTransitionTime.Equal(fixedTime) { + t.Errorf("Ready.LastTransitionTime = %v, want %v (preserved on False→False)", ready.LastTransitionTime, fixedTime) + } + // CreatedTime preserved from existing. + if !ready.CreatedTime.Equal(fixedTime) { + t.Errorf("Ready.CreatedTime = %v, want %v (preserved)", ready.CreatedTime, fixedTime) + } +} + +// TestBuildSyntheticConditions_LifecycleChange_ReadyTrueToFalse verifies that on a lifecycle +// change when Ready was True, Ready resets to False with lut=now and ltt=observedTime (now). +func TestBuildSyntheticConditions_LifecycleChange_ReadyTrueToFalse(t *testing.T) { + fixedTime := time.Now().Add(-5 * time.Minute) + now := time.Now() + + existingConditions := []api.ResourceCondition{ + { + Type: api.ConditionTypeAvailable, + Status: api.ConditionTrue, + ObservedGeneration: 1, + LastUpdatedTime: fixedTime, + LastTransitionTime: fixedTime, + CreatedTime: fixedTime, + }, + { + Type: api.ConditionTypeReady, + Status: api.ConditionTrue, + ObservedGeneration: 1, + LastUpdatedTime: fixedTime, + LastTransitionTime: fixedTime, + CreatedTime: fixedTime, + }, + } + existingJSON, err := json.Marshal(existingConditions) + if err != nil { + t.Fatalf("failed to marshal existing conditions: %v", err) + } + + available, ready := BuildSyntheticConditions( + context.Background(), existingJSON, nil, []string{"dns"}, 2, now, now, true) + + // Available frozen at True@1. + if available.Status != api.ConditionTrue { + t.Errorf("Available.Status = %v, want True (frozen)", available.Status) + } + if available.ObservedGeneration != 1 { + t.Errorf("Available.ObservedGeneration = %v, want 1 (frozen)", available.ObservedGeneration) + } + + // Ready: True→False, lut=now, ltt=observedTime=now. + if ready.Status != api.ConditionFalse { + t.Errorf("Ready.Status = %v, want False", ready.Status) + } + if ready.ObservedGeneration != 2 { + t.Errorf("Ready.ObservedGeneration = %v, want 2", ready.ObservedGeneration) + } + if !ready.LastUpdatedTime.Equal(now) { + t.Errorf("Ready.LastUpdatedTime = %v, want now=%v", ready.LastUpdatedTime, now) + } + // True→False: ltt=observedTime=now. + if !ready.LastTransitionTime.Equal(now) { + t.Errorf("Ready.LastTransitionTime = %v, want now=%v (True→False transition)", ready.LastTransitionTime, now) + } + // CreatedTime preserved from existing. + if !ready.CreatedTime.Equal(fixedTime) { + t.Errorf("Ready.CreatedTime = %v, want %v (preserved)", ready.CreatedTime, fixedTime) + } +} + +// TestBuildSyntheticConditions_Available_TrueToFalse_Ltt verifies that on a True→False +// transition, Available's LastTransitionTime is set to observedTime (spec: ltt=obs_time). +// Complements TestBuildSyntheticConditions_AvailableLastUpdatedTime_UpdatesOnChange which +// only checked LastUpdatedTime. +func TestBuildSyntheticConditions_Available_TrueToFalse_Ltt(t *testing.T) { + fixedLtt := time.Now().Add(-5 * time.Minute) + now := time.Now() + adapterReportTime := now.Add(-10 * time.Second) + + adapterStatuses := api.AdapterStatusList{ + makeAdapterStatus("dns", 1, ptr(adapterReportTime), makeConditionsJSON(t, []struct{ Type, Status string }{ + {api.ConditionTypeAvailable, string(api.AdapterConditionFalse)}, + })), + } + + existingConditions := []api.ResourceCondition{ + { + Type: api.ConditionTypeAvailable, + Status: api.ConditionTrue, + ObservedGeneration: 1, + LastUpdatedTime: fixedLtt, + LastTransitionTime: fixedLtt, + CreatedTime: fixedLtt, + }, + } + existingJSON, err := json.Marshal(existingConditions) + if err != nil { + t.Fatalf("failed to marshal existing conditions: %v", err) + } + + availableCondition, _ := BuildSyntheticConditions( + context.Background(), existingJSON, adapterStatuses, []string{"dns"}, 1, now, adapterReportTime, false) + + if availableCondition.Status != api.ConditionFalse { + t.Errorf("Available.Status = %v, want False", availableCondition.Status) + } + // True→False: ltt=observedTime=adapterReportTime. + if !availableCondition.LastTransitionTime.Equal(adapterReportTime) { + t.Errorf("Available.LastTransitionTime = %v, want %v (observedTime on True→False)", + availableCondition.LastTransitionTime, adapterReportTime) + } + // True→False: lut=observedTime=adapterReportTime. + if !availableCondition.LastUpdatedTime.Equal(adapterReportTime) { + t.Errorf("Available.LastUpdatedTime = %v, want %v (observedTime on True→False)", + availableCondition.LastUpdatedTime, adapterReportTime) + } +} + +// TestBuildSyntheticConditions_Available_FalseToTrue_Ltt verifies that on a False→True +// transition, Available's LastTransitionTime is set to observedTime and LastUpdatedTime +// is set to min(LRTs) (spec: ltt=obs_time, lut=min_lut on False→True). +func TestBuildSyntheticConditions_Available_FalseToTrue_Ltt(t *testing.T) { + fixedLtt := time.Now().Add(-5 * time.Minute) + now := time.Now() + reportTime1 := now.Add(-20 * time.Second) // earlier — will be min(LRTs) + reportTime2 := now.Add(-10 * time.Second) + observedTime := now.Add(-5 * time.Second) // distinct from both LRTs + + adapterStatuses := api.AdapterStatusList{ + makeAdapterStatus("dns", 1, ptr(reportTime1), makeConditionsJSON(t, []struct{ Type, Status string }{ + {api.ConditionTypeAvailable, string(api.AdapterConditionTrue)}, + })), + makeAdapterStatus("validator", 1, ptr(reportTime2), makeConditionsJSON(t, []struct{ Type, Status string }{ + {api.ConditionTypeAvailable, string(api.AdapterConditionTrue)}, + })), + } + + existingConditions := []api.ResourceCondition{ + { + Type: api.ConditionTypeAvailable, + Status: api.ConditionFalse, + ObservedGeneration: 1, + LastUpdatedTime: fixedLtt, + LastTransitionTime: fixedLtt, + CreatedTime: fixedLtt, + }, + } + existingJSON, err := json.Marshal(existingConditions) + if err != nil { + t.Fatalf("failed to marshal existing conditions: %v", err) + } + + availableCondition, _ := BuildSyntheticConditions( + context.Background(), existingJSON, adapterStatuses, []string{"dns", "validator"}, 1, now, observedTime, false) + + if availableCondition.Status != api.ConditionTrue { + t.Errorf("Available.Status = %v, want True (all at same gen, all True)", availableCondition.Status) + } + // False→True: ltt=observedTime (status transition). + if !availableCondition.LastTransitionTime.Equal(observedTime) { + t.Errorf("Available.LastTransitionTime = %v, want %v (observedTime on False→True)", + availableCondition.LastTransitionTime, observedTime) + } + // False→True: lut=min(LRTs)=reportTime1. + if !availableCondition.LastUpdatedTime.Equal(reportTime1) { + t.Errorf("Available.LastUpdatedTime = %v, want %v (min_lut on False→True)", + availableCondition.LastUpdatedTime, reportTime1) + } +} + +// TestBuildSyntheticConditions_Available_TrueToTrue_LttPreserved verifies that when +// Available stays True, LastTransitionTime is preserved from the existing condition +// (spec: ltt=— on True→True, no change). +func TestBuildSyntheticConditions_Available_TrueToTrue_LttPreserved(t *testing.T) { + fixedLtt := time.Now().Add(-5 * time.Minute) + now := time.Now() + reportTime := now.Add(-10 * time.Second) + + adapterStatuses := api.AdapterStatusList{ + makeAdapterStatus("dns", 1, ptr(reportTime), makeConditionsJSON(t, []struct{ Type, Status string }{ + {api.ConditionTypeAvailable, string(api.AdapterConditionTrue)}, + })), + } + + existingConditions := []api.ResourceCondition{ + { + Type: api.ConditionTypeAvailable, + Status: api.ConditionTrue, + ObservedGeneration: 1, + LastUpdatedTime: fixedLtt, + LastTransitionTime: fixedLtt, + CreatedTime: fixedLtt, + }, + } + existingJSON, err := json.Marshal(existingConditions) + if err != nil { + t.Fatalf("failed to marshal existing conditions: %v", err) + } + + availableCondition, _ := BuildSyntheticConditions( + context.Background(), existingJSON, adapterStatuses, []string{"dns"}, 1, now, now, false) + + if availableCondition.Status != api.ConditionTrue { + t.Errorf("Available.Status = %v, want True", availableCondition.Status) + } + // True→True: ltt preserved from existing. + if !availableCondition.LastTransitionTime.Equal(fixedLtt) { + t.Errorf("Available.LastTransitionTime = %v, want %v (preserved on True→True)", + availableCondition.LastTransitionTime, fixedLtt) + } + // True→True: lut=min(LRTs)=reportTime. + if !availableCondition.LastUpdatedTime.Equal(reportTime) { + t.Errorf("Available.LastUpdatedTime = %v, want %v (min_lut on True→True)", + availableCondition.LastUpdatedTime, reportTime) + } +} + +// TestBuildSyntheticConditions_Available_FalseToFalse_LttPreserved verifies that when +// Available stays False with all_at_X=true (consistent gen, adapter reports False), +// LastTransitionTime is preserved from the existing condition (spec: ltt=— on False→False). +// This is distinct from the all_at_X=false path (TestBuildSyntheticConditions_Available_MixedGenerations) +// where buildAvailableCondition is never reached. +func TestBuildSyntheticConditions_Available_FalseToFalse_LttPreserved(t *testing.T) { + fixedLtt := time.Now().Add(-5 * time.Minute) + now := time.Now() + reportTime := now.Add(-10 * time.Second) + + // Single adapter at gen=1 reports False → all_at_X=true, consistent=true, newStatus=False. + adapterStatuses := api.AdapterStatusList{ + makeAdapterStatus("dns", 1, ptr(reportTime), makeConditionsJSON(t, []struct{ Type, Status string }{ + {api.ConditionTypeAvailable, string(api.AdapterConditionFalse)}, + })), + } + + existingConditions := []api.ResourceCondition{ + { + Type: api.ConditionTypeAvailable, + Status: api.ConditionFalse, + ObservedGeneration: 1, + LastUpdatedTime: fixedLtt, + LastTransitionTime: fixedLtt, + CreatedTime: fixedLtt, + }, + } + existingJSON, err := json.Marshal(existingConditions) + if err != nil { + t.Fatalf("failed to marshal existing conditions: %v", err) + } + + availableCondition, _ := BuildSyntheticConditions( + context.Background(), existingJSON, adapterStatuses, []string{"dns"}, 1, now, now, false) + + if availableCondition.Status != api.ConditionFalse { + t.Errorf("Available.Status = %v, want False", availableCondition.Status) + } + // False→False: ltt preserved from existing (no status transition occurred). + if !availableCondition.LastTransitionTime.Equal(fixedLtt) { + t.Errorf("Available.LastTransitionTime = %v, want %v (preserved on False→False, all_at_X=true path)", + availableCondition.LastTransitionTime, fixedLtt) + } +} + +// TestComputeReadyLastUpdated_NotReady_NilLastReportTime verifies that when isReady=false +// and a required adapter has nil LastReportTime, the function falls back to now. +// Complements TestComputeReadyLastUpdated_NilLastReportTime which only tests isReady=true. +func TestComputeReadyLastUpdated_NotReady_NilLastReportTime(t *testing.T) { + now := time.Now() + statuses := api.AdapterStatusList{ + makeAdapterStatus("dns", 1, nil, makeConditionsJSON(t, []struct{ Type, Status string }{ + {api.ConditionTypeAvailable, string(api.AdapterConditionFalse)}, + })), + } + result := computeReadyLastUpdated(context.Background(),statuses, []string{"dns"}, 1, now, false) + if !result.Equal(now) { + t.Errorf("expected now (nil LastReportTime, isReady=false), got %v", result) + } +} + +// TestBuildSyntheticConditions_ReadyStaysFalse_LutIsMinLRT verifies that when Ready stays +// False and all required adapters have real LRTs, Ready.LastUpdatedTime equals min(LRTs) +// rather than now (spec: lut=min(LRTs) when Ready stays False with real adapter LRTs). +func TestBuildSyntheticConditions_ReadyStaysFalse_LutIsMinLRT(t *testing.T) { + now := time.Now() + dnsLRT := now.Add(-30 * time.Second) // earlier — will be min(LRTs) + validatorLRT := now.Add(-10 * time.Second) // later + + // dns is Available=False → Ready cannot be True. + adapterStatuses := api.AdapterStatusList{ + makeAdapterStatus("dns", 1, ptr(dnsLRT), makeConditionsJSON(t, []struct{ Type, Status string }{ + {api.ConditionTypeAvailable, string(api.AdapterConditionFalse)}, + })), + makeAdapterStatus("validator", 1, ptr(validatorLRT), makeConditionsJSON(t, []struct{ Type, Status string }{ + {api.ConditionTypeAvailable, string(api.AdapterConditionTrue)}, + })), + } + + _, readyCondition := BuildSyntheticConditions( + context.Background(), []byte("[]"), adapterStatuses, []string{"dns", "validator"}, 1, now, now, false) + + if readyCondition.Status != api.ConditionFalse { + t.Errorf("Ready.Status = %v, want False (dns is Available=False)", readyCondition.Status) + } + // Ready stays False: lut=min(all LRTs)=dnsLRT. + if !readyCondition.LastUpdatedTime.Equal(dnsLRT) { + t.Errorf("Ready.LastUpdatedTime = %v, want %v (min(LRTs) when Ready stays False)", + readyCondition.LastUpdatedTime, dnsLRT) + } +} diff --git a/test/e2e-curl/01-initial-state/test.sh b/test/e2e-curl/01-initial-state/test.sh new file mode 100755 index 0000000..e02a57d --- /dev/null +++ b/test/e2e-curl/01-initial-state/test.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +# Test 01 — Initial state: cluster created, no adapter reports + +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "$SCRIPT_DIR/../common.sh" + +# ── Purpose ─────────────────────────────────────────────────────────────────── +echo -e "\n${BOLD}Test 01: Initial state — no adapter statuses${NC}" +echo "" +echo " A freshly created cluster must immediately expose Available=False and" +echo " Ready=False at generation 1 with valid timestamps. No adapter reports" +echo " have been posted — the API must bootstrap these conditions itself." +echo "" +echo -e " ${YELLOW}Starting state:${NC} Cluster just created, gen=1, no adapter statuses" +echo -e " ${YELLOW}Event:${NC} GET cluster immediately after creation (no events)" +echo -e " ${YELLOW}Expected:${NC} Available=False@gen1, Ready=False@gen1, timestamps set" +echo "" + +# ── Setup ───────────────────────────────────────────────────────────────────── +CLUSTER_NAME="tc01-$(rand_hex4)" +log_step "Creating cluster '$CLUSTER_NAME'" +CLUSTER=$(create_cluster "$CLUSTER_NAME") +CLUSTER_ID=$(echo "$CLUSTER" | jq -r '.id') +GENERATION=$(echo "$CLUSTER" | jq -r '.generation') +log_received "cluster id=$CLUSTER_ID generation=$GENERATION" + +# ── Event: GET immediately (no adapter statuses posted) ─────────────────────── +log_step "GET cluster — no adapter statuses have been posted" +CLUSTER=$(get_cluster "$CLUSTER_ID") +show_state "received" "$CLUSTER" + +# ── Validate ────────────────────────────────────────────────────────────────── +echo "" +echo " Available condition:" +AVAIL_STATUS=$(condition_field "$CLUSTER" Available status) +AVAIL_OBSGEN=$(condition_field "$CLUSTER" Available observed_generation) +AVAIL_UPDATED=$(condition_field "$CLUSTER" Available last_updated_time) +AVAIL_TRANSITION=$(condition_field "$CLUSTER" Available last_transition_time) + +assert_eq "Available.status" "False" "$AVAIL_STATUS" +assert_eq "Available.observed_generation" "1" "$AVAIL_OBSGEN" +assert_nonempty "Available.last_updated_time" "$AVAIL_UPDATED" +assert_nonempty "Available.last_transition_time" "$AVAIL_TRANSITION" + +echo "" +echo " Ready condition:" +READY_STATUS=$(condition_field "$CLUSTER" Ready status) +READY_OBSGEN=$(condition_field "$CLUSTER" Ready observed_generation) +READY_UPDATED=$(condition_field "$CLUSTER" Ready last_updated_time) +READY_TRANSITION=$(condition_field "$CLUSTER" Ready last_transition_time) + +assert_eq "Ready.status" "False" "$READY_STATUS" +assert_eq "Ready.observed_generation" "1" "$READY_OBSGEN" +assert_nonempty "Ready.last_updated_time" "$READY_UPDATED" +assert_nonempty "Ready.last_transition_time" "$READY_TRANSITION" + +# ── Conclusion ──────────────────────────────────────────────────────────────── +echo "" +echo -e "${BOLD}Conclusion:${NC}" +echo " The API correctly bootstraps both synthetic conditions at resource creation:" +echo " • Available=False because no adapter has reported yet — cannot be available" +echo " • Ready=False for the same reason — no adapter at current generation" +echo " • observed_generation=1 matches the cluster's creation generation" +echo " • Both timestamps are non-empty, establishing a baseline for future" +echo " change-detection checks (assert_changed relies on these being set)" +echo "" +test_summary diff --git a/test/e2e-curl/02-partial-adapters/test.sh b/test/e2e-curl/02-partial-adapters/test.sh new file mode 100755 index 0000000..eb86892 --- /dev/null +++ b/test/e2e-curl/02-partial-adapters/test.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +# Test 02 — Partial adapters: one required adapter reports True, other is still missing + +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "$SCRIPT_DIR/../common.sh" + +# ── Purpose ─────────────────────────────────────────────────────────────────── +echo -e "\n${BOLD}Test 02: Partial adapters — one required adapter True, one missing${NC}" +echo "" +echo " When only some required adapters have reported, the snapshot is" +echo " inconsistent (adapters not all at the same generation). Available must" +echo " stay unchanged. Ready stays False and refreshes its last_updated_time" +echo " so Sentinel knows the cluster is actively being reconciled." +echo "" +echo -e " ${YELLOW}Starting state:${NC} Available=False@gen1, Ready=False@gen1 (initial)" +echo -e " ${YELLOW}Event:${NC} POST ${ADAPTER2}@gen1=True (${ADAPTER1} still missing)" +echo -e " ${YELLOW}Expected:${NC} Available=False unchanged (guard fires), Ready=False," +echo -e " ${YELLOW} ${NC} Available timestamps frozen, Ready.LUT refreshed" +echo "" + +# ── Setup ───────────────────────────────────────────────────────────────────── +CLUSTER_NAME="tc02-$(rand_hex4)" +log_step "Creating cluster '$CLUSTER_NAME'" +CLUSTER=$(create_cluster "$CLUSTER_NAME") +CLUSTER_ID=$(echo "$CLUSTER" | jq -r '.id') +log_received "cluster id=$CLUSTER_ID" + +log_step "GET cluster — baseline (initial state)" +BASELINE=$(get_cluster "$CLUSTER_ID") +show_state "baseline" "$BASELINE" +BASELINE_AVAIL_UPDATED=$(condition_field "$BASELINE" Available last_updated_time) +BASELINE_AVAIL_TRANSITION=$(condition_field "$BASELINE" Available last_transition_time) +BASELINE_READY_UPDATED=$(condition_field "$BASELINE" Ready last_updated_time) +log_received "Available.LUT=$BASELINE_AVAIL_UPDATED" +log_received "Ready.LUT=$BASELINE_READY_UPDATED" + +# Brief pause so a timestamp change would be detectable +sleep 1 + +# ── Event ───────────────────────────────────────────────────────────────────── +log_step "POST ${ADAPTER2}@gen1=True (${ADAPTER1} is still missing)" +CODE=$(post_adapter_status "$CLUSTER_ID" "$ADAPTER2" 1 "True") +assert_http "${ADAPTER2} report accepted" "201" "$CODE" + +# ── Validate ────────────────────────────────────────────────────────────────── +log_step "GET cluster — post-event" +CLUSTER=$(get_cluster "$CLUSTER_ID") +show_state "after event" "$CLUSTER" + +echo "" +echo " Available condition (${ADAPTER1} missing → snapshot inconsistent → guard preserves):" +AVAIL_STATUS=$(condition_field "$CLUSTER" Available status) +AVAIL_OBSGEN=$(condition_field "$CLUSTER" Available observed_generation) +AVAIL_UPDATED=$(condition_field "$CLUSTER" Available last_updated_time) +AVAIL_TRANSITION=$(condition_field "$CLUSTER" Available last_transition_time) + +assert_eq "Available.status" "False" "$AVAIL_STATUS" +assert_eq "Available.observed_generation" "1" "$AVAIL_OBSGEN" +assert_eq "Available.last_updated_time preserved" "$BASELINE_AVAIL_UPDATED" "$AVAIL_UPDATED" +assert_eq "Available.last_transition_time preserved" "$BASELINE_AVAIL_TRANSITION" "$AVAIL_TRANSITION" + +echo "" +echo " Ready condition (always refreshes last_updated_time when False):" +READY_STATUS=$(condition_field "$CLUSTER" Ready status) +READY_OBSGEN=$(condition_field "$CLUSTER" Ready observed_generation) +READY_UPDATED=$(condition_field "$CLUSTER" Ready last_updated_time) + +assert_eq "Ready.status" "False" "$READY_STATUS" +assert_eq "Ready.observed_generation" "1" "$READY_OBSGEN" +assert_changed "Ready.last_updated_time refreshed" "$BASELINE_READY_UPDATED" "$READY_UPDATED" + +# ── Conclusion ──────────────────────────────────────────────────────────────── +echo "" +echo -e "${BOLD}Conclusion:${NC}" +echo " With ${ADAPTER1} still missing the snapshot is inconsistent — not all" +echo " required adapters have reported at the same generation — so the Available" +echo " guard fires and all Available timestamps are frozen unchanged." +echo " Ready=False always sets last_updated_time=now, which keeps Sentinel's" +echo " activity clock ticking even while the cluster is still converging." +echo "" +test_summary diff --git a/test/e2e-curl/03-all-adapters-ready/test.sh b/test/e2e-curl/03-all-adapters-ready/test.sh new file mode 100755 index 0000000..e26b981 --- /dev/null +++ b/test/e2e-curl/03-all-adapters-ready/test.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +# Test 03 — All required adapters True at gen=1 → both conditions flip to True + +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "$SCRIPT_DIR/../common.sh" + +# ── Purpose ─────────────────────────────────────────────────────────────────── +echo -e "\n${BOLD}Test 03: All required adapters True at gen=1 → Available=True, Ready=True${NC}" +echo "" +echo " When all required adapters report Available=True at the same generation," +echo " both synthetic conditions must flip to True and their LastTransitionTime" +echo " values must advance (False→True). Ready.last_updated_time is set to the" +echo " earliest adapter LastReportTime (not the wall clock)." +echo "" +echo -e " ${YELLOW}Starting state:${NC} Available=False@gen1, Ready=False@gen1 (initial)" +echo -e " ${YELLOW}Event:${NC} POST ${ADAPTER1}@gen1=True, POST ${ADAPTER2}@gen1=True" +echo -e " ${YELLOW}Expected:${NC} Available=True@gen1, Ready=True@gen1," +echo -e " ${YELLOW} ${NC} both LTT advance (False→True flip)" +echo "" + +# ── Setup ───────────────────────────────────────────────────────────────────── +CLUSTER_NAME="tc03-$(rand_hex4)" +log_step "Creating cluster '$CLUSTER_NAME'" +CLUSTER=$(create_cluster "$CLUSTER_NAME") +CLUSTER_ID=$(echo "$CLUSTER" | jq -r '.id') +log_received "cluster id=$CLUSTER_ID" + +log_step "GET cluster — baseline (before any adapter reports)" +BASELINE=$(get_cluster "$CLUSTER_ID") +show_state "baseline" "$BASELINE" +BASELINE_AVAIL_TRANSITION=$(condition_field "$BASELINE" Available last_transition_time) +BASELINE_READY_TRANSITION=$(condition_field "$BASELINE" Ready last_transition_time) + +# ── Events ──────────────────────────────────────────────────────────────────── +log_step "POST ${ADAPTER1}@gen1=True" +CODE=$(post_adapter_status "$CLUSTER_ID" "$ADAPTER1" 1 "True") +assert_http "${ADAPTER1} report accepted" "201" "$CODE" + +log_step "POST ${ADAPTER2}@gen1=True" +CODE=$(post_adapter_status "$CLUSTER_ID" "$ADAPTER2" 1 "True") +assert_http "${ADAPTER2} report accepted" "201" "$CODE" + +# ── Validate ────────────────────────────────────────────────────────────────── +log_step "GET cluster — post-event" +CLUSTER=$(get_cluster "$CLUSTER_ID") +show_state "after both adapters True" "$CLUSTER" + +echo "" +echo " Available condition (all required adapters True@gen1 — any-gen semantics):" +AVAIL_STATUS=$(condition_field "$CLUSTER" Available status) +AVAIL_OBSGEN=$(condition_field "$CLUSTER" Available observed_generation) +AVAIL_TRANSITION=$(condition_field "$CLUSTER" Available last_transition_time) +AVAIL_UPDATED=$(condition_field "$CLUSTER" Available last_updated_time) + +assert_eq "Available.status" "True" "$AVAIL_STATUS" +assert_eq "Available.observed_generation" "1" "$AVAIL_OBSGEN" +assert_changed "Available.last_transition_time advanced (False→True)" \ + "$BASELINE_AVAIL_TRANSITION" "$AVAIL_TRANSITION" +assert_nonempty "Available.last_updated_time (= min LRT)" "$AVAIL_UPDATED" + +echo "" +echo " Ready condition (all adapters True at current generation gen=1):" +READY_STATUS=$(condition_field "$CLUSTER" Ready status) +READY_OBSGEN=$(condition_field "$CLUSTER" Ready observed_generation) +READY_TRANSITION=$(condition_field "$CLUSTER" Ready last_transition_time) +READY_UPDATED=$(condition_field "$CLUSTER" Ready last_updated_time) + +assert_eq "Ready.status" "True" "$READY_STATUS" +assert_eq "Ready.observed_generation" "1" "$READY_OBSGEN" +assert_changed "Ready.last_transition_time advanced (False→True)" \ + "$BASELINE_READY_TRANSITION" "$READY_TRANSITION" +assert_nonempty "Ready.last_updated_time (= min LRT)" "$READY_UPDATED" + +echo "" +echo " Adapter statuses persisted in the database:" +STATUSES=$(get_statuses "$CLUSTER_ID") +assert_eq "${ADAPTER1} status stored" "1" "$(count_adapter_status "$STATUSES" "$ADAPTER1")" +assert_eq "${ADAPTER2} status stored" "1" "$(count_adapter_status "$STATUSES" "$ADAPTER2")" + +# ── Conclusion ──────────────────────────────────────────────────────────────── +echo "" +echo -e "${BOLD}Conclusion:${NC}" +echo " With all required adapters True at a consistent generation (gen=1):" +echo " • Available flips to True@gen1 — any-gen semantics: all adapters at the" +echo " same gen and all True is sufficient, regardless of the resource generation" +echo " • Ready flips to True@gen1 — current-gen semantics: all adapters at the" +echo " current resource generation (also gen=1 here)" +echo " • Both LastTransitionTime values advance — status changed False→True" +echo " • Ready.last_updated_time = min(adapter LastReportTimes), not wall clock" +echo "" +test_summary diff --git a/test/e2e-curl/04-generation-bump/test.sh b/test/e2e-curl/04-generation-bump/test.sh new file mode 100755 index 0000000..82f806a --- /dev/null +++ b/test/e2e-curl/04-generation-bump/test.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash +# Test 04 — Generation bump: spec updated, adapters still at old generation + +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "$SCRIPT_DIR/../common.sh" + +# ── Purpose ─────────────────────────────────────────────────────────────────── +echo -e "\n${BOLD}Test 04: Generation bump — adapters at gen1, cluster bumps to gen2${NC}" +echo "" +echo " When a cluster's spec is updated its generation increments. Adapters" +echo " are still reporting for the old generation. Ready must immediately drop" +echo " to False (adapters not caught up). Available must preserve its last-known-" +echo " good value — the spec says the cluster was fine before the change." +echo "" +echo -e " ${YELLOW}Starting state:${NC} Available=True@gen1, Ready=True@gen1" +echo -e " ${YELLOW}Event:${NC} PATCH cluster spec (generation bumps gen1→gen2)" +echo -e " ${YELLOW}Expected:${NC} Available=True@gen1 preserved (any-gen semantics)," +echo -e " ${YELLOW} ${NC} Ready=False@gen2, Ready.LTT advances (True→False)" +echo "" + +# ── Setup ───────────────────────────────────────────────────────────────────── +CLUSTER_NAME="tc04-$(rand_hex4)" +log_step "Creating cluster '$CLUSTER_NAME'" +CLUSTER=$(create_cluster "$CLUSTER_NAME") +CLUSTER_ID=$(echo "$CLUSTER" | jq -r '.id') +log_received "cluster id=$CLUSTER_ID" + +log_step "Setup: POST ${ADAPTER1}@gen1=True" +post_adapter_status "$CLUSTER_ID" "$ADAPTER1" 1 "True" > /dev/null + +log_step "Setup: POST ${ADAPTER2}@gen1=True" +post_adapter_status "$CLUSTER_ID" "$ADAPTER2" 1 "True" > /dev/null + +log_step "GET cluster — baseline (Ready=True@gen1)" +BASELINE=$(get_cluster "$CLUSTER_ID") +show_state "baseline" "$BASELINE" +BASELINE_AVAIL_UPDATED=$(condition_field "$BASELINE" Available last_updated_time) +BASELINE_AVAIL_TRANSITION=$(condition_field "$BASELINE" Available last_transition_time) +BASELINE_READY_TRANSITION=$(condition_field "$BASELINE" Ready last_transition_time) + +assert_eq "baseline Available.status" "True" "$(condition_field "$BASELINE" Available status)" +assert_eq "baseline Ready.status" "True" "$(condition_field "$BASELINE" Ready status)" + +sleep 2 + +# ── Event ───────────────────────────────────────────────────────────────────── +log_step "PATCH cluster spec → generation bumps to 2 (adapters still at gen1)" +PATCHED=$(patch_cluster "$CLUSTER_ID" '{"v":2,"bumped":true}') +NEW_GEN=$(echo "$PATCHED" | jq -r '.generation') +log_received "new generation=$NEW_GEN" +assert_eq "generation bumped to 2" "2" "$NEW_GEN" + +# ── Validate ────────────────────────────────────────────────────────────────── +log_step "GET cluster — post-patch" +CLUSTER=$(get_cluster "$CLUSTER_ID") +show_state "after generation bump" "$CLUSTER" + +echo "" +echo " Available condition (adapters still True@gen1 — last-known-good preserved):" +AVAIL_STATUS=$(condition_field "$CLUSTER" Available status) +AVAIL_OBSGEN=$(condition_field "$CLUSTER" Available observed_generation) +AVAIL_UPDATED=$(condition_field "$CLUSTER" Available last_updated_time) +AVAIL_TRANSITION=$(condition_field "$CLUSTER" Available last_transition_time) + +assert_eq "Available.status" "True" "$AVAIL_STATUS" +assert_eq "Available.observed_generation" "1" "$AVAIL_OBSGEN" +assert_eq "Available.last_updated_time preserved" "$BASELINE_AVAIL_UPDATED" "$AVAIL_UPDATED" +assert_eq "Available.last_transition_time preserved" "$BASELINE_AVAIL_TRANSITION" "$AVAIL_TRANSITION" + +echo "" +echo " Ready condition (adapters haven't caught up to gen2 → drops to False):" +READY_STATUS=$(condition_field "$CLUSTER" Ready status) +READY_OBSGEN=$(condition_field "$CLUSTER" Ready observed_generation) +READY_TRANSITION=$(condition_field "$CLUSTER" Ready last_transition_time) + +assert_eq "Ready.status" "False" "$READY_STATUS" +assert_eq "Ready.observed_generation" "2" "$READY_OBSGEN" +assert_changed "Ready.last_transition_time advanced (True→False)" \ + "$BASELINE_READY_TRANSITION" "$READY_TRANSITION" + +# ── Conclusion ──────────────────────────────────────────────────────────────── +echo "" +echo -e "${BOLD}Conclusion:${NC}" +echo " A spec update immediately makes the cluster 'not ready' — adapters must" +echo " reconcile the new spec before Ready can return to True." +echo " • Available=True@gen1 preserved: any-gen semantics allow the last-known-" +echo " good state to persist. The cluster was available before the change." +echo " • Ready=False@gen2: adapters are still at gen1, not caught up to gen2." +echo " LastTransitionTime advances because the status flipped True→False." +echo " This is the key safety property: spec changes put the cluster into a" +echo " 'pending reconciliation' state immediately, without waiting for adapters." +echo "" +test_summary diff --git a/test/e2e-curl/05-mixed-generations/test.sh b/test/e2e-curl/05-mixed-generations/test.sh new file mode 100755 index 0000000..d01ad43 --- /dev/null +++ b/test/e2e-curl/05-mixed-generations/test.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +# Test 05 — Mixed generations: one adapter upgrades to gen2, the other stays at gen1 + +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "$SCRIPT_DIR/../common.sh" + +# ── Purpose ─────────────────────────────────────────────────────────────────── +echo -e "\n${BOLD}Test 05: Mixed generations — one adapter at gen2, one at gen1, both True${NC}" +echo "" +echo " When adapters are at different observed generations the snapshot is" +echo " inconsistent. Available must stay unchanged (guard fires). Ready stays" +echo " False because not all adapters are at the current resource generation." +echo " This is the normal in-flight state during a rolling adapter upgrade." +echo "" +echo -e " ${YELLOW}Starting state:${NC} Available=True@gen1, Ready=False@gen2" +echo -e " ${YELLOW} ${NC} (${ADAPTER1}@gen1=True, ${ADAPTER2}@gen1=True, cluster at gen2)" +echo -e " ${YELLOW}Event:${NC} POST ${ADAPTER2}@gen2=True (${ADAPTER1} still at gen1)" +echo -e " ${YELLOW}Expected:${NC} Available=True@gen1 unchanged (inconsistent snapshot)," +echo -e " ${YELLOW} ${NC} Ready=False@gen2 (${ADAPTER1} not at gen2), no timestamp changes" +echo "" + +# ── Setup ───────────────────────────────────────────────────────────────────── +CLUSTER_NAME="tc05-$(rand_hex4)" +log_step "Creating cluster '$CLUSTER_NAME'" +CLUSTER=$(create_cluster "$CLUSTER_NAME") +CLUSTER_ID=$(echo "$CLUSTER" | jq -r '.id') +log_received "cluster id=$CLUSTER_ID" + +log_step "Setup: POST ${ADAPTER1}@gen1=True" +post_adapter_status "$CLUSTER_ID" "$ADAPTER1" 1 "True" > /dev/null +log_step "Setup: POST ${ADAPTER2}@gen1=True" +post_adapter_status "$CLUSTER_ID" "$ADAPTER2" 1 "True" > /dev/null +log_step "Setup: PATCH cluster → gen2" +PATCHED=$(patch_cluster "$CLUSTER_ID" '{"v":2}') +assert_eq "generation=2" "2" "$(echo "$PATCHED" | jq -r '.generation')" + +log_step "GET cluster — pre-event baseline (Available=True@gen1, Ready=False@gen2)" +AFTER_PATCH=$(get_cluster "$CLUSTER_ID") +show_state "pre-event" "$AFTER_PATCH" +AVAIL_UPDATED_AFTER_PATCH=$(condition_field "$AFTER_PATCH" Available last_updated_time) +AVAIL_TRANSITION_AFTER_PATCH=$(condition_field "$AFTER_PATCH" Available last_transition_time) +READY_TRANSITION_AFTER_PATCH=$(condition_field "$AFTER_PATCH" Ready last_transition_time) + +assert_eq "pre-event Available.status" "True" "$(condition_field "$AFTER_PATCH" Available status)" +assert_eq "pre-event Available.obsgen" "1" "$(condition_field "$AFTER_PATCH" Available observed_generation)" +assert_eq "pre-event Ready.status" "False" "$(condition_field "$AFTER_PATCH" Ready status)" + +sleep 1 + +# ── Event ───────────────────────────────────────────────────────────────────── +log_step "POST ${ADAPTER2}@gen2=True (${ADAPTER1} still at gen1 — snapshot inconsistent)" +CODE=$(post_adapter_status "$CLUSTER_ID" "$ADAPTER2" 2 "True") +assert_http "${ADAPTER2}@gen2 accepted" "201" "$CODE" + +# ── Validate ────────────────────────────────────────────────────────────────── +log_step "GET cluster — post-event" +CLUSTER=$(get_cluster "$CLUSTER_ID") +show_state "after event" "$CLUSTER" + +echo "" +echo " Available condition (snapshot inconsistent — ${ADAPTER1}@gen1, ${ADAPTER2}@gen2 → guard fires):" +AVAIL_STATUS=$(condition_field "$CLUSTER" Available status) +AVAIL_OBSGEN=$(condition_field "$CLUSTER" Available observed_generation) +AVAIL_UPDATED=$(condition_field "$CLUSTER" Available last_updated_time) +AVAIL_TRANSITION=$(condition_field "$CLUSTER" Available last_transition_time) + +assert_eq "Available.status" "True" "$AVAIL_STATUS" +assert_eq "Available.observed_generation (min=gen1)" "1" "$AVAIL_OBSGEN" +assert_eq "Available.last_updated_time preserved" "$AVAIL_UPDATED_AFTER_PATCH" "$AVAIL_UPDATED" +assert_eq "Available.last_transition_time preserved" "$AVAIL_TRANSITION_AFTER_PATCH" "$AVAIL_TRANSITION" + +echo "" +echo " Ready condition (${ADAPTER1} at gen1 ≠ resource gen2 → stays False, no flip):" +READY_STATUS=$(condition_field "$CLUSTER" Ready status) +READY_OBSGEN=$(condition_field "$CLUSTER" Ready observed_generation) +READY_TRANSITION=$(condition_field "$CLUSTER" Ready last_transition_time) + +assert_eq "Ready.status" "False" "$READY_STATUS" +assert_eq "Ready.observed_generation" "2" "$READY_OBSGEN" +assert_eq "Ready.last_transition_time preserved (no flip)" \ + "$READY_TRANSITION_AFTER_PATCH" "$READY_TRANSITION" + +# ── Conclusion ──────────────────────────────────────────────────────────────── +echo "" +echo -e "${BOLD}Conclusion:${NC}" +echo " With adapters at different generations the snapshot is inconsistent:" +echo " • Available guard fires → all Available timestamps are frozen. The" +echo " last-known-good True@gen1 is preserved until adapters converge." +echo " • Ready stays False because ${ADAPTER1} is still at gen1, not the current gen2." +echo " No status flip → LastTransitionTime is also preserved." +echo " This is the expected in-flight state while a rolling reconciliation is" +echo " in progress — adapters update one at a time." +echo "" +test_summary diff --git a/test/e2e-curl/06-stale-report/test.sh b/test/e2e-curl/06-stale-report/test.sh new file mode 100755 index 0000000..1da0626 --- /dev/null +++ b/test/e2e-curl/06-stale-report/test.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# Test 06 — Stale report: ObsGen(new) < ObsGen(existing) → silently discarded + +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "$SCRIPT_DIR/../common.sh" + +# ── Purpose ─────────────────────────────────────────────────────────────────── +echo -e "\n${BOLD}Test 06: Stale report — ObsGen(new) < ObsGen(existing) → DISCARD (204)${NC}" +echo "" +echo " If an adapter sends a report for an observed generation that is older" +echo " than what is already stored, the server must silently discard it. The" +echo " report carries no new information and must not overwrite newer state." +echo " All cluster conditions must remain completely unchanged." +echo "" +echo -e " ${YELLOW}Starting state:${NC} ${ADAPTER2} stored at gen2; cluster at gen2" +echo -e " ${YELLOW}Event:${NC} POST ${ADAPTER2}@gen1=False (stale — gen1 < stored gen2)" +echo -e " ${YELLOW}Expected:${NC} HTTP 204, ALL condition fields byte-identical to pre-event" +echo "" + +# ── Setup ───────────────────────────────────────────────────────────────────── +CLUSTER_NAME="tc06-$(rand_hex4)" +log_step "Creating cluster '$CLUSTER_NAME'" +CLUSTER=$(create_cluster "$CLUSTER_NAME") +CLUSTER_ID=$(echo "$CLUSTER" | jq -r '.id') +log_received "cluster id=$CLUSTER_ID" + +log_step "Setup: POST ${ADAPTER1}@gen1=True" +post_adapter_status "$CLUSTER_ID" "$ADAPTER1" 1 "True" > /dev/null +log_step "Setup: POST ${ADAPTER2}@gen1=True" +post_adapter_status "$CLUSTER_ID" "$ADAPTER2" 1 "True" > /dev/null +log_step "Setup: PATCH cluster → gen2" +patch_cluster "$CLUSTER_ID" '{"v":2}' > /dev/null +log_step "Setup: POST ${ADAPTER2}@gen2=True (${ADAPTER2} now stored at gen2)" +post_adapter_status "$CLUSTER_ID" "$ADAPTER2" 2 "True" > /dev/null + +log_step "GET cluster — pre-stale baseline" +PRE=$(get_cluster "$CLUSTER_ID") +show_state "pre-stale" "$PRE" +PRE_AVAIL_STATUS=$(condition_field "$PRE" Available status) +PRE_AVAIL_OBSGEN=$(condition_field "$PRE" Available observed_generation) +PRE_AVAIL_UPDATED=$(condition_field "$PRE" Available last_updated_time) +PRE_AVAIL_TRANSITION=$(condition_field "$PRE" Available last_transition_time) +PRE_READY_STATUS=$(condition_field "$PRE" Ready status) +PRE_READY_OBSGEN=$(condition_field "$PRE" Ready observed_generation) +PRE_READY_UPDATED=$(condition_field "$PRE" Ready last_updated_time) +PRE_READY_TRANSITION=$(condition_field "$PRE" Ready last_transition_time) +log_received "Available=$PRE_AVAIL_STATUS@obsgen$PRE_AVAIL_OBSGEN Ready=$PRE_READY_STATUS@obsgen$PRE_READY_OBSGEN" + +sleep 1 + +# ── Event ───────────────────────────────────────────────────────────────────── +log_step "POST ${ADAPTER2}@gen1=False (STALE — gen1 < stored gen2)" +CODE=$(post_adapter_status "$CLUSTER_ID" "$ADAPTER2" 1 "False") + +echo "" +echo " API response (stale detection must return 204 without touching state):" +assert_http "stale report discarded" "204" "$CODE" + +# ── Validate ────────────────────────────────────────────────────────────────── +log_step "GET cluster — all fields must be byte-identical to pre-event snapshot" +POST=$(get_cluster "$CLUSTER_ID") +show_state "after stale" "$POST" + +echo "" +echo " Available condition (must be identical to pre-stale snapshot):" +assert_eq "Available.status" "$PRE_AVAIL_STATUS" "$(condition_field "$POST" Available status)" +assert_eq "Available.observed_generation" "$PRE_AVAIL_OBSGEN" "$(condition_field "$POST" Available observed_generation)" +assert_eq "Available.last_updated_time" "$PRE_AVAIL_UPDATED" "$(condition_field "$POST" Available last_updated_time)" +assert_eq "Available.last_transition_time" "$PRE_AVAIL_TRANSITION" "$(condition_field "$POST" Available last_transition_time)" + +echo "" +echo " Ready condition (must be identical to pre-stale snapshot):" +assert_eq "Ready.status" "$PRE_READY_STATUS" "$(condition_field "$POST" Ready status)" +assert_eq "Ready.observed_generation" "$PRE_READY_OBSGEN" "$(condition_field "$POST" Ready observed_generation)" +assert_eq "Ready.last_updated_time" "$PRE_READY_UPDATED" "$(condition_field "$POST" Ready last_updated_time)" +assert_eq "Ready.last_transition_time" "$PRE_READY_TRANSITION" "$(condition_field "$POST" Ready last_transition_time)" + +# ── Conclusion ──────────────────────────────────────────────────────────────── +echo "" +echo -e "${BOLD}Conclusion:${NC}" +echo " The stale-detection rule (observed_generation < stored_generation) fired:" +echo " • The server returned 204 without persisting the report or running aggregation" +echo " • Every Available and Ready field is byte-identical to the pre-event snapshot" +echo " This prevents adapters from accidentally 'going back in time' — a late-" +echo " arriving network packet from a previous reconciliation cycle cannot corrupt" +echo " the current state." +echo "" +test_summary diff --git a/test/e2e-curl/07-all-adapters-new-gen/test.sh b/test/e2e-curl/07-all-adapters-new-gen/test.sh new file mode 100755 index 0000000..70fb7b3 --- /dev/null +++ b/test/e2e-curl/07-all-adapters-new-gen/test.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +# Test 07 — All adapters converge at gen2: Available ObsGen advances, Ready=True + +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "$SCRIPT_DIR/../common.sh" + +# ── Purpose ─────────────────────────────────────────────────────────────────── +echo -e "\n${BOLD}Test 07: All adapters converge at gen2 → Available@gen2, Ready=True${NC}" +echo "" +echo " When all required adapters finally report True at the current generation" +echo " the cluster is fully reconciled. Available's observed_generation must" +echo " advance from gen1 to gen2 (status stays True, LTT is preserved because" +echo " there is no True↔False flip). Ready flips from False to True." +echo "" +echo -e " ${YELLOW}Starting state:${NC} Available=True@gen1, Ready=False@gen2" +echo -e " ${YELLOW} ${NC} (${ADAPTER1}@gen1=True, ${ADAPTER2}@gen2=True, cluster at gen2)" +echo -e " ${YELLOW}Event:${NC} POST ${ADAPTER1}@gen2=True (now both adapters at gen2)" +echo -e " ${YELLOW}Expected:${NC} Available=True@gen2 (ObsGen advances, LTT preserved)," +echo -e " ${YELLOW} ${NC} Ready=True@gen2, Ready.LTT advances (False→True)" +echo "" + +# ── Setup ───────────────────────────────────────────────────────────────────── +CLUSTER_NAME="tc07-$(rand_hex4)" +log_step "Creating cluster '$CLUSTER_NAME'" +CLUSTER=$(create_cluster "$CLUSTER_NAME") +CLUSTER_ID=$(echo "$CLUSTER" | jq -r '.id') +log_received "cluster id=$CLUSTER_ID" + +log_step "Setup: POST ${ADAPTER1}@gen1=True" +post_adapter_status "$CLUSTER_ID" "$ADAPTER1" 1 "True" > /dev/null +log_step "Setup: POST ${ADAPTER2}@gen1=True" +post_adapter_status "$CLUSTER_ID" "$ADAPTER2" 1 "True" > /dev/null +log_step "Setup: PATCH cluster → gen2" +patch_cluster "$CLUSTER_ID" '{"v":2}' > /dev/null +log_step "Setup: POST ${ADAPTER2}@gen2=True (${ADAPTER1} still at gen1)" +post_adapter_status "$CLUSTER_ID" "$ADAPTER2" 2 "True" > /dev/null + +log_step "GET cluster — pre-event baseline (Available=True@gen1, Ready=False@gen2)" +BASELINE=$(get_cluster "$CLUSTER_ID") +show_state "pre-event" "$BASELINE" +BASELINE_AVAIL_UPDATED=$(condition_field "$BASELINE" Available last_updated_time) +BASELINE_AVAIL_TRANSITION=$(condition_field "$BASELINE" Available last_transition_time) +BASELINE_READY_TRANSITION=$(condition_field "$BASELINE" Ready last_transition_time) + +assert_eq "baseline Available.status" "True" "$(condition_field "$BASELINE" Available status)" +assert_eq "baseline Available.obsgen" "1" "$(condition_field "$BASELINE" Available observed_generation)" +assert_eq "baseline Ready.status" "False" "$(condition_field "$BASELINE" Ready status)" + +sleep 1 + +# ── Event ───────────────────────────────────────────────────────────────────── +log_step "POST ${ADAPTER1}@gen2=True (all adapters now at gen2 — snapshot consistent)" +CODE=$(post_adapter_status "$CLUSTER_ID" "$ADAPTER1" 2 "True") +assert_http "${ADAPTER1}@gen2 accepted" "201" "$CODE" + +# ── Validate ────────────────────────────────────────────────────────────────── +log_step "GET cluster — post-event" +CLUSTER=$(get_cluster "$CLUSTER_ID") +show_state "after convergence" "$CLUSTER" + +echo "" +echo " Available condition (ObsGen advances gen1→gen2, status stays True):" +AVAIL_STATUS=$(condition_field "$CLUSTER" Available status) +AVAIL_OBSGEN=$(condition_field "$CLUSTER" Available observed_generation) +AVAIL_UPDATED=$(condition_field "$CLUSTER" Available last_updated_time) +AVAIL_TRANSITION=$(condition_field "$CLUSTER" Available last_transition_time) + +assert_eq "Available.status" "True" "$AVAIL_STATUS" +assert_eq "Available.observed_generation" "2" "$AVAIL_OBSGEN" +assert_changed "Available.last_updated_time refreshed (ObsGen changed gen1→gen2)" \ + "$BASELINE_AVAIL_UPDATED" "$AVAIL_UPDATED" +assert_eq "Available.last_transition_time preserved (status stayed True, no flip)" \ + "$BASELINE_AVAIL_TRANSITION" "$AVAIL_TRANSITION" + +echo "" +echo " Ready condition (all adapters True@gen2 = current generation → flips to True):" +READY_STATUS=$(condition_field "$CLUSTER" Ready status) +READY_OBSGEN=$(condition_field "$CLUSTER" Ready observed_generation) +READY_TRANSITION=$(condition_field "$CLUSTER" Ready last_transition_time) +READY_UPDATED=$(condition_field "$CLUSTER" Ready last_updated_time) + +assert_eq "Ready.status" "True" "$READY_STATUS" +assert_eq "Ready.observed_generation" "2" "$READY_OBSGEN" +assert_changed "Ready.last_transition_time advanced (False→True)" \ + "$BASELINE_READY_TRANSITION" "$READY_TRANSITION" +assert_nonempty "Ready.last_updated_time (= min adapter LRT)" "$READY_UPDATED" + +# ── Conclusion ──────────────────────────────────────────────────────────────── +echo "" +echo -e "${BOLD}Conclusion:${NC}" +echo " When all adapters converge at the current generation:" +echo " • Available's observed_generation advances from gen1 to gen2. The status" +echo " stays True (no True↔False flip) so LastTransitionTime is preserved." +echo " LastUpdatedTime refreshes because the observed_generation changed." +echo " • Ready flips to True (False→True) because all adapters are now at gen2." +echo " LastTransitionTime advances. LastUpdatedTime = min(adapter LRTs) — the" +echo " earliest of the two adapter report times, not the wall clock." +echo " The cluster is now fully reconciled to gen2." +echo "" +test_summary diff --git a/test/e2e-curl/08-adapter-goes-false/test.sh b/test/e2e-curl/08-adapter-goes-false/test.sh new file mode 100755 index 0000000..3fec76a --- /dev/null +++ b/test/e2e-curl/08-adapter-goes-false/test.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env bash +# Test 08 — Adapter goes False: one required adapter reports False → both conditions drop + +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "$SCRIPT_DIR/../common.sh" + +# ── Purpose ─────────────────────────────────────────────────────────────────── +echo -e "\n${BOLD}Test 08: One required adapter goes False → Available=False, Ready=False${NC}" +echo "" +echo " When a required adapter reports Available=False the cluster can no longer" +echo " be considered available or ready. Both conditions must flip to False and" +echo " their LastTransitionTime values must advance (True→False). The timestamps" +echo " reflect the adapter's observed_time, not the server's wall clock." +echo " Available.observed_generation must be the resource generation, NOT zero." +echo "" +echo -e " ${YELLOW}Starting state:${NC} Available=True@gen1, Ready=True@gen1 (both adapters True)" +echo -e " ${YELLOW}Event:${NC} POST ${ADAPTER1}@gen1=False" +echo -e " ${YELLOW}Expected:${NC} Available=False@rG1 (rG not zero), Ready=False@rG1," +echo -e " ${YELLOW} ${NC} both LTT advance (True→False), both LUT refreshed" +echo "" + +# ── Setup ───────────────────────────────────────────────────────────────────── +CLUSTER_NAME="tc08-$(rand_hex4)" +log_step "Creating cluster '$CLUSTER_NAME'" +CLUSTER=$(create_cluster "$CLUSTER_NAME") +CLUSTER_ID=$(echo "$CLUSTER" | jq -r '.id') +RESOURCE_GEN=$(echo "$CLUSTER" | jq -r '.generation') +log_received "cluster id=$CLUSTER_ID generation=$RESOURCE_GEN" + +log_step "Setup: POST ${ADAPTER1}@gen1=True" +post_adapter_status "$CLUSTER_ID" "$ADAPTER1" 1 "True" > /dev/null +log_step "Setup: POST ${ADAPTER2}@gen1=True" +post_adapter_status "$CLUSTER_ID" "$ADAPTER2" 1 "True" > /dev/null + +log_step "GET cluster — baseline (both adapters True → cluster Ready)" +BASELINE=$(get_cluster "$CLUSTER_ID") +show_state "baseline" "$BASELINE" +BASELINE_AVAIL_TRANSITION=$(condition_field "$BASELINE" Available last_transition_time) +BASELINE_AVAIL_UPDATED=$(condition_field "$BASELINE" Available last_updated_time) +BASELINE_READY_TRANSITION=$(condition_field "$BASELINE" Ready last_transition_time) +BASELINE_READY_UPDATED=$(condition_field "$BASELINE" Ready last_updated_time) + +assert_eq "baseline Available.status" "True" "$(condition_field "$BASELINE" Available status)" +assert_eq "baseline Ready.status" "True" "$(condition_field "$BASELINE" Ready status)" + +sleep 2 + +# ── Event ───────────────────────────────────────────────────────────────────── +log_step "POST ${ADAPTER1}@gen1=False (one required adapter reports a problem)" +CODE=$(post_adapter_status "$CLUSTER_ID" "$ADAPTER1" 1 "False") +assert_http "${ADAPTER1} False report accepted" "201" "$CODE" + +# ── Validate ────────────────────────────────────────────────────────────────── +log_step "GET cluster — post-event" +CLUSTER=$(get_cluster "$CLUSTER_ID") +show_state "after ${ADAPTER1}=False" "$CLUSTER" + +echo "" +echo " Available condition (False; observed_generation=rG=$RESOURCE_GEN, NOT zero):" +AVAIL_STATUS=$(condition_field "$CLUSTER" Available status) +AVAIL_OBSGEN=$(condition_field "$CLUSTER" Available observed_generation) +AVAIL_TRANSITION=$(condition_field "$CLUSTER" Available last_transition_time) +AVAIL_UPDATED=$(condition_field "$CLUSTER" Available last_updated_time) + +assert_eq "Available.status" "False" "$AVAIL_STATUS" +assert_eq "Available.observed_generation (=rG not 0)" "$RESOURCE_GEN" "$AVAIL_OBSGEN" +assert_changed "Available.last_transition_time advanced (True→False)" \ + "$BASELINE_AVAIL_TRANSITION" "$AVAIL_TRANSITION" +assert_changed "Available.last_updated_time refreshed (= observed_time)" \ + "$BASELINE_AVAIL_UPDATED" "$AVAIL_UPDATED" + +echo "" +echo " Ready condition (one adapter False → also drops to False):" +READY_STATUS=$(condition_field "$CLUSTER" Ready status) +READY_OBSGEN=$(condition_field "$CLUSTER" Ready observed_generation) +READY_TRANSITION=$(condition_field "$CLUSTER" Ready last_transition_time) +READY_UPDATED=$(condition_field "$CLUSTER" Ready last_updated_time) + +assert_eq "Ready.status" "False" "$READY_STATUS" +assert_eq "Ready.observed_generation" "$RESOURCE_GEN" "$READY_OBSGEN" +assert_changed "Ready.last_transition_time advanced (True→False)" \ + "$BASELINE_READY_TRANSITION" "$READY_TRANSITION" +assert_changed "Ready.last_updated_time refreshed" \ + "$BASELINE_READY_UPDATED" "$READY_UPDATED" + +# ── Conclusion ──────────────────────────────────────────────────────────────── +echo "" +echo -e "${BOLD}Conclusion:${NC}" +echo " A single False report from any required adapter immediately drops both" +echo " Available and Ready to False:" +echo " • Available=False@rG1: the observed_generation is set to the resource" +echo " generation (gen=1), NOT zero. This is an explicit spec requirement." +echo " • Both LastTransitionTime values advance — the status flipped True→False." +echo " • Both LastUpdatedTime values are refreshed to the adapter's observed_time." +echo " This is the timestamp when the problem was first detected, not when the" +echo " server processed the report." +echo "" +test_summary diff --git a/test/e2e-curl/09-stable-true/test.sh b/test/e2e-curl/09-stable-true/test.sh new file mode 100755 index 0000000..6a8436a --- /dev/null +++ b/test/e2e-curl/09-stable-true/test.sh @@ -0,0 +1,106 @@ +#!/usr/bin/env bash +# Test 09 — Stable True: keep-alive re-reports when cluster is already Ready + +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "$SCRIPT_DIR/../common.sh" + +# ── Purpose ─────────────────────────────────────────────────────────────────── +echo -e "\n${BOLD}Test 09: Stable True re-evaluation — both Available and Ready.LUT refreshed${NC}" +echo "" +echo " Adapters periodically re-report True to keep the cluster 'alive' in" +echo " Sentinel's view. Per spec §5 (Available=True, Available=True row):" +echo " lut = min(statuses[].lut) — both Available and Ready refresh their" +echo " last_updated_time to the new min adapter LRT on every heartbeat." +echo " last_transition_time is preserved for both (no status flip)." +echo "" +echo -e " ${YELLOW}Starting state:${NC} Available=True@gen1, Ready=True@gen1" +echo -e " ${YELLOW}Event:${NC} POST ${ADAPTER1}@gen1=True, POST ${ADAPTER2}@gen1=True (re-reports)" +echo -e " ${YELLOW}Expected:${NC} Available=True@gen1, Available.LUT refreshed (= new min LRT)," +echo -e " ${YELLOW} ${NC} Ready=True@gen1, Ready.LUT refreshed (= new min LRT)," +echo -e " ${YELLOW} ${NC} both LTTs preserved (no True↔False flip)" +echo "" + +# ── Setup ───────────────────────────────────────────────────────────────────── +CLUSTER_NAME="tc09-$(rand_hex4)" +log_step "Creating cluster '$CLUSTER_NAME'" +CLUSTER=$(create_cluster "$CLUSTER_NAME") +CLUSTER_ID=$(echo "$CLUSTER" | jq -r '.id') +log_received "cluster id=$CLUSTER_ID" + +log_step "Setup: POST ${ADAPTER1}@gen1=True (initial report)" +post_adapter_status "$CLUSTER_ID" "$ADAPTER1" 1 "True" > /dev/null +log_step "Setup: POST ${ADAPTER2}@gen1=True (initial report)" +post_adapter_status "$CLUSTER_ID" "$ADAPTER2" 1 "True" > /dev/null + +log_step "GET cluster — baseline (Ready=True@gen1)" +BASELINE=$(get_cluster "$CLUSTER_ID") +show_state "baseline" "$BASELINE" +BASELINE_AVAIL_UPDATED=$(condition_field "$BASELINE" Available last_updated_time) +BASELINE_AVAIL_TRANSITION=$(condition_field "$BASELINE" Available last_transition_time) +BASELINE_READY_UPDATED=$(condition_field "$BASELINE" Ready last_updated_time) +BASELINE_READY_TRANSITION=$(condition_field "$BASELINE" Ready last_transition_time) +log_received "Available.LUT=$BASELINE_AVAIL_UPDATED" +log_received "Ready.LUT=$BASELINE_READY_UPDATED" + +assert_eq "baseline Available.status" "True" "$(condition_field "$BASELINE" Available status)" +assert_eq "baseline Ready.status" "True" "$(condition_field "$BASELINE" Ready status)" + +# Ensure new reports will have newer timestamps (2s for second-precision timestamps) +sleep 2 + +# ── Events ──────────────────────────────────────────────────────────────────── +log_step "POST ${ADAPTER1}@gen1=True (heartbeat re-report)" +CODE=$(post_adapter_status "$CLUSTER_ID" "$ADAPTER1" 1 "True") +assert_http "${ADAPTER1} re-report accepted" "201" "$CODE" + +log_step "POST ${ADAPTER2}@gen1=True (heartbeat re-report)" +CODE=$(post_adapter_status "$CLUSTER_ID" "$ADAPTER2" 1 "True") +assert_http "${ADAPTER2} re-report accepted" "201" "$CODE" + +# ── Validate ────────────────────────────────────────────────────────────────── +log_step "GET cluster — post re-reports" +CLUSTER=$(get_cluster "$CLUSTER_ID") +show_state "after re-reports" "$CLUSTER" + +echo "" +echo " Available condition (status and ObsGen unchanged, LUT advances to new min LRT):" +AVAIL_STATUS=$(condition_field "$CLUSTER" Available status) +AVAIL_OBSGEN=$(condition_field "$CLUSTER" Available observed_generation) +AVAIL_UPDATED=$(condition_field "$CLUSTER" Available last_updated_time) +AVAIL_TRANSITION=$(condition_field "$CLUSTER" Available last_transition_time) + +assert_eq "Available.status" "True" "$AVAIL_STATUS" +assert_eq "Available.observed_generation" "1" "$AVAIL_OBSGEN" +assert_changed "Available.last_updated_time refreshed (= new min adapter LRT)" \ + "$BASELINE_AVAIL_UPDATED" "$AVAIL_UPDATED" +assert_eq "Available.last_transition_time preserved" "$BASELINE_AVAIL_TRANSITION" "$AVAIL_TRANSITION" + +echo "" +echo " Ready condition (last_updated_time = min(LRT) — freshened by new reports):" +READY_STATUS=$(condition_field "$CLUSTER" Ready status) +READY_OBSGEN=$(condition_field "$CLUSTER" Ready observed_generation) +READY_UPDATED=$(condition_field "$CLUSTER" Ready last_updated_time) +READY_TRANSITION=$(condition_field "$CLUSTER" Ready last_transition_time) + +assert_eq "Ready.status" "True" "$READY_STATUS" +assert_eq "Ready.observed_generation" "1" "$READY_OBSGEN" +assert_changed "Ready.last_updated_time refreshed (= new min adapter LRT)" \ + "$BASELINE_READY_UPDATED" "$READY_UPDATED" +assert_eq "Ready.last_transition_time preserved (status stayed True, no flip)" \ + "$BASELINE_READY_TRANSITION" "$READY_TRANSITION" + +# ── Conclusion ──────────────────────────────────────────────────────────────── +echo "" +echo -e "${BOLD}Conclusion:${NC}" +echo " Heartbeat re-reports (same status, same generation) are handled correctly:" +echo " • Available: per spec lut=min(statuses[].lut), so last_updated_time advances" +echo " to the new min(adapter LRTs) on every heartbeat. last_transition_time is" +echo " preserved because the status stayed True (no True↔False flip)." +echo " • Ready: same rule — last_updated_time advances to the new min(adapter LRTs)." +echo " This freshens Sentinel's staleness clock — if adapters stop reporting," +echo " Ready.LUT will eventually age past the 30-minute threshold and Sentinel" +echo " will trigger a re-reconciliation." +echo " • LastTransitionTime for both stays frozen — no status flip occurred." +echo "" +test_summary diff --git a/test/e2e-curl/10-stable-false/test.sh b/test/e2e-curl/10-stable-false/test.sh new file mode 100755 index 0000000..9e97730 --- /dev/null +++ b/test/e2e-curl/10-stable-false/test.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +# Test 10 — Stable False: re-report when both conditions are already False + +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "$SCRIPT_DIR/../common.sh" + +# ── Purpose ─────────────────────────────────────────────────────────────────── +echo -e "\n${BOLD}Test 10: Stable False re-evaluation — Available.LUT preserved, Ready.LUT refreshed${NC}" +echo "" +echo " When a required adapter is missing the cluster stays Available=False and" +echo " Ready=False. Re-reports from the present adapter must not churn Available" +echo " timestamps (nothing changed). Ready.last_updated_time uses min(adapter LRTs);" +echo " because ${ADAPTER1} has never reported, the fallback is 'now', so it refreshes." +echo "" +echo -e " ${YELLOW}Starting state:${NC} Available=False@gen2, Ready=False@gen2" +echo -e " ${YELLOW} ${NC} (${ADAPTER1} missing, ${ADAPTER2}@gen2=True, cluster at gen2)" +echo -e " ${YELLOW}Event:${NC} POST ${ADAPTER2}@gen2=True (${ADAPTER1} still missing)" +echo -e " ${YELLOW}Expected:${NC} Available=False unchanged (guard fires), Ready=False," +echo -e " ${YELLOW} ${NC} Available.LUT preserved, Ready.LUT refreshed (now, ${ADAPTER1} absent)" +echo "" + +# ── Setup ───────────────────────────────────────────────────────────────────── +CLUSTER_NAME="tc10-$(rand_hex4)" +log_step "Creating cluster '$CLUSTER_NAME'" +CLUSTER=$(create_cluster "$CLUSTER_NAME") +CLUSTER_ID=$(echo "$CLUSTER" | jq -r '.id') +log_received "cluster id=$CLUSTER_ID" + +log_step "Setup: POST ${ADAPTER2}@gen1=True (${ADAPTER1} missing → False@rG1)" +post_adapter_status "$CLUSTER_ID" "$ADAPTER2" 1 "True" > /dev/null +log_step "Setup: PATCH cluster → gen2" +patch_cluster "$CLUSTER_ID" '{"v":2}' > /dev/null +log_step "Setup: POST ${ADAPTER2}@gen2=True (${ADAPTER1} still missing → False@rG2)" +post_adapter_status "$CLUSTER_ID" "$ADAPTER2" 2 "True" > /dev/null + +log_step "GET cluster — pre-event baseline (Available=False@gen2, Ready=False@gen2)" +BASELINE=$(get_cluster "$CLUSTER_ID") +show_state "baseline" "$BASELINE" +BASELINE_AVAIL_UPDATED=$(condition_field "$BASELINE" Available last_updated_time) +BASELINE_AVAIL_TRANSITION=$(condition_field "$BASELINE" Available last_transition_time) +BASELINE_READY_UPDATED=$(condition_field "$BASELINE" Ready last_updated_time) +BASELINE_READY_TRANSITION=$(condition_field "$BASELINE" Ready last_transition_time) + +assert_eq "baseline Available.status" "False" "$(condition_field "$BASELINE" Available status)" +assert_eq "baseline Available.obsgen" "2" "$(condition_field "$BASELINE" Available observed_generation)" +assert_eq "baseline Ready.status" "False" "$(condition_field "$BASELINE" Ready status)" +log_received "Available.LUT=$BASELINE_AVAIL_UPDATED" +log_received "Ready.LUT=$BASELINE_READY_UPDATED" + +sleep 2 + +# ── Event ───────────────────────────────────────────────────────────────────── +log_step "POST ${ADAPTER2}@gen2=True (${ADAPTER1} still missing — re-evaluation)" +CODE=$(post_adapter_status "$CLUSTER_ID" "$ADAPTER2" 2 "True") +assert_http "${ADAPTER2} re-report accepted" "201" "$CODE" + +# ── Validate ────────────────────────────────────────────────────────────────── +log_step "GET cluster — post re-evaluation" +CLUSTER=$(get_cluster "$CLUSTER_ID") +show_state "after re-evaluation" "$CLUSTER" + +echo "" +echo " Available condition (guard matches False@gen2→False@gen2 — timestamps frozen):" +assert_eq "Available.status" "False" "$(condition_field "$CLUSTER" Available status)" +assert_eq "Available.observed_generation" "2" "$(condition_field "$CLUSTER" Available observed_generation)" +assert_eq "Available.last_updated_time preserved" "$BASELINE_AVAIL_UPDATED" "$(condition_field "$CLUSTER" Available last_updated_time)" +assert_eq "Available.last_transition_time preserved" "$BASELINE_AVAIL_TRANSITION" "$(condition_field "$CLUSTER" Available last_transition_time)" + +echo "" +echo " Ready condition (last_updated_time = now when False — Sentinel 10s clock):" +assert_eq "Ready.status" "False" "$(condition_field "$CLUSTER" Ready status)" +assert_eq "Ready.observed_generation" "2" "$(condition_field "$CLUSTER" Ready observed_generation)" +assert_changed "Ready.last_updated_time refreshed (= min(LRTs); fallback=now since ${ADAPTER1} absent)" \ + "$BASELINE_READY_UPDATED" "$(condition_field "$CLUSTER" Ready last_updated_time)" +assert_eq "Ready.last_transition_time preserved (no flip)" \ + "$BASELINE_READY_TRANSITION" "$(condition_field "$CLUSTER" Ready last_transition_time)" + +# ── Conclusion ──────────────────────────────────────────────────────────────── +echo "" +echo -e "${BOLD}Conclusion:${NC}" +echo " Stable False re-evaluation is handled correctly:" +echo " • Available: guard fires (same status False AND same ObsGen=2) — all" +echo " Available timestamps are frozen. No spurious churn while waiting for" +echo " ${ADAPTER1} to appear." +echo " • Ready: last_updated_time = min(adapter LRTs). Because ${ADAPTER1} has" +echo " never reported, the fallback applies and LUT is set to 'now'. When all" +echo " required adapters have reported, LUT will reflect their earliest LRT." +echo " • Neither LastTransitionTime changes — the status did not flip." +echo "" +test_summary diff --git a/test/e2e-curl/11-unknown-subsequent/test.sh b/test/e2e-curl/11-unknown-subsequent/test.sh new file mode 100755 index 0000000..2ec7562 --- /dev/null +++ b/test/e2e-curl/11-unknown-subsequent/test.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +# Test 11 — Unknown on subsequent report: always discarded (P3 rule) + +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "$SCRIPT_DIR/../common.sh" + +# ── Purpose ─────────────────────────────────────────────────────────────────── +echo -e "\n${BOLD}Test 11: Unknown on subsequent report → discarded (204), state unchanged${NC}" +echo "" +echo " The P3 rule discards any adapter report where Available=Unknown. This" +echo " applies to ALL reports — first, subsequent, at any generation. When a" +echo " prior adapter status already exists, the Unknown report must not overwrite" +echo " it and must not trigger aggregation. The stored True value is preserved." +echo "" +echo -e " ${YELLOW}Starting state:${NC} ${ADAPTER2} stored with Available=True; cluster Ready=True" +echo -e " ${YELLOW}Event:${NC} POST ${ADAPTER2}@gen1=Unknown (prior True status exists)" +echo -e " ${YELLOW}Expected:${NC} HTTP 204, all cluster conditions byte-identical to baseline," +echo -e " ${YELLOW} ${NC} stored adapter status retains its prior True value" +echo "" + +# ── Setup ───────────────────────────────────────────────────────────────────── +CLUSTER_NAME="tc11-$(rand_hex4)" +log_step "Creating cluster '$CLUSTER_NAME'" +CLUSTER=$(create_cluster "$CLUSTER_NAME") +CLUSTER_ID=$(echo "$CLUSTER" | jq -r '.id') +log_received "cluster id=$CLUSTER_ID" + +log_step "Setup: POST ${ADAPTER2}@gen1=True (establish prior status)" +CODE=$(post_adapter_status "$CLUSTER_ID" "$ADAPTER2" 1 "True") +assert_http "initial ${ADAPTER2} True accepted" "201" "$CODE" + +log_step "Setup: POST ${ADAPTER1}@gen1=True" +post_adapter_status "$CLUSTER_ID" "$ADAPTER1" 1 "True" > /dev/null + +log_step "GET cluster — baseline (both adapters True → cluster Ready)" +BASELINE=$(get_cluster "$CLUSTER_ID") +show_state "baseline" "$BASELINE" +PRE_AVAIL_STATUS=$(condition_field "$BASELINE" Available status) +PRE_AVAIL_OBSGEN=$(condition_field "$BASELINE" Available observed_generation) +PRE_AVAIL_UPDATED=$(condition_field "$BASELINE" Available last_updated_time) +PRE_AVAIL_TRANSITION=$(condition_field "$BASELINE" Available last_transition_time) +PRE_READY_STATUS=$(condition_field "$BASELINE" Ready status) +PRE_READY_OBSGEN=$(condition_field "$BASELINE" Ready observed_generation) +PRE_READY_UPDATED=$(condition_field "$BASELINE" Ready last_updated_time) +PRE_READY_TRANSITION=$(condition_field "$BASELINE" Ready last_transition_time) +log_received "Available=$PRE_AVAIL_STATUS Ready=$PRE_READY_STATUS" + +sleep 1 + +# ── Event ───────────────────────────────────────────────────────────────────── +log_step "POST ${ADAPTER2}@gen1=Unknown (P3 rule — should be discarded)" +CODE=$(post_adapter_status "$CLUSTER_ID" "$ADAPTER2" 1 "Unknown") + +echo "" +echo " API response (P3 rule must discard Unknown and return 204):" +assert_http "Unknown report discarded" "204" "$CODE" + +# ── Validate ────────────────────────────────────────────────────────────────── +log_step "GET cluster — all fields must be byte-identical to baseline" +CLUSTER=$(get_cluster "$CLUSTER_ID") +show_state "after Unknown" "$CLUSTER" + +echo "" +echo " Available condition (must be identical to baseline — no aggregation ran):" +assert_eq "Available.status" "$PRE_AVAIL_STATUS" "$(condition_field "$CLUSTER" Available status)" +assert_eq "Available.observed_generation" "$PRE_AVAIL_OBSGEN" "$(condition_field "$CLUSTER" Available observed_generation)" +assert_eq "Available.last_updated_time" "$PRE_AVAIL_UPDATED" "$(condition_field "$CLUSTER" Available last_updated_time)" +assert_eq "Available.last_transition_time" "$PRE_AVAIL_TRANSITION" "$(condition_field "$CLUSTER" Available last_transition_time)" + +echo "" +echo " Ready condition (must be identical to baseline):" +assert_eq "Ready.status" "$PRE_READY_STATUS" "$(condition_field "$CLUSTER" Ready status)" +assert_eq "Ready.observed_generation" "$PRE_READY_OBSGEN" "$(condition_field "$CLUSTER" Ready observed_generation)" +assert_eq "Ready.last_updated_time" "$PRE_READY_UPDATED" "$(condition_field "$CLUSTER" Ready last_updated_time)" +assert_eq "Ready.last_transition_time" "$PRE_READY_TRANSITION" "$(condition_field "$CLUSTER" Ready last_transition_time)" + +echo "" +echo " Stored adapter status retains its prior True value (not overwritten):" +STATUSES=$(get_statuses "$CLUSTER_ID") +VAL_STATUS=$(echo "$STATUSES" | jq -r --arg a "$ADAPTER2" \ + '.items[] | select(.adapter==$a) | .conditions[] | select(.type=="Available") | .status') +assert_eq "${ADAPTER2} stored Available still True" "True" "$VAL_STATUS" + +# ── Conclusion ──────────────────────────────────────────────────────────────── +echo "" +echo -e "${BOLD}Conclusion:${NC}" +echo " The P3 rule discarded the Unknown report before any state was changed:" +echo " • The server returned 204 — no adapter status was persisted" +echo " • The stored adapter status still shows Available=True (unchanged)" +echo " • All cluster condition fields are byte-identical to the pre-event snapshot" +echo " • No aggregation was triggered — Ready.LUT was not refreshed" +echo " Unknown means the adapter doesn't yet know its state. The cluster should" +echo " not regress from a known-good state just because an adapter is uncertain." +echo "" +test_summary diff --git a/test/e2e-curl/12-unknown-first/test.sh b/test/e2e-curl/12-unknown-first/test.sh new file mode 100755 index 0000000..28583ff --- /dev/null +++ b/test/e2e-curl/12-unknown-first/test.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +# Test 12 — Unknown first report: P3 rule discards ALL Unknown reports (204) + +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "$SCRIPT_DIR/../common.sh" + +# ── Purpose ─────────────────────────────────────────────────────────────────── +echo -e "\n${BOLD}Test 12: Unknown report (first or subsequent) → always discarded (204)${NC}" +echo "" +echo " The P3 rule discards every report where Available=Unknown — including" +echo " the very first report from an adapter that has never reported before." +echo " No adapter status is persisted. No aggregation runs. Cluster conditions" +echo " stay at their initial False@gen1 state. A second Unknown is also discarded." +echo "" +echo -e " ${YELLOW}Starting state:${NC} Fresh cluster, gen=1, no adapter statuses at all" +echo -e " ${YELLOW}Event 1:${NC} POST ${ADAPTER2}@gen1=Unknown (first-ever report)" +echo -e " ${YELLOW}Event 2:${NC} POST ${ADAPTER2}@gen1=Unknown (second report)" +echo -e " ${YELLOW}Expected:${NC} Both return 204; no adapter status stored;" +echo -e " ${YELLOW} ${NC} cluster conditions unchanged from initial state" +echo "" + +# ── Setup ───────────────────────────────────────────────────────────────────── +CLUSTER_NAME="tc12-$(rand_hex4)" +log_step "Creating cluster '$CLUSTER_NAME'" +CLUSTER=$(create_cluster "$CLUSTER_NAME") +CLUSTER_ID=$(echo "$CLUSTER" | jq -r '.id') +log_received "cluster id=$CLUSTER_ID" + +log_step "GET cluster — initial state (no adapter statuses)" +INITIAL=$(get_cluster "$CLUSTER_ID") +show_state "initial" "$INITIAL" +INIT_AVAIL_STATUS=$(condition_field "$INITIAL" Available status) +INIT_AVAIL_OBSGEN=$(condition_field "$INITIAL" Available observed_generation) +INIT_AVAIL_UPDATED=$(condition_field "$INITIAL" Available last_updated_time) +INIT_AVAIL_TRANSITION=$(condition_field "$INITIAL" Available last_transition_time) +INIT_READY_STATUS=$(condition_field "$INITIAL" Ready status) +INIT_READY_OBSGEN=$(condition_field "$INITIAL" Ready observed_generation) +INIT_READY_UPDATED=$(condition_field "$INITIAL" Ready last_updated_time) +INIT_READY_TRANSITION=$(condition_field "$INITIAL" Ready last_transition_time) + +assert_eq "initial Available.status" "False" "$INIT_AVAIL_STATUS" +assert_eq "initial Ready.status" "False" "$INIT_READY_STATUS" +log_received "Available.LUT=$INIT_AVAIL_UPDATED" + +# ── Event 1: first-ever Unknown report ──────────────────────────────────────── +log_step "POST ${ADAPTER2}@gen1=Unknown (first-ever report — P3 rule applies)" +CODE=$(post_adapter_status "$CLUSTER_ID" "$ADAPTER2" 1 "Unknown") + +echo "" +echo " API response (P3 discards ALL Unknown, including the first-ever report):" +assert_http "first Unknown discarded (204)" "204" "$CODE" + +# ── Validate: no adapter status stored ──────────────────────────────────────── +log_step "GET /statuses — adapter record must NOT exist" +STATUSES=$(get_statuses "$CLUSTER_ID") +VAL_COUNT=$(count_adapter_status "$STATUSES" "$ADAPTER2") + +echo "" +echo " Adapter status persistence (report was discarded — nothing stored):" +assert_eq "${ADAPTER2} status record NOT stored" "0" "$VAL_COUNT" + +# ── Validate: cluster conditions completely unchanged ───────────────────────── +log_step "GET cluster — conditions must be byte-identical to initial state" +CLUSTER=$(get_cluster "$CLUSTER_ID") +show_state "after first Unknown" "$CLUSTER" + +echo "" +echo " Available condition (unchanged — P3 discarded before any write):" +assert_eq "Available.status" "$INIT_AVAIL_STATUS" "$(condition_field "$CLUSTER" Available status)" +assert_eq "Available.observed_generation" "$INIT_AVAIL_OBSGEN" "$(condition_field "$CLUSTER" Available observed_generation)" +assert_eq "Available.last_updated_time" "$INIT_AVAIL_UPDATED" "$(condition_field "$CLUSTER" Available last_updated_time)" +assert_eq "Available.last_transition_time" "$INIT_AVAIL_TRANSITION" "$(condition_field "$CLUSTER" Available last_transition_time)" + +echo "" +echo " Ready condition (unchanged):" +assert_eq "Ready.status" "$INIT_READY_STATUS" "$(condition_field "$CLUSTER" Ready status)" +assert_eq "Ready.observed_generation" "$INIT_READY_OBSGEN" "$(condition_field "$CLUSTER" Ready observed_generation)" +assert_eq "Ready.last_updated_time" "$INIT_READY_UPDATED" "$(condition_field "$CLUSTER" Ready last_updated_time)" +assert_eq "Ready.last_transition_time" "$INIT_READY_TRANSITION" "$(condition_field "$CLUSTER" Ready last_transition_time)" + +# ── Event 2: second Unknown report ──────────────────────────────────────────── +echo "" +log_step "POST ${ADAPTER2}@gen1=Unknown (second report — also discarded)" +CODE2=$(post_adapter_status "$CLUSTER_ID" "$ADAPTER2" 1 "Unknown") + +echo "" +echo " API response (second Unknown also discarded):" +assert_http "second Unknown discarded (204)" "204" "$CODE2" + +# ── Conclusion ──────────────────────────────────────────────────────────────── +echo "" +echo -e "${BOLD}Conclusion:${NC}" +echo " The P3 rule applies uniformly to ALL Available=Unknown reports:" +echo " • Both the first and second Unknown reports returned 204" +echo " • No adapter status was written to the database (count=0)" +echo " • No aggregation ran — cluster conditions are byte-identical to initial state" +echo " Unknown means the adapter hasn't determined its state yet. The cluster" +echo " should not be affected at all — not even by a first-time adapter report." +echo "" +test_summary diff --git a/test/e2e-curl/README.md b/test/e2e-curl/README.md new file mode 100644 index 0000000..ace5e42 --- /dev/null +++ b/test/e2e-curl/README.md @@ -0,0 +1,84 @@ +# Status Aggregation — E2E Curl Tests + +Black-box tests for the cluster status aggregation logic. Each test drives the +live API with `curl`, then asserts on the `Available` and `Ready` synthetic +conditions returned by `GET /clusters/:id`. + +## Prerequisites + +- `curl`, `jq` +- A running HyperFleet API server with auth disabled and two required adapters + configured: + +```bash +HYPERFLEET_ADAPTERS_REQUIRED_CLUSTER='["adapter1","adapter2"]' make run-no-auth +``` + +Override the adapter names to match your server configuration: + +```bash +ADAPTER1=dns ADAPTER2=validation ./run_all.sh +``` + +## Running + +```bash +# All 12 tests +./run_all.sh + +# Selected tests by prefix +./run_all.sh 03 07 09 +``` + +Each test creates its own cluster (name suffixed with a random hex, e.g. +`tc07-3a1f`), so tests are fully isolated and can run against a shared server. + +## Test Catalogue + +| # | Directory | Scenario | +|---|-----------|----------| +| 01 | `01-initial-state` | Cluster just created — no adapter reports yet; both conditions False | +| 02 | `02-partial-adapters` | One required adapter True, the other missing — Available stays False | +| 03 | `03-all-adapters-ready` | All adapters True at gen=1 — Available=True, Ready=True | +| 04 | `04-generation-bump` | Cluster spec changes (gen→2) — Available frozen, Ready resets to False | +| 05 | `05-mixed-generations` | One adapter at gen1, one at gen2 — `all_at_X=false`, Available preserved | +| 06 | `06-stale-report` | Adapter reports an older `observed_generation` — discarded (204) | +| 07 | `07-all-adapters-new-gen` | Both adapters converge at gen2 — Available.ObsGen advances, Ready=True | +| 08 | `08-adapter-goes-false` | One required adapter flips to False — both Available and Ready become False | +| 09 | `09-stable-true` | Heartbeat re-reports (same status, same gen) — both LUTs refresh to new min(LRT) | +| 10 | `10-stable-false` | Heartbeat False re-reports — Available.LUT preserved (no change), Ready.LUT refreshes | +| 11 | `11-unknown-subsequent` | `Available=Unknown` on a subsequent report — discarded (204), state unchanged | +| 12 | `12-unknown-first` | `Available=Unknown` on a first report — discarded (204), nothing stored | + +## Structure + +``` +e2e-curl/ + common.sh # Shared helpers: API wrappers, assertions, logging + run_all.sh # Suite runner with per-test pass/fail tracking + 01-initial-state/ + test.sh + ... +``` + +### `common.sh` helpers + +| Helper | Purpose | +|--------|---------| +| `create_cluster NAME` | `POST /clusters` | +| `patch_cluster ID SPEC` | `PATCH /clusters/:id` (bumps generation) | +| `get_cluster ID` | `GET /clusters/:id` | +| `post_adapter_status ID ADAPTER GEN STATUS` | `POST /clusters/:id/statuses` | +| `condition_field JSON TYPE FIELD` | Extract a field from a named condition | +| `assert_eq LABEL EXPECTED ACTUAL` | Equality assertion | +| `assert_changed LABEL BEFORE AFTER` | Asserts a value changed | +| `assert_nonempty LABEL VALUE` | Asserts a value is non-null/non-empty | +| `show_state LABEL JSON` | One-line `Available=?@genN Ready=?@genN` summary | + +## Environment Variables + +| Variable | Default | Description | +|----------|---------|-------------| +| `HYPERFLEET_URL` | `http://localhost:8000` | Base URL of the API server | +| `ADAPTER1` | `adapter1` | Name of the first required adapter | +| `ADAPTER2` | `adapter2` | Name of the second required adapter | diff --git a/test/e2e-curl/common.sh b/test/e2e-curl/common.sh new file mode 100755 index 0000000..5b72cc1 --- /dev/null +++ b/test/e2e-curl/common.sh @@ -0,0 +1,184 @@ +#!/usr/bin/env bash +# common.sh — Shared helpers for HyperFleet status-aggregation e2e tests. +# +# Requires: curl, jq +# Server must be running with --enable-authz=false --enable-jwt=false +# and HYPERFLEET_CLUSTER_ADAPTERS='["dns","validation"]' + +BASE_URL="${HYPERFLEET_URL:-http://localhost:8000}" +API="$BASE_URL/api/hyperfleet/v1" + +# Required adapter names — must match HYPERFLEET_CLUSTER_ADAPTERS on the server. +# Override via env vars: ADAPTER1=dns ADAPTER2=validation ./run_all.sh +ADAPTER1="${ADAPTER1:-adapter1}" +ADAPTER2="${ADAPTER2:-adapter2}" + +# ── Counters ──────────────────────────────────────────────────────────────── +PASS_COUNT=0 +FAIL_COUNT=0 + +# ── Colours (disabled when not a terminal) ────────────────────────────────── +if [ -t 1 ]; then + RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m' + BOLD='\033[1m'; NC='\033[0m' +else + RED=''; GREEN=''; YELLOW=''; BOLD=''; NC='' +fi + +# ── Time helpers ───────────────────────────────────────────────────────────── +now_rfc3339() { + date -u +%Y-%m-%dT%H:%M:%SZ +} + +# Unique 4-char hex suffix — cheap, no external deps +rand_hex4() { + printf '%04x' $((RANDOM * RANDOM % 65536)) +} + +# ── Logging ────────────────────────────────────────────────────────────────── +log_step() { echo -e " ${YELLOW}>${NC} $*"; } + +# Print a feedback value received from a step (HTTP code, extracted field, etc.) +log_received() { echo " → $*"; } + +# Print a one-line Available+Ready state summary after a GET. +# Usage: show_state "label" "$CLUSTER_JSON" +show_state() { + local label="$1" json="$2" + local av_s av_g re_s re_g + av_s=$(condition_field "$json" Available status) + av_g=$(condition_field "$json" Available observed_generation) + re_s=$(condition_field "$json" Ready status) + re_g=$(condition_field "$json" Ready observed_generation) + echo " → ${label}: Available=${av_s}@gen${av_g} Ready=${re_s}@gen${re_g}" +} + +# ── Assertions ─────────────────────────────────────────────────────────────── +assert_eq() { + local label="$1" expected="$2" actual="$3" + if [ "$expected" = "$actual" ]; then + echo -e " ${GREEN}PASS${NC} [$label]: '$actual'" + PASS_COUNT=$((PASS_COUNT + 1)) + else + echo -e " ${RED}FAIL${NC} [$label]: expected='$expected' got='$actual'" + FAIL_COUNT=$((FAIL_COUNT + 1)) + fi +} + +assert_http() { + local label="$1" expected="$2" actual="$3" + assert_eq "HTTP $label" "$expected" "$actual" +} + +# Asserts two values are NOT equal (e.g. timestamp was refreshed) +assert_changed() { + local label="$1" before="$2" after="$3" + if [ "$before" != "$after" ]; then + echo -e " ${GREEN}PASS${NC} [$label]: changed from '$before' to '$after'" + PASS_COUNT=$((PASS_COUNT + 1)) + else + echo -e " ${RED}FAIL${NC} [$label]: expected change but value stayed '$after'" + FAIL_COUNT=$((FAIL_COUNT + 1)) + fi +} + +# Asserts a value is non-empty +assert_nonempty() { + local label="$1" actual="$2" + if [ -n "$actual" ] && [ "$actual" != "null" ]; then + echo -e " ${GREEN}PASS${NC} [$label]: '$actual'" + PASS_COUNT=$((PASS_COUNT + 1)) + else + echo -e " ${RED}FAIL${NC} [$label]: value is empty or null" + FAIL_COUNT=$((FAIL_COUNT + 1)) + fi +} + +# ── API helpers ─────────────────────────────────────────────────────────────── + +# Create a cluster. Prints JSON on stdout, exits non-zero on failure. +create_cluster() { + local name="$1" + local spec="${2:-{\"v\":1}}" + curl -sf -X POST "$API/clusters" \ + -H "Content-Type: application/json" \ + -d "{\"kind\":\"Cluster\",\"name\":\"$name\",\"spec\":$spec}" +} + +# Patch a cluster spec — bumps generation when spec content changes. +patch_cluster() { + local id="$1" + local spec="${2:-{\"v\":2}}" + curl -sf -X PATCH "$API/clusters/$id" \ + -H "Content-Type: application/json" \ + -d "{\"spec\":$spec}" +} + +# Get a single cluster. Prints JSON on stdout. +get_cluster() { + local id="$1" + curl -sf "$API/clusters/$id" +} + +# Post adapter status. Returns the HTTP status code as a string ("201", "204", ...). +# Usage: code=$(post_adapter_status CLUSTER_ID ADAPTER GEN STATUS) +# STATUS: True | False | Unknown +post_adapter_status() { + local cluster_id="$1" adapter="$2" gen="$3" available="$4" + curl -s -o /dev/null -w "%{http_code}" \ + -X POST "$API/clusters/$cluster_id/statuses" \ + -H "Content-Type: application/json" \ + -d "{ + \"adapter\": \"$adapter\", + \"observed_generation\": $gen, + \"observed_time\": \"$(now_rfc3339)\", + \"conditions\": [ + {\"type\": \"Available\", \"status\": \"$available\", \"reason\": \"Testing\"}, + {\"type\": \"Applied\", \"status\": \"True\", \"reason\": \"Testing\"}, + {\"type\": \"Health\", \"status\": \"True\", \"reason\": \"Testing\"} + ] + }" +} + +# List adapter statuses for a cluster. Prints JSON on stdout. +get_statuses() { + local cluster_id="$1" + curl -sf "$API/clusters/$cluster_id/statuses" +} + +# ── JSON field extraction ───────────────────────────────────────────────────── + +# Extract a field from a specific condition type in cluster.status.conditions +# Usage: val=$(condition_field "$CLUSTER_JSON" Available status) +condition_field() { + local json="$1" ctype="$2" field="$3" + echo "$json" | jq -r ".status.conditions[] | select(.type == \"$ctype\") | .$field" +} + +# Count adapter statuses stored for a cluster (by adapter name) +count_adapter_status() { + local statuses_json="$1" adapter="$2" + echo "$statuses_json" | jq "[.items[] | select(.adapter == \"$adapter\")] | length" +} + +# ── Test summary ────────────────────────────────────────────────────────────── +test_summary() { + local total=$((PASS_COUNT + FAIL_COUNT)) + echo "" + if [ "$FAIL_COUNT" -eq 0 ]; then + echo -e "${GREEN}${BOLD}ALL PASSED${NC} ($PASS_COUNT/$total)" + return 0 + else + echo -e "${RED}${BOLD}FAILED${NC} ($FAIL_COUNT/$total failed)" + return 1 + fi +} + +# ── Preflight check ─────────────────────────────────────────────────────────── +check_server() { + if ! curl -sf "$BASE_URL/api/hyperfleet/v1/clusters?pageSize=1" > /dev/null 2>&1; then + echo -e "${RED}ERROR${NC}: Cannot reach $BASE_URL" + echo "Start the server: HYPERFLEET_CLUSTER_ADAPTERS='[\"dns\",\"validation\"]' make run-no-auth" + exit 1 + fi +} diff --git a/test/e2e-curl/run_all.sh b/test/e2e-curl/run_all.sh new file mode 100755 index 0000000..18ff591 --- /dev/null +++ b/test/e2e-curl/run_all.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +# run_all.sh — Execute all 12 status-aggregation e2e tests and report a summary. +# +# Usage: +# ./run_all.sh # runs all tests +# ./run_all.sh 03 07 # runs only tests 03 and 07 +# +# Prerequisites: +# • Server running: HYPERFLEET_CLUSTER_ADAPTERS='["dns","validation"]' make run-no-auth +# • Tools: curl, jq + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "$SCRIPT_DIR/common.sh" + +echo -e "${BOLD}HyperFleet Status Aggregation — E2E Tests${NC}" +echo -e "Server: $BASE_URL" +echo -e "Required adapters: dns, validation" +echo "" + +check_server + +# ── Discover tests ──────────────────────────────────────────────────────────── +TESTS=( + "01-initial-state" + "02-partial-adapters" + "03-all-adapters-ready" + "04-generation-bump" + "05-mixed-generations" + "06-stale-report" + "07-all-adapters-new-gen" + "08-adapter-goes-false" + "09-stable-true" + "10-stable-false" + "11-unknown-subsequent" + "12-unknown-first" +) + +# Filter by argument prefixes if provided (e.g. "03" "07") +SELECTED=() +if [ $# -gt 0 ]; then + for filter in "$@"; do + for t in "${TESTS[@]}"; do + [[ "$t" == "${filter}"* ]] && SELECTED+=("$t") + done + done +else + SELECTED=("${TESTS[@]}") +fi + +# ── Run each test ───────────────────────────────────────────────────────────── +TOTAL_PASS=0 +TOTAL_FAIL=0 +SUITE_FAIL=() + +for test_name in "${SELECTED[@]}"; do + test_file="$SCRIPT_DIR/$test_name/test.sh" + if [ ! -f "$test_file" ]; then + echo -e "${YELLOW}SKIP${NC} $test_name (no test.sh found)" + continue + fi + + echo -e "${BOLD}── $test_name${NC}" + set +e + bash "$test_file" + rc=$? + set -e + sleep 1 # let the server drain before the next test + + # Read PASS_COUNT and FAIL_COUNT set by the child process via exported env. + # Since child is a subshell we cannot inherit counters directly. + # Instead each test.sh writes its result to stdout with the summary line. + if [ $rc -eq 0 ]; then + TOTAL_PASS=$((TOTAL_PASS + 1)) + echo "" + else + TOTAL_FAIL=$((TOTAL_FAIL + 1)) + SUITE_FAIL+=("$test_name") + echo "" + fi +done + +# ── Suite summary ───────────────────────────────────────────────────────────── +echo "══════════════════════════════════════════════" +TOTAL=$((TOTAL_PASS + TOTAL_FAIL)) +if [ "$TOTAL_FAIL" -eq 0 ]; then + echo -e "${GREEN}${BOLD}ALL TESTS PASSED${NC} ($TOTAL_PASS/$TOTAL)" + exit 0 +else + echo -e "${RED}${BOLD}$TOTAL_FAIL TEST(S) FAILED${NC} ($TOTAL_PASS/$TOTAL passed)" + for f in "${SUITE_FAIL[@]}"; do + echo -e " ${RED}✗${NC} $f" + done + exit 1 +fi diff --git a/test/integration/adapter_status_test.go b/test/integration/adapter_status_test.go index 14d5a5a..b9ec79e 100644 --- a/test/integration/adapter_status_test.go +++ b/test/integration/adapter_status_test.go @@ -432,9 +432,9 @@ func TestAdapterStatusIdempotency(t *testing.T) { To(Equal(openapi.AdapterConditionStatusTrue), "Conditions should be updated to latest") } -// TestClusterStatusPost_FirstUnknownAccepted tests that first status reports with Unknown -// Available condition are accepted, subsequent ones are rejected (HYPERFLEET-657) -func TestClusterStatusPost_FirstUnknownAccepted(t *testing.T) { +// TestClusterStatusPost_UnknownAvailableAlwaysDiscarded tests that status reports with +// Available=Unknown are always discarded (P3 rule), regardless of whether it's the first report. +func TestClusterStatusPost_UnknownAvailableAlwaysDiscarded(t *testing.T) { h, client := test.RegisterIntegration(t) account := h.NewRandAccount() @@ -468,30 +468,26 @@ func TestClusterStatusPost_FirstUnknownAccepted(t *testing.T) { nil, ) - // First report with Unknown Available condition: should be accepted + // First report with Unknown Available condition: discarded per P3 (204 No Content) resp, err := client.PostClusterStatusesWithResponse( ctx, cluster.ID, openapi.PostClusterStatusesJSONRequestBody(statusInput), test.WithAuthToken(ctx), ) Expect(err).NotTo(HaveOccurred(), "Error posting cluster status: %v", err) Expect(resp.StatusCode()). - To(Equal(http.StatusCreated), "Expected 201 Created for first status with Unknown Available condition") + To(Equal(http.StatusNoContent), "Expected 204 No Content: Available=Unknown is always discarded") - // Verify status was stored + // Verify status was NOT stored listResp, err := client.GetClusterStatusesWithResponse(ctx, cluster.ID, nil, test.WithAuthToken(ctx)) Expect(err).NotTo(HaveOccurred()) Expect(listResp.JSON200).NotTo(BeNil()) - found := false for _, s := range listResp.JSON200.Items { - if s.Adapter == "test-adapter-unknown" { - found = true - break - } + Expect(s.Adapter).NotTo(Equal("test-adapter-unknown"), + "Status with Available=Unknown must not be stored") } - Expect(found).To(BeTrue(), "First status with Unknown Available condition should be stored") - // Subsequent report with same adapter: should be rejected (204 No Content) + // Subsequent report with same adapter: also discarded (204 No Content) resp2, err := client.PostClusterStatusesWithResponse( ctx, cluster.ID, openapi.PostClusterStatusesJSONRequestBody(statusInput), test.WithAuthToken(ctx), @@ -501,9 +497,9 @@ func TestClusterStatusPost_FirstUnknownAccepted(t *testing.T) { To(Equal(http.StatusNoContent), "Expected 204 No Content for subsequent Unknown status report") } -// TestNodePoolStatusPost_FirstUnknownAccepted tests that first status reports with Unknown -// Available condition are accepted, subsequent ones are rejected (HYPERFLEET-657) -func TestNodePoolStatusPost_FirstUnknownAccepted(t *testing.T) { +// TestNodePoolStatusPost_UnknownAvailableAlwaysDiscarded tests that status reports with +// Available=Unknown are always discarded (P3 rule), regardless of whether it's the first report. +func TestNodePoolStatusPost_UnknownAvailableAlwaysDiscarded(t *testing.T) { h, client := test.RegisterIntegration(t) account := h.NewRandAccount() @@ -537,32 +533,28 @@ func TestNodePoolStatusPost_FirstUnknownAccepted(t *testing.T) { nil, ) - // First report with Unknown Available condition: should be accepted + // First report with Unknown Available condition: discarded per P3 (204 No Content) resp, err := client.PostNodePoolStatusesWithResponse( ctx, nodePool.OwnerID, nodePool.ID, openapi.PostNodePoolStatusesJSONRequestBody(statusInput), test.WithAuthToken(ctx), ) Expect(err).NotTo(HaveOccurred(), "Error posting nodepool status: %v", err) Expect(resp.StatusCode()). - To(Equal(http.StatusCreated), "Expected 201 Created for first status with Unknown Available condition") + To(Equal(http.StatusNoContent), "Expected 204 No Content: Available=Unknown is always discarded") - // Verify status was stored + // Verify status was NOT stored listResp, err := client.GetNodePoolsStatusesWithResponse( ctx, nodePool.OwnerID, nodePool.ID, nil, test.WithAuthToken(ctx), ) Expect(err).NotTo(HaveOccurred()) Expect(listResp.JSON200).NotTo(BeNil()) - found := false for _, s := range listResp.JSON200.Items { - if s.Adapter == "test-nodepool-adapter-unknown" { - found = true - break - } + Expect(s.Adapter).NotTo(Equal("test-nodepool-adapter-unknown"), + "Status with Available=Unknown must not be stored") } - Expect(found).To(BeTrue(), "First status with Unknown Available condition should be stored") - // Subsequent report with same adapter: should be rejected (204 No Content) + // Subsequent report with same adapter: also discarded (204 No Content) resp2, err := client.PostNodePoolStatusesWithResponse( ctx, nodePool.OwnerID, nodePool.ID, openapi.PostNodePoolStatusesJSONRequestBody(statusInput), test.WithAuthToken(ctx), @@ -572,8 +564,8 @@ func TestNodePoolStatusPost_FirstUnknownAccepted(t *testing.T) { To(Equal(http.StatusNoContent), "Expected 204 No Content for subsequent Unknown status report") } -// TestClusterStatusPost_MultipleConditionsWithUnknownAvailable tests that -// first report with Unknown Available is accepted, subsequent ones rejected (HYPERFLEET-657) +// TestClusterStatusPost_MultipleConditionsWithUnknownAvailable tests that reports with +// Available=Unknown are discarded (P3 rule) even when other conditions are present. func TestClusterStatusPost_MultipleConditionsWithUnknownAvailable(t *testing.T) { h, client := test.RegisterIntegration(t) @@ -616,16 +608,16 @@ func TestClusterStatusPost_MultipleConditionsWithUnknownAvailable(t *testing.T) nil, ) - // First report with Unknown Available condition: should be accepted + // First report with Available=Unknown among multiple conditions: discarded per P3 (204 No Content) resp, err := client.PostClusterStatusesWithResponse( ctx, cluster.ID, openapi.PostClusterStatusesJSONRequestBody(statusInput), test.WithAuthToken(ctx), ) Expect(err).NotTo(HaveOccurred(), "Error posting cluster status: %v", err) - Expect(resp.StatusCode()).To(Equal(http.StatusCreated), - "Expected 201 Created for first report with Available=Unknown among multiple conditions") + Expect(resp.StatusCode()).To(Equal(http.StatusNoContent), + "Expected 204 No Content: Available=Unknown is always discarded regardless of other conditions") - // Subsequent report: should be rejected (204 No Content) + // Subsequent report: also discarded (204 No Content) resp2, err := client.PostClusterStatusesWithResponse( ctx, cluster.ID, openapi.PostClusterStatusesJSONRequestBody(statusInput), test.WithAuthToken(ctx), @@ -883,9 +875,9 @@ func TestClusterStatusPost_MissingMandatoryConditionsRejected(t *testing.T) { Expect(storedConditionTypes["Health2"]).To(BeFalse(), "Health2 should not be present") } -// TestClusterStatusPost_FirstUnknownAcceptedSubsequentRejected tests that first status with -// Unknown Available is accepted, subsequent ones are rejected -func TestClusterStatusPost_FirstUnknownAcceptedSubsequentRejected(t *testing.T) { +// TestClusterStatusPost_UnknownAvailableNeverStored tests that Available=Unknown reports +// are never stored, regardless of first or subsequent attempts. +func TestClusterStatusPost_UnknownAvailableNeverStored(t *testing.T) { h, client := test.RegisterIntegration(t) account := h.NewRandAccount() @@ -895,7 +887,7 @@ func TestClusterStatusPost_FirstUnknownAcceptedSubsequentRejected(t *testing.T) cluster, err := h.Factories.NewClusters(h.NewID()) Expect(err).NotTo(HaveOccurred()) - // Send first status with all mandatory conditions but Available=Unknown + // Send status with all mandatory conditions but Available=Unknown statusWithUnknown := newAdapterStatusRequest( "adapter1", cluster.Generation, @@ -922,29 +914,30 @@ func TestClusterStatusPost_FirstUnknownAcceptedSubsequentRejected(t *testing.T) nil, ) - // First report: should be accepted + // First report: discarded per P3 (204 No Content) resp, err := client.PostClusterStatusesWithResponse( ctx, cluster.ID, openapi.PostClusterStatusesJSONRequestBody(statusWithUnknown), test.WithAuthToken(ctx), ) Expect(err).NotTo(HaveOccurred()) - Expect(resp.StatusCode()).To(Equal(http.StatusCreated), "First status with Unknown Available should be accepted") + Expect(resp.StatusCode()).To(Equal(http.StatusNoContent), + "Available=Unknown status must always be discarded") - // Verify status was stored + // Verify status was NOT stored respGet, err := client.GetClusterStatusesWithResponse(ctx, cluster.ID, nil, test.WithAuthToken(ctx)) Expect(err).NotTo(HaveOccurred()) Expect(respGet.StatusCode()).To(Equal(http.StatusOK)) Expect(respGet.JSON200).ToNot(BeNil()) - Expect(len(respGet.JSON200.Items)).To(Equal(1), "First status with Unknown Available should be stored") + Expect(len(respGet.JSON200.Items)).To(Equal(0), "Available=Unknown status must not be stored") - // Subsequent report: should be rejected (204 No Content) + // Subsequent report: also discarded (204 No Content) resp2, err := client.PostClusterStatusesWithResponse( ctx, cluster.ID, openapi.PostClusterStatusesJSONRequestBody(statusWithUnknown), test.WithAuthToken(ctx), ) Expect(err).NotTo(HaveOccurred()) Expect(resp2.StatusCode()).To(Equal(http.StatusNoContent), - "Subsequent status with Unknown Available should be rejected") + "Subsequent Available=Unknown status must also be discarded") } // TestClusterStatusPost_DuplicateConditionsRejected tests that adapter status updates