diff --git a/core/cmd/cnpgi/instance.go b/core/cmd/cnpgi/instance.go index 4be079a97..34ed22592 100644 --- a/core/cmd/cnpgi/instance.go +++ b/core/cmd/cnpgi/instance.go @@ -57,6 +57,10 @@ var instanceCmd = &cobra.Command{ clusterNamespace, _ := cmd.Flags().GetString("cluster-namespace") pgData, _ := cmd.Flags().GetString("pgdata") + // Seed the backup counter series at 0 so a first failure (or success) + // is a visible increment for rate()/increase()-based panels. + opentelemetry.InitPluginBackupSeries(cmd.Context(), clusterName) + // One sidecar serves every phase of the instance: the restore hooks // while the cluster bootstraps from a Klio backup, backup and WAL // archiving afterwards. Each service picks its repository per request diff --git a/core/internal/cnpgi/backup.go b/core/internal/cnpgi/backup.go index 5c3272eb6..15b51b37b 100644 --- a/core/internal/cnpgi/backup.go +++ b/core/internal/cnpgi/backup.go @@ -117,15 +117,15 @@ func (b backupServiceImplementation) Backup( ) backupStart := time.Now() - recordBackupStart(ctx) - defer recordBackupFinished(ctx) + recordBackupStart(ctx, cluster.Name) + defer recordBackupFinished(ctx, cluster.Name) metadata, err := b.runBackup( ctx, backupName, ) if err != nil { - recordBackupFailure(ctx, time.Since(backupStart), err) + recordBackupFailure(ctx, cluster.Name, time.Since(backupStart), err) span.RecordError(err) span.SetStatus(codes.Error, "backup failed") @@ -137,14 +137,14 @@ func (b backupServiceImplementation) Backup( // verification is folded into the successful-backup recording below. corruption, verifyErr := b.runVerify(ctx, backupName) if corruption { - recordBackupFailure(ctx, time.Since(backupStart), verifyErr) + recordBackupFailure(ctx, cluster.Name, time.Since(backupStart), verifyErr) span.RecordError(verifyErr) span.SetStatus(codes.Error, "verification detected corruption") return nil, verifyErr } - recordBackupSuccess(ctx, time.Since(backupStart)) + recordBackupSuccess(ctx, cluster.Name, time.Since(backupStart)) return &backup.BackupResult{ BackupName: backupName, diff --git a/core/internal/cnpgi/metrics.go b/core/internal/cnpgi/metrics.go index fe0141f88..908ff563e 100644 --- a/core/internal/cnpgi/metrics.go +++ b/core/internal/cnpgi/metrics.go @@ -23,44 +23,63 @@ import ( "context" "time" + "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/metric" "github.com/cloudnative-pg/klio/core/internal/opentelemetry" ) +// clusterAttr returns the `cluster_name` attribute every plugin backup metric +// carries, so panels can attribute backup activity to a specific PostgreSQL +// cluster even when several clusters share one namespace. +func clusterAttr(clusterName string) attribute.KeyValue { + return opentelemetry.AttributeKeyClusterName.Of(clusterName) +} + // recordBackupStart records that a backup has started. Callers must pair this // with a deferred recordBackupFinished so the in-progress counter decrements // on every exit path, including panics. -func recordBackupStart(ctx context.Context) { - opentelemetry.PluginBackup.LatestStartTime.Record(ctx, time.Now().Unix()) - opentelemetry.PluginBackup.InProgress.Add(ctx, 1) +func recordBackupStart(ctx context.Context, clusterName string) { + cluster := clusterAttr(clusterName) + opentelemetry.PluginBackup.LatestStartTime.Record(ctx, time.Now().Unix(), + metric.WithAttributes(cluster)) + opentelemetry.PluginBackup.InProgress.Add(ctx, 1, metric.WithAttributes(cluster)) } // recordBackupFinished decrements the in-progress counter. Always invoke via // defer immediately after recordBackupStart so concurrent backup accounting -// stays correct even when a backup panics or returns early. -func recordBackupFinished(ctx context.Context) { - opentelemetry.PluginBackup.InProgress.Add(ctx, -1) +// stays correct even when a backup panics or returns early. It must pass the +// same clusterName as recordBackupStart so the up/down counter cancels out per +// cluster. +func recordBackupFinished(ctx context.Context, clusterName string) { + opentelemetry.PluginBackup.InProgress.Add(ctx, -1, + metric.WithAttributes(clusterAttr(clusterName))) } // recordBackupSuccess records a successful backup completion. -func recordBackupSuccess(ctx context.Context, duration time.Duration) { - opentelemetry.PluginBackup.LatestCompletionTime.Record(ctx, time.Now().Unix()) - opentelemetry.PluginBackup.LatestDuration.Record(ctx, duration.Seconds()) +func recordBackupSuccess(ctx context.Context, clusterName string, duration time.Duration) { + cluster := clusterAttr(clusterName) + opentelemetry.PluginBackup.LatestCompletionTime.Record(ctx, time.Now().Unix(), + metric.WithAttributes(cluster)) + opentelemetry.PluginBackup.LatestDuration.Record(ctx, duration.Seconds(), + metric.WithAttributes(cluster)) opentelemetry.PluginBackup.Duration.Record(ctx, duration.Seconds(), - metric.WithAttributes(opentelemetry.OutcomeSuccess.Attribute())) + metric.WithAttributes(cluster, opentelemetry.OutcomeSuccess.Attribute())) opentelemetry.PluginBackup.Runs.Add(ctx, 1, - metric.WithAttributes(opentelemetry.OutcomeSuccess.Attribute())) + metric.WithAttributes(cluster, opentelemetry.OutcomeSuccess.Attribute())) } // recordBackupFailure records a failed backup. -func recordBackupFailure(ctx context.Context, duration time.Duration, err error) { +func recordBackupFailure(ctx context.Context, clusterName string, duration time.Duration, err error) { + cluster := clusterAttr(clusterName) category := classifyRunBackupError(ctx, err) - opentelemetry.PluginBackup.LatestFailureTime.Record(ctx, time.Now().Unix()) + opentelemetry.PluginBackup.LatestFailureTime.Record(ctx, time.Now().Unix(), + metric.WithAttributes(cluster)) opentelemetry.PluginBackup.Duration.Record(ctx, duration.Seconds(), - metric.WithAttributes(opentelemetry.OutcomeFailure.Attribute())) + metric.WithAttributes(cluster, opentelemetry.OutcomeFailure.Attribute())) opentelemetry.PluginBackup.Runs.Add(ctx, 1, metric.WithAttributes( + cluster, opentelemetry.OutcomeFailure.Attribute(), opentelemetry.AttributeKeyFailureCategory.Of(category.Name), )) diff --git a/core/internal/cnpgi/metrics_test.go b/core/internal/cnpgi/metrics_test.go index 666e7e5fd..650ba22d0 100644 --- a/core/internal/cnpgi/metrics_test.go +++ b/core/internal/cnpgi/metrics_test.go @@ -29,6 +29,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" sdkmetric "go.opentelemetry.io/otel/sdk/metric" "go.opentelemetry.io/otel/sdk/metric/metricdata" @@ -36,6 +37,10 @@ import ( "github.com/cloudnative-pg/klio/core/internal/opentelemetry" ) +// testClusterName is the PostgreSQL cluster name every record* call in these +// tests tags its metrics with. +const testClusterName = "cluster-test" + // setupTestMeter installs a test MeterProvider with a ManualReader, // re-creates all instruments against it, and returns the reader. func setupTestMeter(t *testing.T) *sdkmetric.ManualReader { @@ -99,12 +104,10 @@ func findInProgressValue(rm metricdata.ResourceMetrics) (int64, bool) { return 0, false } -func findInt64SumDataPoints( - rm metricdata.ResourceMetrics, name string, -) []metricdata.DataPoint[int64] { +func findInt64SumDataPoints(rm metricdata.ResourceMetrics) []metricdata.DataPoint[int64] { for _, sm := range rm.ScopeMetrics { for _, m := range sm.Metrics { - if m.Name == name { + if m.Name == opentelemetry.PluginBackupRunsMetric { if s, ok := m.Data.(metricdata.Sum[int64]); ok { return s.DataPoints } @@ -123,7 +126,7 @@ func findBackupInt64SumValueByOutcome( ) (int64, bool) { var total int64 var found bool - for _, dp := range findInt64SumDataPoints(rm, opentelemetry.PluginBackupRunsMetric) { + for _, dp := range findInt64SumDataPoints(rm) { v, ok := dp.Attributes.Value("outcome") if !ok { continue @@ -143,7 +146,7 @@ func findBackupInt64SumValueByOutcome( func findBackupInt64SumValueByFailureCategory( rm metricdata.ResourceMetrics, category backupfailure.Category, ) (int64, bool) { - for _, dp := range findInt64SumDataPoints(rm, opentelemetry.PluginBackupRunsMetric) { + for _, dp := range findInt64SumDataPoints(rm) { v, ok := dp.Attributes.Value("failure_category") if !ok { continue @@ -197,7 +200,7 @@ func TestRecordBackupStart(t *testing.T) { reader := setupTestMeter(t) before := time.Now().Unix() - recordBackupStart(context.Background()) + recordBackupStart(context.Background(), testClusterName) rm := collectOTelMetrics(t, reader) @@ -213,11 +216,11 @@ func TestRecordBackupStart(t *testing.T) { func TestRecordBackupSuccess(t *testing.T) { reader := setupTestMeter(t) - recordBackupStart(context.Background()) + recordBackupStart(context.Background(), testClusterName) duration := 42 * time.Second before := time.Now().Unix() - recordBackupSuccess(context.Background(), duration) - recordBackupFinished(context.Background()) + recordBackupSuccess(context.Background(), testClusterName, duration) + recordBackupFinished(context.Background(), testClusterName) rm := collectOTelMetrics(t, reader) @@ -252,15 +255,15 @@ func TestRecordBackupSuccess(t *testing.T) { func TestRecordBackupFailure(t *testing.T) { reader := setupTestMeter(t) - recordBackupStart(t.Context()) + recordBackupStart(t.Context(), testClusterName) before := time.Now().Unix() //nolint:gosec // hardcoded test input exitErr := exec.CommandContext(t.Context(), "sh", "-c", fmt.Sprintf("exit %d", backupfailure.RepositoryError.ExitCode)).Run() - recordBackupFailure(t.Context(), 7*time.Second, exitErr) - recordBackupFinished(t.Context()) + recordBackupFailure(t.Context(), testClusterName, 7*time.Second, exitErr) + recordBackupFinished(t.Context(), testClusterName) rm := collectOTelMetrics(t, reader) @@ -294,16 +297,16 @@ func TestRecordBackupFailureCategoriesAreSeparateSeries(t *testing.T) { expiredCtx, cancelTimeout := context.WithTimeout(t.Context(), 0) defer cancelTimeout() - recordBackupFailure(expiredCtx, time.Second, nil) + recordBackupFailure(expiredCtx, testClusterName, time.Second, nil) canceledCtx, cancelCanceled := context.WithCancel(t.Context()) cancelCanceled() - recordBackupFailure(canceledCtx, time.Second, nil) - recordBackupFailure(canceledCtx, time.Second, nil) + recordBackupFailure(canceledCtx, testClusterName, time.Second, nil) + recordBackupFailure(canceledCtx, testClusterName, time.Second, nil) //nolint:gosec // hardcoded test input exitErr := exec.CommandContext(t.Context(), "sh", "-c", fmt.Sprintf("exit %d", backupfailure.Verification.ExitCode)).Run() - recordBackupFailure(t.Context(), time.Second, exitErr) + recordBackupFailure(t.Context(), testClusterName, time.Second, exitErr) rm := collectOTelMetrics(t, reader) @@ -327,7 +330,7 @@ func TestRecordBackupFailureCategoriesAreSeparateSeries(t *testing.T) { func TestRecordBackupFailureNilErrorDefaultsToUnknown(t *testing.T) { reader := setupTestMeter(t) - recordBackupFailure(context.Background(), time.Second, nil) + recordBackupFailure(context.Background(), testClusterName, time.Second, nil) rm := collectOTelMetrics(t, reader) @@ -337,23 +340,145 @@ func TestRecordBackupFailureNilErrorDefaultsToUnknown(t *testing.T) { assert.Equal(t, int64(1), unknown) } +// metricAttributeSets returns the attribute set of every data point on a +// metric, across the aggregation types the plugin backup instruments use +// (Int64/Float64 gauges, the Int64 up/down counter and runs counter, and the +// Float64 duration histogram). +func metricAttributeSets(a metricdata.Aggregation) []attribute.Set { + var sets []attribute.Set + switch d := a.(type) { + case metricdata.Gauge[int64]: + for _, dp := range d.DataPoints { + sets = append(sets, dp.Attributes) + } + case metricdata.Gauge[float64]: + for _, dp := range d.DataPoints { + sets = append(sets, dp.Attributes) + } + case metricdata.Sum[int64]: + for _, dp := range d.DataPoints { + sets = append(sets, dp.Attributes) + } + case metricdata.Histogram[float64]: + for _, dp := range d.DataPoints { + sets = append(sets, dp.Attributes) + } + } + + return sets +} + +// dataPointClusterNames collects the distinct `cluster_name` attribute values +// seen across every plugin backup instrument (the empty string keys the data +// points that carry no cluster_name), so a test can assert every series is +// attributed to a cluster. +func dataPointClusterNames(rm metricdata.ResourceMetrics) map[string]int { + names := map[string]int{} + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + for _, set := range metricAttributeSets(m.Data) { + if v, ok := set.Value("cluster_name"); ok { + names[v.AsString()]++ + } else { + names[""]++ + } + } + } + } + + return names +} + +func TestBackupMetricsCarryClusterName(t *testing.T) { + reader := setupTestMeter(t) + + recordBackupStart(context.Background(), testClusterName) + recordBackupSuccess(context.Background(), testClusterName, 5*time.Second) + recordBackupFinished(context.Background(), testClusterName) + recordBackupFailure(context.Background(), testClusterName, time.Second, assert.AnError) + + rm := collectOTelMetrics(t, reader) + + names := dataPointClusterNames(rm) + require.NotEmpty(t, names) + assert.Equal(t, 0, names[""], "every plugin backup data point must carry a cluster_name attribute") + assert.Contains(t, names, testClusterName) +} + +func TestBackupMetricsClustersAreSeparateSeries(t *testing.T) { + reader := setupTestMeter(t) + + recordBackupStart(context.Background(), "cluster-a") + recordBackupSuccess(context.Background(), "cluster-a", 5*time.Second) + recordBackupFinished(context.Background(), "cluster-a") + + recordBackupStart(context.Background(), "cluster-b") + recordBackupSuccess(context.Background(), "cluster-b", 9*time.Second) + recordBackupFinished(context.Background(), "cluster-b") + + rm := collectOTelMetrics(t, reader) + + // The runs counter must expose one data point per (cluster_name, outcome), + // not a single folded series. + perCluster := map[string]int64{} + for _, dp := range findInt64SumDataPoints(rm) { + v, ok := dp.Attributes.Value("cluster_name") + require.True(t, ok, "runs data point missing cluster_name") + perCluster[v.AsString()] += dp.Value + } + assert.Equal(t, int64(1), perCluster["cluster-a"]) + assert.Equal(t, int64(1), perCluster["cluster-b"]) +} + +func TestInitPluginBackupSeriesSeedsZero(t *testing.T) { + reader := setupTestMeter(t) + + opentelemetry.InitPluginBackupSeries(context.Background(), testClusterName) + + rm := collectOTelMetrics(t, reader) + + inProgress, ok := findInProgressValue(rm) + require.True(t, ok) + assert.Equal(t, int64(0), inProgress, "in-progress must be seeded at 0") + + dps := findInt64SumDataPoints(rm) + require.NotEmpty(t, dps) + + sawSuccess := false + seededCategories := map[string]bool{} + for _, dp := range dps { + assert.Equal(t, int64(0), dp.Value, "seeded runs series must start at 0") + if v, ok := dp.Attributes.Value("outcome"); ok && + v.AsString() == string(opentelemetry.OutcomeSuccess) { + sawSuccess = true + } + if v, ok := dp.Attributes.Value("failure_category"); ok { + seededCategories[v.AsString()] = true + } + } + assert.True(t, sawSuccess, "the success runs series must be seeded") + for _, name := range backupfailure.Names() { + assert.True(t, seededCategories[name], "failure category %q must be seeded", name) + } +} + func TestBackupMetricsMultipleRuns(t *testing.T) { reader := setupTestMeter(t) // First backup: success. - recordBackupStart(context.Background()) - recordBackupSuccess(context.Background(), 10*time.Second) - recordBackupFinished(context.Background()) + recordBackupStart(context.Background(), testClusterName) + recordBackupSuccess(context.Background(), testClusterName, 10*time.Second) + recordBackupFinished(context.Background(), testClusterName) // Second backup: failure. - recordBackupStart(context.Background()) - recordBackupFailure(context.Background(), 3*time.Second, assert.AnError) - recordBackupFinished(context.Background()) + recordBackupStart(context.Background(), testClusterName) + recordBackupFailure(context.Background(), testClusterName, 3*time.Second, assert.AnError) + recordBackupFinished(context.Background(), testClusterName) // Third backup: success. - recordBackupStart(context.Background()) - recordBackupSuccess(context.Background(), 30*time.Second) - recordBackupFinished(context.Background()) + recordBackupStart(context.Background(), testClusterName) + recordBackupSuccess(context.Background(), testClusterName, 30*time.Second) + recordBackupFinished(context.Background(), testClusterName) rm := collectOTelMetrics(t, reader) diff --git a/core/internal/consumer/backup.go b/core/internal/consumer/backup.go index 9ba0266d3..41170222b 100644 --- a/core/internal/consumer/backup.go +++ b/core/internal/consumer/backup.go @@ -441,7 +441,7 @@ func (d *Backup) verifyTier1(ctx context.Context, clusterName string) error { }) if err != nil { if _, ok := errors.AsType[*klioclientkopia.BackupVerificationError](err); ok { - recordVerificationFailure(ctx, opentelemetry.Tier1) + recordVerificationFailure(ctx, clusterName, opentelemetry.Tier1) return fmt.Errorf("tier1 verification detected corruption: %w", err) } @@ -450,7 +450,7 @@ func (d *Backup) verifyTier1(ctx context.Context, clusterName string) error { return err } - recordVerificationSuccess(ctx, opentelemetry.Tier1) + recordVerificationSuccess(ctx, clusterName, opentelemetry.Tier1) return nil } @@ -465,13 +465,13 @@ func (d *Backup) verifyTier2Backups(ctx context.Context, clusterName string) err }) if err != nil { if _, ok := errors.AsType[*klioclientkopia.BackupVerificationError](err); ok { - recordVerificationFailure(ctx, opentelemetry.Tier2) + recordVerificationFailure(ctx, clusterName, opentelemetry.Tier2) return fmt.Errorf("tier2 verification detected corruption: %w", err) } contextLogger.Error(err, "Tier2 verification encountered infrastructure error, continuing") } else { - recordVerificationSuccess(ctx, opentelemetry.Tier2) + recordVerificationSuccess(ctx, clusterName, opentelemetry.Tier2) } return nil diff --git a/core/internal/consumer/metrics.go b/core/internal/consumer/metrics.go index 13bf66a08..338f9677e 100644 --- a/core/internal/consumer/metrics.go +++ b/core/internal/consumer/metrics.go @@ -60,17 +60,19 @@ func outcomeOf(err error) opentelemetry.Outcome { return opentelemetry.OutcomeSuccess } -func recordVerificationSuccess(ctx context.Context, tier opentelemetry.Tier) { +func recordVerificationSuccess(ctx context.Context, clusterName string, tier opentelemetry.Tier) { opentelemetry.ServerBackup.Verifications.Add(ctx, 1, metric.WithAttributes( + opentelemetry.AttributeKeyClusterName.Of(clusterName), tier.Attribute(), opentelemetry.OutcomeSuccess.Attribute(), )) } -func recordVerificationFailure(ctx context.Context, tier opentelemetry.Tier) { +func recordVerificationFailure(ctx context.Context, clusterName string, tier opentelemetry.Tier) { opentelemetry.ServerBackup.Verifications.Add(ctx, 1, metric.WithAttributes( + opentelemetry.AttributeKeyClusterName.Of(clusterName), tier.Attribute(), opentelemetry.OutcomeFailure.Attribute(), )) diff --git a/core/internal/opentelemetry/catalog.go b/core/internal/opentelemetry/catalog.go index eb1eb66ab..f7ea0f46c 100644 --- a/core/internal/opentelemetry/catalog.go +++ b/core/internal/opentelemetry/catalog.go @@ -137,7 +137,10 @@ const ( ) // PluginBackupMetrics holds OTel instruments for backup lifecycle tracking. -// The Runs counter carries an `outcome` attribute (`success` / `failure`) +// Every instrument carries a `cluster_name` attribute identifying the +// PostgreSQL cluster the plugin sidecar serves, so panels can attribute backup +// activity per cluster even when several clusters share a namespace. The Runs +// counter additionally carries an `outcome` attribute (`success` / `failure`) // so a single instrument exposes both flavors. Runs failure data points // additionally carry a `failure_category` attribute classifying the failure // (see opentelemetry.FailureCategory); verification failures are recorded @@ -158,15 +161,17 @@ type PluginBackupMetrics struct { // - The Verifications counter, paired with `klio.plugin.backup.*` from the // plugin sidecar: the plugin records backup lifecycle, the server records // the verifications it runs against those backups. Each recording carries -// a `tier` attribute that distinguishes tier-1 verification (post-backup -// local check) from tier-2 verification (post-upload remote check), and -// an `outcome` attribute (`success` / `failure`) so one instrument -// exposes both flavors. +// a `cluster_name` attribute, a `tier` attribute that distinguishes tier-1 +// verification (post-backup local check) from tier-2 verification +// (post-upload remote check), and an `outcome` attribute (`success` / +// `failure`) so one instrument exposes both flavors. // - Base snapshot gauges populated from Kopia, describing the current set // of base backups stored on the server. Each recording carries a // `tier` attribute (tier-1 for local disk, tier-2 for remote object // store) and a `snapshot_source` attribute identifying the Kopia -// source descriptor (`userName@hostName:path`). +// source descriptor, formatted `userName@hostName:path`. Its `hostName` +// component is the PostgreSQL cluster name, which the Grafana dashboard +// relies on to attribute snapshots per cluster. // - PostgreSQL backup gauges populated from the snapshotted backup // metadata, describing the retention window of physical backups per // cluster. Each recording carries a `tier` attribute and a @@ -254,39 +259,63 @@ func InitPluginBackupMetrics() { meter := otel.Meter(Meter) PluginBackup.LatestStartTime, _ = meter.Int64Gauge(PluginBackupLatestStartTimeMetric, - metric.WithDescription("Unix epoch timestamp when the most recent backup started."), + metric.WithDescription("Unix epoch timestamp when the most recent backup started, "+ + "broken down by `cluster_name`."), metric.WithUnit("s"), ) PluginBackup.LatestCompletionTime, _ = meter.Int64Gauge(PluginBackupLatestCompletionTimeMetric, - metric.WithDescription("Unix epoch timestamp when the most recent backup completed successfully."), + metric.WithDescription("Unix epoch timestamp when the most recent backup completed "+ + "successfully, broken down by `cluster_name`."), metric.WithUnit("s"), ) PluginBackup.LatestFailureTime, _ = meter.Int64Gauge(PluginBackupLatestFailureTimeMetric, - metric.WithDescription("Unix epoch timestamp when the most recent backup failed."), + metric.WithDescription("Unix epoch timestamp when the most recent backup failed, "+ + "broken down by `cluster_name`."), metric.WithUnit("s"), ) PluginBackup.LatestDuration, _ = meter.Float64Gauge(PluginBackupLatestDurationMetric, - metric.WithDescription("Duration of the most recent backup."), + metric.WithDescription("Duration of the most recent backup, broken down by `cluster_name`."), metric.WithUnit("s"), ) PluginBackup.Duration, _ = meter.Float64Histogram(PluginBackupDurationMetric, - metric.WithDescription("Distribution of backup durations, split by the `outcome` "+ - "attribute (`success` / `failure`)."), + metric.WithDescription("Distribution of backup durations, broken down by `cluster_name` "+ + "and split by the `outcome` attribute (`success` / `failure`)."), metric.WithUnit("s"), ) PluginBackup.InProgress, _ = meter.Int64UpDownCounter(PluginBackupInProgressMetric, - metric.WithDescription("Number of backups currently in progress."), + metric.WithDescription("Number of backups currently in progress, broken down by `cluster_name`."), metric.WithUnit("{backups}"), ) PluginBackup.Runs, _ = meter.Int64Counter(PluginBackupRunsMetric, - metric.WithDescription("Total number of backup runs, split by the `outcome` "+ - "attribute (`success` / `failure`). Failure data points additionally "+ - "carry a `failure_category` attribute (`"+ + metric.WithDescription("Total number of backup runs, broken down by `cluster_name` and "+ + "split by the `outcome` attribute (`success` / `failure`). Failure data points "+ + "additionally carry a `failure_category` attribute (`"+ strings.Join(backupfailure.Names(), "`, `")+"`)."), metric.WithUnit("{backups}"), ) } +// InitPluginBackupSeries seeds the plugin backup runs counter at 0 for every +// outcome and failure category of clusterName (and the in-progress gauge), so +// the first backup is a visible 0->1 that rate()/increase() can measure. See +// https://prometheus.io/docs/practices/instrumentation/#avoid-missing-metrics +func InitPluginBackupSeries(ctx context.Context, clusterName string) { + if clusterName == "" { + return + } + cluster := AttributeKeyClusterName.Of(clusterName) + + PluginBackup.InProgress.Add(ctx, 0, metric.WithAttributes(cluster)) + PluginBackup.Runs.Add(ctx, 0, metric.WithAttributes(cluster, OutcomeSuccess.Attribute())) + for _, category := range backupfailure.Names() { + PluginBackup.Runs.Add(ctx, 0, metric.WithAttributes( + cluster, + OutcomeFailure.Attribute(), + AttributeKeyFailureCategory.Of(category), + )) + } +} + // InitServerBackupMetrics creates OTel instruments for server-side backup // state: verification counters and Kopia base snapshot gauges. It is called // once automatically when this package is loaded; tests can call it again diff --git a/documentation/web/docs/user/grafana-dashboards.md b/documentation/web/docs/user/grafana-dashboards.md index 4010b9a49..acdd67130 100644 --- a/documentation/web/docs/user/grafana-dashboards.md +++ b/documentation/web/docs/user/grafana-dashboards.md @@ -21,19 +21,20 @@ The dashboard is a single dashboard split into row sections: - **Client / Plugin** — the backup lifecycle as seen by the plugin sidecar running in each PostgreSQL pod: backups in progress, time since the last backup started, succeeded and failed, the latest backup duration, the - p50/p95/p99 backup duration distribution, backup run and failure rates, + p50/p90/p99 backup duration distribution, backup run and failure rates, and the backup success ratio. Also the WAL streaming client the sidecar supervises as a child process: the PostgreSQL - timeline it is currently streaming and the p50/p95/p99 latency of sending + timeline it is currently streaming and the p50/p90/p99 latency of sending a WAL block to the server. ![Klio client and plugin metrics](images/klio_client_and_plugin_metrics.png) - **Server** — the state of the Klio server StatefulSet: uptime, WAL ingest throughput and freshness per tier, the latest written LSN, backup - verification outcomes, the base snapshot inventory (including file and - directory counts), p50/p95/p99 WAL block/get/upload duration by path, - stage and tier, tier-2 relay and maintenance run rates, the retention + verification totals per cluster, the base snapshot inventory (including + file and directory counts), p50/p90/p99 WAL block/get/upload duration by + path, stage and tier (each as both a rolling-window and a since-restart + total variant), tier-2 relay and maintenance run totals, the retention window of the physical PostgreSQL backups (counts by tier, latest/oldest backup age, start/end LSN and PostgreSQL timeline per cluster), and the embedded NATS @@ -41,43 +42,29 @@ The dashboard is a single dashboard split into row sections: ![Klio server metrics](images/klio_server_metrics.png) -- **WAL Replication Lag** — how far Klio's WAL streaming client trails the - PostgreSQL primary, using CloudNativePG's replication metrics: the replay - lag in bytes and the flush lag in seconds. These panels read the - `cnpg_pg_stat_replication_*` metrics, so they require CloudNativePG - monitoring to be scraped into the same Prometheus (see the prerequisites - below). +- **WAL replication lag** — how far behind Klio is in copying WAL: + - **Tier-1 replication lag (bytes)** and **(seconds)**: how much WAL, + and how much time, Klio is behind the PostgreSQL primary. The *write* + line is WAL Klio has received; the *flush* line is WAL Klio has + safely saved to disk. These two panels need CloudNativePG monitoring + (see the prerequisites below). + - **Tier-2 archival lag (bytes)**: how much WAL is on Klio's local disk + (tier 1) but not yet copied to remote storage (tier 2). ![Klio WAL replication lag metrics](images/klio_wal_replication_lag_metrics.png) Some panels need extra context to interpret correctly. Two are derived from the alerting guidance in [OpenTelemetry](opentelemetry.md): -- **Time since last WAL written by tier** surfaces the staleness signal +- **Time since last WAL written** surfaces the staleness signal described under *Alerting on stalled WAL processing*: a stale tier-1 value means PostgreSQL is no longer shipping WALs, while a stale tier-2 value means the remote backend is no longer receiving them. -- **Tier-2 archival backlog (LSN gap)** plots the LSN difference between +- **Tier-2 archival lag (bytes)** plots the LSN difference between tier 1 (local disk) and tier 2 (remote storage). Read together with the staleness panel, it tells a slow pipeline (timestamps advancing, gap growing) apart from a stalled one (timestamps and LSN both frozen). -Two more are a statistical caveat rather than an alerting signal. Both are -histogram percentiles that need enough recent samples to be reliable: - -- **WAL block send duration (p50/p95/p99) by cluster** is most meaningful - under active write load. On an idle or low-write cluster, WAL blocks are - sent too infrequently for the underlying `histogram_quantile` to produce - a reliable percentile, so the line can look sparse or noisy rather than - simply absent. -- **Backup duration (p50/p95/p99)** has the same limitation, more acutely: - backups are infrequent, so this panel is computed over the whole selected - range (rather than a short rate window) to stay populated between runs. - Widen the dashboard range to span several backups for a stable reading; if - the selected range contains no backup, the panel is empty. Use it to spot - backup runtime trending up over time rather than to read an instantaneous - value. - ## Prerequisites The dashboard reads the Prometheus names of Klio's metrics (for example @@ -90,11 +77,16 @@ described in [OpenTelemetry](opentelemetry.md) works: - The Klio Prometheus exporter (`OTEL_METRICS_EXPORTER=prometheus`), scraped directly. -The **WAL Replication Lag** row additionally reads CloudNativePG's +The **WAL replication lag** row additionally reads CloudNativePG's `cnpg_pg_stat_replication_*` metrics. To populate it, scrape the CloudNativePG cluster monitoring (its `PodMonitor`) into the same Prometheus. The rest of the dashboard works without it. +The LSN table panels (**Latest backup LSN**, **Oldest backup LSN**, +**Latest written LSN**) need Grafana 13 or later: they use SQL +Expressions, available starting in that version. Without it, these +three panels show an error instead of a table. + :::note When you route metrics through an OpenTelemetry Collector, enable `resource_to_telemetry_conversion` on the Prometheus exporter so that @@ -113,8 +105,33 @@ this. The dashboard declares a `datasource` template variable, so it is portable across Grafana installations and is not tied to a specific data source UID. -The `namespace` and `cluster` template variables at the top filter the panels -by Kubernetes namespace and PostgreSQL cluster. +Three more template variables at the top filter the panels: + +- `namespace`: the Kubernetes namespace, matched against + `k8s.namespace.name`. It scopes the Client / Plugin and WAL Replication + Lag panels, whose metrics are emitted from the PostgreSQL pods and + therefore carry the *cluster's* namespace. +- `server`: the Klio server, matched against the OpenTelemetry + `service.name` (not the pod host name, which two servers of the same name + in different namespaces would share). It scopes the Server panels. +- `cluster`: the PostgreSQL cluster, matched against `cluster_name`. It + scopes every per-cluster panel across all sections. Because the + server-side metrics carry the server's own namespace, per-cluster Server + panels are filtered by `server` and `cluster` rather than `namespace`, so a + cluster backed up by a server in another namespace is still attributed + correctly. + +Every aggregation groups by the identifying label (cluster, server, tier), +so multiple clusters or servers are never folded into a single misleading +value. + +:::note +Give each Klio server a distinct `OTEL_SERVICE_NAME` (as the sample server +manifests do). The `server` variable identifies servers by their OpenTelemetry +`service.name`; if it is left unset, every server reports the SDK default +(`unknown_service:klio`) and they collapse into a single, indistinguishable +entry. +::: ### Example: kube-prometheus-stack diff --git a/documentation/web/docs/user/images/klio_client_and_plugin_metrics.png b/documentation/web/docs/user/images/klio_client_and_plugin_metrics.png index 038e2a3cd..5b54b9d0d 100644 Binary files a/documentation/web/docs/user/images/klio_client_and_plugin_metrics.png and b/documentation/web/docs/user/images/klio_client_and_plugin_metrics.png differ diff --git a/documentation/web/docs/user/images/klio_server_metrics.png b/documentation/web/docs/user/images/klio_server_metrics.png index 0357dbad6..fd8084780 100644 Binary files a/documentation/web/docs/user/images/klio_server_metrics.png and b/documentation/web/docs/user/images/klio_server_metrics.png differ diff --git a/documentation/web/docs/user/images/klio_wal_replication_lag_metrics.png b/documentation/web/docs/user/images/klio_wal_replication_lag_metrics.png index 7cd48860c..137651ef1 100644 Binary files a/documentation/web/docs/user/images/klio_wal_replication_lag_metrics.png and b/documentation/web/docs/user/images/klio_wal_replication_lag_metrics.png differ diff --git a/documentation/web/docs/user/opentelemetry.md b/documentation/web/docs/user/opentelemetry.md index 3c52aa8cd..b7a89af61 100644 --- a/documentation/web/docs/user/opentelemetry.md +++ b/documentation/web/docs/user/opentelemetry.md @@ -161,7 +161,7 @@ attribute key. | Attribute | Values | Applies to | |---|---|---| | `tier` | `tier1` (local disk on the Klio server), `tier2` (remote object store) | All `klio.server.wal.*` and `klio.server.backup.*` instruments. | -| `cluster_name` | Name of the PostgreSQL cluster the recording belongs to | All `klio.server.wal.*` instruments (counters, gauges, and the WAL duration histograms), `klio.client.wal.*`, the `klio.server.backup.*` PostgreSQL backup gauges (`backups`, `latest_backup_*`, `oldest_backup_*`) and the `klio.server.backup.relay` / `klio.server.backup.maintenance` counters. | +| `cluster_name` | Name of the PostgreSQL cluster the recording belongs to | All `klio.plugin.backup.*` instruments, all `klio.server.wal.*` instruments (counters, gauges, and the WAL duration histograms), `klio.client.wal.*`, the `klio.server.backup.*` PostgreSQL backup gauges (`backups`, `latest_backup_*`, `oldest_backup_*`) and the `klio.server.backup.relay` / `klio.server.backup.maintenance` counters. | | `outcome` | `success`, `failure` | `klio.plugin.backup.runs`, `klio.server.backup.relay`, `klio.server.backup.maintenance`, `klio.server.backup.verifications`, and all WAL duration histograms (`klio.server.wal.*_duration`, `klio.client.wal.block_duration`). | | `failure_category` | `repository_error`, `source_error`, `verification`, `timeout`, `canceled`, `unknown` | `klio.plugin.backup.runs` failure data points only. | | `path` | `put` (WAL ingest), `get` (WAL serve) | `klio.server.wal.block_duration`, `klio.client.wal.block_duration`. | @@ -172,17 +172,20 @@ attribute key. ### Backup lifecycle metrics (plugin sidecar) These metrics are emitted by the plugin sidecar and track backup -operations on each PostgreSQL instance: +operations on each PostgreSQL instance. Every instrument carries a +`cluster_name` attribute identifying the PostgreSQL cluster the sidecar +serves, so backup activity can be attributed per cluster even when several +clusters share a namespace: | Metric Name | Type | Unit | Description | |---|---|---|---| -| `klio.plugin.backup.in_progress` | UpDownCounter | `{backups}` | Number of backups currently in progress | -| `klio.plugin.backup.latest_start_time` | Gauge | s | Unix epoch timestamp when the most recent backup started | -| `klio.plugin.backup.latest_completion_time` | Gauge | s | Unix epoch timestamp when the most recent backup completed successfully | -| `klio.plugin.backup.latest_failure_time` | Gauge | s | Unix epoch timestamp when the most recent backup failed | -| `klio.plugin.backup.latest_duration` | Gauge | s | Duration of the most recent backup | -| `klio.plugin.backup.duration` | Histogram | s | Distribution of backup durations, split by the `outcome` attribute (`success` / `failure`) | -| `klio.plugin.backup.runs` | Counter | `{backups}` | Total number of backup runs, split by the `outcome` attribute (`success` / `failure`). Failure data points additionally carry a `failure_category` attribute classifying the failure. Backup verification is part of a run: a verification failure is recorded here with `failure_category="verification"`, and a clean verification is included in the `outcome="success"` count | +| `klio.plugin.backup.in_progress` | UpDownCounter | `{backups}` | Number of backups currently in progress, broken down by `cluster_name` | +| `klio.plugin.backup.latest_start_time` | Gauge | s | Unix epoch timestamp when the most recent backup started, broken down by `cluster_name` | +| `klio.plugin.backup.latest_completion_time` | Gauge | s | Unix epoch timestamp when the most recent backup completed successfully, broken down by `cluster_name` | +| `klio.plugin.backup.latest_failure_time` | Gauge | s | Unix epoch timestamp when the most recent backup failed, broken down by `cluster_name` | +| `klio.plugin.backup.latest_duration` | Gauge | s | Duration of the most recent backup, broken down by `cluster_name` | +| `klio.plugin.backup.duration` | Histogram | s | Distribution of backup durations, broken down by `cluster_name` and split by the `outcome` attribute (`success` / `failure`) | +| `klio.plugin.backup.runs` | Counter | `{backups}` | Total number of backup runs, broken down by `cluster_name` and split by the `outcome` attribute (`success` / `failure`). Failure data points additionally carry a `failure_category` attribute classifying the failure. Backup verification is part of a run: a verification failure is recorded here with `failure_category="verification"`, and a clean verification is included in the `outcome="success"` count | The `failure_category` attribute on `klio.plugin.backup.runs` failure data points takes one of the following values: diff --git a/observability/grafana/README.md b/observability/grafana/README.md index 6263232c2..09ad9639f 100644 --- a/observability/grafana/README.md +++ b/observability/grafana/README.md @@ -16,7 +16,7 @@ metrics. It is split into row sections: window of physical PostgreSQL backups (start/end time, LSN and timeline per cluster and tier), and queue metrics emitted by the Klio server StatefulSet (`klio_server_*`). Built in [`server.go`](server.go). -- **WAL Replication Lag** — how far Klio's WAL streaming client trails the +- **WAL replication lag** — how far Klio's WAL streaming client trails the PostgreSQL primary, from CloudNativePG's `cnpg_pg_stat_replication_*` metrics (requires CloudNativePG monitoring scraped into the same Prometheus). Built in [`replication.go`](replication.go). diff --git a/observability/grafana/build.go b/observability/grafana/build.go index 8eed8ce2c..b9a52044f 100644 --- a/observability/grafana/build.go +++ b/observability/grafana/build.go @@ -56,27 +56,36 @@ func build() *dashboard.DashboardBuilder { labelVariable("namespace", "Namespace", `label_values({__name__=~"klio_.+"}, k8s_namespace_name)`), ). + // Server identity is the OpenTelemetry service.name, not the pod + // host_name: two Servers with the same name in different namespaces + // share a host_name (e.g. both "klio-a-klio-0") and would collapse into + // one ambiguous entry, whereas their service.name stays distinct. WithVariable( labelVariable("server", "Server", - `label_values(klio_server_uptime_seconds{k8s_namespace_name=~"$namespace"}, host_name)`), + `label_values(klio_server_uptime_seconds, service_name)`), ). + // The cluster list is narrowed by the selected server (service.name), + // NOT by namespace: a server tags its series with its own namespace, so + // filtering clusters by $namespace would drop clusters served + // cross-namespace. WithVariable( labelVariable("cluster", "Cluster", - `label_values(klio_server_wal_written_total{k8s_namespace_name=~"$namespace",host_name=~"$server"}, cluster_name)`), + `label_values(klio_server_wal_written_total{service_name=~"$server"}, cluster_name)`), ) y := 0 layoutSection(builder, &y, "Client / Plugin", clientPanels()) layoutSection(builder, &y, "Server", serverPanels()) - layoutSection(builder, &y, "WAL Replication Lag", replicationPanels()) + layoutSection(builder, &y, "WAL replication lag", replicationPanels()) return builder } // layoutSection adds a row header at the current y offset, then packs the -// panels into a dense grid: each grid row is filled to the full 24-column width -// (slack is distributed across the row's panels) so there are no empty gaps -// between panels. y is advanced past the section. +// panels into a dense grid: each grid row is filled to the full 24-column +// width (slack is distributed across the row's panels) so there are no empty +// gaps between panels. Widths that add up to 24 are used as they are. y is +// advanced past the section. func layoutSection(builder *dashboard.DashboardBuilder, y *int, title string, panels []sizedPanel) { builder.WithRow(dashboard.NewRowBuilder(title). GridPos(dashboard.GridPos{X: 0, Y: uint32(*y), W: 24, H: 1})) //nolint:gosec // small positive ints diff --git a/observability/grafana/client.go b/observability/grafana/client.go index 4dcda25f6..ddf5087b1 100644 --- a/observability/grafana/client.go +++ b/observability/grafana/client.go @@ -21,115 +21,133 @@ package main import ( "fmt" -) -// clientWalMatcher selects the WAL streaming client's per-cluster series -// (klio_client_wal_*). The streaming client runs as a child process of the -// plugin sidecar (spawned via `klio send-wal`, same container, different -// PID), but unlike klio_plugin_backup_* it carries cluster_name and no -// host_name, so these panels are scoped by $namespace and $cluster. -const clientWalMatcher = `k8s_namespace_name=~"$namespace",cluster_name=~"$cluster"` + "github.com/grafana/grafana-foundation-sdk/go/common" + "github.com/grafana/grafana-foundation-sdk/go/units" +) // clientPanels returns the "Client / Plugin" section panels. These metrics are // emitted by the Klio plugin sidecar that runs in each PostgreSQL pod: the // backup lifecycle (`klio.plugin.backup.*`, exported to Prometheus as // `klio_plugin_backup_*`) and the WAL streaming client it supervises as a -// child process (`klio.client.wal.*`, exported as `klio_client_wal_*`). -// Backup queries are scoped by $namespace; WAL streaming queries additionally -// carry cluster_name and are scoped by $cluster. +// child process (`klio.client.wal.*`, exported as `klio_client_wal_*`). Both +// families carry a cluster_name, so every panel groups by cluster_name and is +// scoped by $namespace and $cluster; nothing folds several clusters that share +// a namespace into a single value. func clientPanels() []sizedPanel { return []sizedPanel{ - // Current backup state. - sized(4, panelHeight, statPanel("Backups in progress", "none", - query(fmt.Sprintf("sum(klio_plugin_backup_in_progress{%s})", nsMatcher), "in progress"), + sized(gridWidth, descriptionPanelHeight, descriptionPanel( + "Backup lifecycle and WAL streaming as seen by the plugin sidecar running in each PostgreSQL pod.")), + // Backup activity. + // Current backup state, grouped by cluster so a namespace hosting + // several clusters shows one series each instead of a folded total. + // Orientation forced to horizontal: with several clusters selected, + // stat panels laid out "auto" stack tiles vertically and clip inside + // the fixed panel height, hiding clusters below the fold. + sized(mediumPanelWidth, mediumPanelHeight, statPanel("Backups in progress", units.Number, + query(fmt.Sprintf("sum by (cluster_name) (klio_plugin_backup_in_progress{%s})", clientMatcher), + "{{cluster_name}}"), ).Decimals(0). - Description("Base backups currently running across the plugin sidecars in the namespace.")), - sized(4, panelHeight, statPanel("Time since last successful backup", "dtdurations", - query(fmt.Sprintf("time() - max(klio_plugin_backup_latest_completion_time_seconds{%s})", nsMatcher), - "since success"), - ).Description("Elapsed time since the most recent base backup completed successfully. A value well "+ - "above the backup interval means backups have stopped succeeding.")), - sized(4, panelHeight, statPanel("Time since last failed backup", "dtdurations", - query(fmt.Sprintf("time() - max(klio_plugin_backup_latest_failure_time_seconds{%s})", nsMatcher), - "since failure"), - ).Description("Elapsed time since the most recent base backup failure. A small value means a "+ - "failure happened recently.")), - sized(4, panelHeight, statPanel("Latest backup duration", "dtdurations", - query(fmt.Sprintf("max(klio_plugin_backup_latest_duration_seconds{%s})", nsMatcher), "latest duration"), - ).Description("Wall-clock duration of the most recent base backup.")), - sized(4, panelHeight, statPanel("Time since last backup started", "dtdurations", - query(fmt.Sprintf("time() - max(klio_plugin_backup_latest_start_time_seconds{%s})", nsMatcher), - "since start"), - ).Description("Elapsed time since the most recent base backup started. Compare against the latest "+ - "duration to tell whether a backup is still running or overdue.")), - // Derived: share of backup runs that succeeded over the selected range. - sized(4, panelHeight, statPanel("Backup success ratio", "percentunit", - query( - fmt.Sprintf("sum(increase(klio_plugin_backup_runs_total{outcome=\"success\",%s}[$__range])) / "+ - "clamp_min(sum(increase(klio_plugin_backup_runs_total{%s}[$__range])), 1)", nsMatcher, nsMatcher), - "success ratio", - ), - ).Description("Fraction of base backup runs that succeeded over the selected time range "+ - "(successful runs / total runs).")), - // Derived: count of successful backups over fixed trailing windows. The - // runs counter resets when the plugin sidecar restarts, so increase() - // over long windows is an approximation across restarts. - sized(4, panelHeight, statPanel("Successful backups (24h)", "none", - query(fmt.Sprintf("sum(increase(klio_plugin_backup_runs_total{outcome=\"success\",%s}[24h]))", nsMatcher), - "last 24h"), + Orientation(common.VizOrientationHorizontal). + Description("Base backups currently running, per cluster.")), + + sized(mediumPanelWidth, mediumPanelHeight, statPanel("Successful backup runs", units.Number, + query(fmt.Sprintf("sum by (cluster_name) (increase(klio_plugin_backup_runs_total{outcome=\"success\",%s}"+ + "[$__range]))", clientMatcher), "{{cluster_name}}"), ).Decimals(0). - Description("Base backups that completed successfully in the last 24 hours. Counter resets on "+ - "plugin restart make long-window counts approximate.")), - sized(4, panelHeight, statPanel("Successful backups (7d)", "none", - query(fmt.Sprintf("sum(increase(klio_plugin_backup_runs_total{outcome=\"success\",%s}[7d]))", nsMatcher), - "last 7d"), + Orientation(common.VizOrientationHorizontal). + Description("Base backups completed successfully by the running clients sidecars, per cluster, "+ + "over the selected time range.")), + + sized(mediumPanelWidth, mediumPanelHeight, statPanel("Failed backup runs", units.Number, + query(fmt.Sprintf("sum by (cluster_name) (increase(klio_plugin_backup_runs_total{outcome=\"failure\",%s}"+ + "[$__range]))", clientMatcher), "{{cluster_name}}"), ).Decimals(0). - Description("Base backups that completed successfully in the last 7 days. Counter resets on "+ - "plugin restart make long-window counts approximate.")), - - // Backup throughput and outcomes. - sized(8, panelHeight, timeseriesPanel("Backup run rate by outcome", "ops", - query(fmt.Sprintf("sum by (outcome) (rate(klio_plugin_backup_runs_total{%s}[$__rate_interval]))", nsMatcher), - "{{outcome}}"), - ).Description("Rate of base backup runs, broken down by outcome (success or failure).")), - sized(8, panelHeight, timeseriesPanel("Backup failure rate by category", "ops", + Orientation(common.VizOrientationHorizontal). + Description("Base backups that failed on the running clients sidecars, per cluster, "+ + "over the selected time range.")), + + sized(mediumPanelWidth, mediumPanelHeight, statPanel("Backup run success ratio", units.PercentUnit, query( - fmt.Sprintf("sum by (failure_category) "+ - "(rate(klio_plugin_backup_runs_total{outcome=\"failure\",%s}[$__rate_interval]))", nsMatcher), - "{{failure_category}}", + fmt.Sprintf("sum by (cluster_name) (increase(klio_plugin_backup_runs_total{outcome=\"success\",%s}"+ + "[$__range])) / clamp_min(sum by (cluster_name) (increase(klio_plugin_backup_runs_total{%s}"+ + "[$__range])), 1)", clientMatcher, clientMatcher), + "{{cluster_name}}", ), - ).Description("Rate of failed base backup runs, broken down by failure category, to show why "+ - "backups are failing.")), - // Distribution of backup wall-clock durations. The latest-duration stat - // tile above shows only the most recent run; these percentiles show the - // spread and let an admin see backup runtime trending up over time. - sized(8, panelHeight, timeseriesPanel("Backup duration (p50/p95/p99)", "s", - quantileTargetsRange("klio_plugin_backup_duration_seconds_bucket", "le", nsMatcher, "")..., - ).Description("Percentile wall-clock duration of base backup runs (across all outcomes), computed "+ - "over the whole selected range so the lines stay populated between runs. Widen the dashboard "+ - "range to span several backups for a stable reading; if the range contains no backup, the "+ - "panel is empty.")), - // Backup volume over time: count of runs per bucket, split by outcome. - // Bars aggregate how many backups happened, which is more useful for - // infrequent backups than instantaneous duration percentiles. - sized(8, panelHeight, barPanel("Backups by outcome", "short", - query( - fmt.Sprintf("sum by (outcome) (increase(klio_plugin_backup_runs_total{%s}[$__interval]))", nsMatcher), - "{{outcome}}"), - ).Description("Count of base backup runs per time bucket, split by outcome. Bars aggregate the "+ - "number of backups over each interval (per day on a multi-day range).")), - - // WAL streaming client, run as a child process of this same sidecar. - sized(8, panelHeight, barGaugePanel("Streaming timeline by cluster", - query(fmt.Sprintf("max by (cluster_name) (klio_client_wal_timeline{%s})", clientWalMatcher), + ).Orientation(common.VizOrientationHorizontal). + Description("Fraction of base backup runs that succeeded over the selected time range "+ + "(successful runs / total runs), per cluster.")), + + // State of the latest backup. + sized(mediumPanelWidth, mediumPanelHeight, + statPanel("Latest backup duration", units.DurationInDaysHoursMinutesSeconds, + query(fmt.Sprintf("max by (cluster_name) (klio_plugin_backup_latest_duration_seconds{%s})", clientMatcher), + "{{cluster_name}}"), + ).Orientation(common.VizOrientationHorizontal). + Description("Wall-clock duration of the most recent base backup, per cluster.")), + + sized(mediumPanelWidth, mediumPanelHeight, + statPanel("Latest successful backup age", units.DurationInDaysHoursMinutesSeconds, + query(fmt.Sprintf("time() - max by (cluster_name) (klio_plugin_backup_latest_completion_time_seconds{%s})", + clientMatcher), "{{cluster_name}}"), + ).Orientation(common.VizOrientationHorizontal). + Description("Elapsed time since the most recent base backup completed successfully, per cluster. A "+ + "value well above the backup interval means that cluster's backups have stopped succeeding.")), + + sized(mediumPanelWidth, mediumPanelHeight, + statPanel("Latest failed backup age", units.DurationInDaysHoursMinutesSeconds, + query(fmt.Sprintf("time() - max by (cluster_name) (klio_plugin_backup_latest_failure_time_seconds{%s})", + clientMatcher), "{{cluster_name}}"), + ).Orientation(common.VizOrientationHorizontal). + Description("Elapsed time since the most recent base backup failure, per cluster. A small value means "+ + "a failure happened recently.")), + + sized(mediumPanelWidth, mediumPanelHeight, + statPanel("Latest backup start age", units.DurationInDaysHoursMinutesSeconds, + query(fmt.Sprintf("time() - max by (cluster_name) (klio_plugin_backup_latest_start_time_seconds{%s})", + clientMatcher), "{{cluster_name}}"), + ).Orientation(common.VizOrientationHorizontal). + Description("Elapsed time since the most recent base backup started, per cluster. Compare against the "+ + "latest duration to tell whether a backup is still running or overdue.")), + + // Backup history charts. + sized(largePanelWidth, largePanelHeight, timeseriesPanel("Backup runs by outcome", units.Number, + query(fmt.Sprintf( + "round(sum by(cluster_name, outcome) (increase(klio_plugin_backup_runs_total{%s}[$__range])), 1)", + clientMatcher), "{{cluster_name}} / {{outcome}}"), + ).Description("Base backup runs, grouped by cluster and outcome (success or failure).")), + + sized(largePanelWidth, largePanelHeight, timeseriesPanel("Failed backup runs by reason", units.Number, + query(fmt.Sprintf( + "round(sum by (cluster_name, failure_category) "+ + "(increase(klio_plugin_backup_runs_total{outcome=\"failure\",%s}[$__range])), 1)", clientMatcher), + "{{cluster_name}} / {{failure_category}}", + ), + ).Description("Failed base backup runs, grouped by cluster and failure reason.")), + + sized(largePanelWidth, largePanelHeight, timeseriesPanel("Backup duration percentiles", units.Seconds, + quantileTargetsAbsolute("klio_plugin_backup_duration_seconds_bucket", "le, cluster_name", + "outcome=\"success\","+clientMatcher, "{{cluster_name}}")..., + ).Description("50th/90th/99th-percentile duration of successful base backup runs, per cluster.")), + + // WAL streaming. + sized(largePanelWidth, mediumPanelHeight, timelinePanel("WAL streaming timeline", + query(fmt.Sprintf("max by (cluster_name) (klio_client_wal_timeline{%s})", clientMatcher), "{{cluster_name}}"), - ).Description("PostgreSQL timeline the WAL streaming client is currently streaming, per cluster.")), - sized(16, panelHeight, timeseriesPanel("WAL block send duration (p50/p95/p99) by cluster", "ns", - quantileTargets("klio_client_wal_block_duration_nanoseconds_bucket", "le, cluster_name", - clientWalMatcher, "{{cluster_name}}")..., - ).Description("Percentile latency of the client's gRPC send of a WAL block to the server, per "+ - "cluster. Most meaningful under active write load; on an idle or low-write cluster, WAL "+ - "blocks are sent too infrequently for the underlying histogram_quantile to be reliable, so "+ - "the line may look sparse or noisy rather than absent.")), + ).Description("PostgreSQL timeline of the WAL streaming client.")), + + sized(largePanelWidth, largePanelHeight, + timeseriesPanel("WAL block send duration percentiles (rate)", units.Nanoseconds, + quantileTargets("klio_client_wal_block_duration_nanoseconds_bucket", "le, cluster_name", + clientMatcher, "{{cluster_name}}")..., + ).Description("50th/90th/99th-percentile duration of the client's gRPC send of a WAL block to the "+ + "server, per cluster. Reflects recent activity, over a rolling few-minute window.")), + + sized(largePanelWidth, largePanelHeight, + timeseriesPanel("WAL block send duration percentiles (total)", units.Nanoseconds, + quantileTargetsAbsolute("klio_client_wal_block_duration_nanoseconds_bucket", "le, cluster_name", + clientMatcher, "{{cluster_name}}")..., + ).Description("50th/90th/99th-percentile duration of the client's gRPC send of a WAL block to the "+ + "server, per cluster. Reflects all activity since the server last restarted.")), } } diff --git a/observability/grafana/klio-dashboard.json b/observability/grafana/klio-dashboard.json index 90b2cc82b..330ada6eb 100644 --- a/observability/grafana/klio-dashboard.json +++ b/observability/grafana/klio-dashboard.json @@ -31,6 +31,21 @@ "id": 0, "panels": [] }, + { + "type": "text", + "transparent": true, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 1 + }, + "repeatDirection": "h", + "options": { + "mode": "markdown", + "content": "Backup lifecycle and WAL streaming as seen by the plugin sidecar running in each PostgreSQL pod." + } + }, { "type": "stat", "targets": [ @@ -39,14 +54,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(klio_plugin_backup_in_progress{k8s_namespace_name=~\"$namespace\"})", + "expr": "sum by (cluster_name) (klio_plugin_backup_in_progress{k8s_namespace_name=~\"$namespace\",cluster_name=~\"$cluster\"})", "instant": false, - "legendFormat": "in progress", + "legendFormat": "{{cluster_name}}", "range": true } ], "title": "Backups in progress", - "description": "Base backups currently running across the plugin sidecars in the namespace.", + "description": "Base backups currently running, per cluster.", "transparent": false, "datasource": { "type": "prometheus", @@ -54,9 +69,9 @@ }, "gridPos": { "h": 6, - "w": 4, + "w": 6, "x": 0, - "y": 1 + "y": 3 }, "repeatDirection": "h", "options": { @@ -76,7 +91,7 @@ "valueSize": 28 }, "percentChangeColorMode": "standard", - "orientation": "" + "orientation": "horizontal" }, "fieldConfig": { "defaults": { @@ -97,14 +112,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "time() - max(klio_plugin_backup_latest_completion_time_seconds{k8s_namespace_name=~\"$namespace\"})", + "expr": "sum by (cluster_name) (increase(klio_plugin_backup_runs_total{outcome=\"success\",k8s_namespace_name=~\"$namespace\",cluster_name=~\"$cluster\"}[$__range]))", "instant": false, - "legendFormat": "since success", + "legendFormat": "{{cluster_name}}", "range": true } ], - "title": "Time since last successful backup", - "description": "Elapsed time since the most recent base backup completed successfully. A value well above the backup interval means backups have stopped succeeding.", + "title": "Successful backup runs", + "description": "Base backups completed successfully by the running clients sidecars, per cluster, over the selected time range.", "transparent": false, "datasource": { "type": "prometheus", @@ -112,9 +127,9 @@ }, "gridPos": { "h": 6, - "w": 4, - "x": 4, - "y": 1 + "w": 6, + "x": 6, + "y": 3 }, "repeatDirection": "h", "options": { @@ -134,11 +149,12 @@ "valueSize": 28 }, "percentChangeColorMode": "standard", - "orientation": "" + "orientation": "horizontal" }, "fieldConfig": { "defaults": { - "unit": "dtdurations", + "unit": "none", + "decimals": 0, "color": { "mode": "palette-classic" } @@ -154,14 +170,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "time() - max(klio_plugin_backup_latest_failure_time_seconds{k8s_namespace_name=~\"$namespace\"})", + "expr": "sum by (cluster_name) (increase(klio_plugin_backup_runs_total{outcome=\"failure\",k8s_namespace_name=~\"$namespace\",cluster_name=~\"$cluster\"}[$__range]))", "instant": false, - "legendFormat": "since failure", + "legendFormat": "{{cluster_name}}", "range": true } ], - "title": "Time since last failed backup", - "description": "Elapsed time since the most recent base backup failure. A small value means a failure happened recently.", + "title": "Failed backup runs", + "description": "Base backups that failed on the running clients sidecars, per cluster, over the selected time range.", "transparent": false, "datasource": { "type": "prometheus", @@ -169,9 +185,9 @@ }, "gridPos": { "h": 6, - "w": 4, - "x": 8, - "y": 1 + "w": 6, + "x": 12, + "y": 3 }, "repeatDirection": "h", "options": { @@ -191,11 +207,12 @@ "valueSize": 28 }, "percentChangeColorMode": "standard", - "orientation": "" + "orientation": "horizontal" }, "fieldConfig": { "defaults": { - "unit": "dtdurations", + "unit": "none", + "decimals": 0, "color": { "mode": "palette-classic" } @@ -211,14 +228,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "max(klio_plugin_backup_latest_duration_seconds{k8s_namespace_name=~\"$namespace\"})", + "expr": "sum by (cluster_name) (increase(klio_plugin_backup_runs_total{outcome=\"success\",k8s_namespace_name=~\"$namespace\",cluster_name=~\"$cluster\"}[$__range])) / clamp_min(sum by (cluster_name) (increase(klio_plugin_backup_runs_total{k8s_namespace_name=~\"$namespace\",cluster_name=~\"$cluster\"}[$__range])), 1)", "instant": false, - "legendFormat": "latest duration", + "legendFormat": "{{cluster_name}}", "range": true } ], - "title": "Latest backup duration", - "description": "Wall-clock duration of the most recent base backup.", + "title": "Backup run success ratio", + "description": "Fraction of base backup runs that succeeded over the selected time range (successful runs / total runs), per cluster.", "transparent": false, "datasource": { "type": "prometheus", @@ -226,9 +243,9 @@ }, "gridPos": { "h": 6, - "w": 4, - "x": 12, - "y": 1 + "w": 6, + "x": 18, + "y": 3 }, "repeatDirection": "h", "options": { @@ -248,11 +265,11 @@ "valueSize": 28 }, "percentChangeColorMode": "standard", - "orientation": "" + "orientation": "horizontal" }, "fieldConfig": { "defaults": { - "unit": "dtdurations", + "unit": "percentunit", "color": { "mode": "palette-classic" } @@ -268,14 +285,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "time() - max(klio_plugin_backup_latest_start_time_seconds{k8s_namespace_name=~\"$namespace\"})", + "expr": "max by (cluster_name) (klio_plugin_backup_latest_duration_seconds{k8s_namespace_name=~\"$namespace\",cluster_name=~\"$cluster\"})", "instant": false, - "legendFormat": "since start", + "legendFormat": "{{cluster_name}}", "range": true } ], - "title": "Time since last backup started", - "description": "Elapsed time since the most recent base backup started. Compare against the latest duration to tell whether a backup is still running or overdue.", + "title": "Latest backup duration", + "description": "Wall-clock duration of the most recent base backup, per cluster.", "transparent": false, "datasource": { "type": "prometheus", @@ -283,9 +300,9 @@ }, "gridPos": { "h": 6, - "w": 4, - "x": 16, - "y": 1 + "w": 6, + "x": 0, + "y": 9 }, "repeatDirection": "h", "options": { @@ -305,11 +322,11 @@ "valueSize": 28 }, "percentChangeColorMode": "standard", - "orientation": "" + "orientation": "horizontal" }, "fieldConfig": { "defaults": { - "unit": "dtdurations", + "unit": "dtdhms", "color": { "mode": "palette-classic" } @@ -325,14 +342,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(increase(klio_plugin_backup_runs_total{outcome=\"success\",k8s_namespace_name=~\"$namespace\"}[$__range])) / clamp_min(sum(increase(klio_plugin_backup_runs_total{k8s_namespace_name=~\"$namespace\"}[$__range])), 1)", + "expr": "time() - max by (cluster_name) (klio_plugin_backup_latest_completion_time_seconds{k8s_namespace_name=~\"$namespace\",cluster_name=~\"$cluster\"})", "instant": false, - "legendFormat": "success ratio", + "legendFormat": "{{cluster_name}}", "range": true } ], - "title": "Backup success ratio", - "description": "Fraction of base backup runs that succeeded over the selected time range (successful runs / total runs).", + "title": "Latest successful backup age", + "description": "Elapsed time since the most recent base backup completed successfully, per cluster. A value well above the backup interval means that cluster's backups have stopped succeeding.", "transparent": false, "datasource": { "type": "prometheus", @@ -340,9 +357,9 @@ }, "gridPos": { "h": 6, - "w": 4, - "x": 20, - "y": 1 + "w": 6, + "x": 6, + "y": 9 }, "repeatDirection": "h", "options": { @@ -362,11 +379,11 @@ "valueSize": 28 }, "percentChangeColorMode": "standard", - "orientation": "" + "orientation": "horizontal" }, "fieldConfig": { "defaults": { - "unit": "percentunit", + "unit": "dtdhms", "color": { "mode": "palette-classic" } @@ -382,14 +399,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(increase(klio_plugin_backup_runs_total{outcome=\"success\",k8s_namespace_name=~\"$namespace\"}[24h]))", + "expr": "time() - max by (cluster_name) (klio_plugin_backup_latest_failure_time_seconds{k8s_namespace_name=~\"$namespace\",cluster_name=~\"$cluster\"})", "instant": false, - "legendFormat": "last 24h", + "legendFormat": "{{cluster_name}}", "range": true } ], - "title": "Successful backups (24h)", - "description": "Base backups that completed successfully in the last 24 hours. Counter resets on plugin restart make long-window counts approximate.", + "title": "Latest failed backup age", + "description": "Elapsed time since the most recent base backup failure, per cluster. A small value means a failure happened recently.", "transparent": false, "datasource": { "type": "prometheus", @@ -397,9 +414,9 @@ }, "gridPos": { "h": 6, - "w": 4, - "x": 0, - "y": 7 + "w": 6, + "x": 12, + "y": 9 }, "repeatDirection": "h", "options": { @@ -419,12 +436,11 @@ "valueSize": 28 }, "percentChangeColorMode": "standard", - "orientation": "" + "orientation": "horizontal" }, "fieldConfig": { "defaults": { - "unit": "none", - "decimals": 0, + "unit": "dtdhms", "color": { "mode": "palette-classic" } @@ -440,14 +456,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(increase(klio_plugin_backup_runs_total{outcome=\"success\",k8s_namespace_name=~\"$namespace\"}[7d]))", + "expr": "time() - max by (cluster_name) (klio_plugin_backup_latest_start_time_seconds{k8s_namespace_name=~\"$namespace\",cluster_name=~\"$cluster\"})", "instant": false, - "legendFormat": "last 7d", + "legendFormat": "{{cluster_name}}", "range": true } ], - "title": "Successful backups (7d)", - "description": "Base backups that completed successfully in the last 7 days. Counter resets on plugin restart make long-window counts approximate.", + "title": "Latest backup start age", + "description": "Elapsed time since the most recent base backup started, per cluster. Compare against the latest duration to tell whether a backup is still running or overdue.", "transparent": false, "datasource": { "type": "prometheus", @@ -455,9 +471,9 @@ }, "gridPos": { "h": 6, - "w": 4, - "x": 4, - "y": 7 + "w": 6, + "x": 18, + "y": 9 }, "repeatDirection": "h", "options": { @@ -477,12 +493,11 @@ "valueSize": 28 }, "percentChangeColorMode": "standard", - "orientation": "" + "orientation": "horizontal" }, "fieldConfig": { "defaults": { - "unit": "none", - "decimals": 0, + "unit": "dtdhms", "color": { "mode": "palette-classic" } @@ -498,24 +513,24 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum by (outcome) (rate(klio_plugin_backup_runs_total{k8s_namespace_name=~\"$namespace\"}[$__rate_interval]))", + "expr": "round(sum by(cluster_name, outcome) (increase(klio_plugin_backup_runs_total{k8s_namespace_name=~\"$namespace\",cluster_name=~\"$cluster\"}[$__range])), 1)", "instant": false, - "legendFormat": "{{outcome}}", + "legendFormat": "{{cluster_name}} / {{outcome}}", "range": true } ], - "title": "Backup run rate by outcome", - "description": "Rate of base backup runs, broken down by outcome (success or failure).", + "title": "Backup runs by outcome", + "description": "Base backup runs, grouped by cluster and outcome (success or failure).", "transparent": false, "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { - "h": 6, + "h": 8, "w": 8, - "x": 8, - "y": 7 + "x": 0, + "y": 15 }, "repeatDirection": "h", "options": { @@ -532,7 +547,7 @@ }, "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "none", "custom": { "gradientMode": "opacity", "fillOpacity": 10 @@ -549,24 +564,24 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum by (failure_category) (rate(klio_plugin_backup_runs_total{outcome=\"failure\",k8s_namespace_name=~\"$namespace\"}[$__rate_interval]))", + "expr": "round(sum by (cluster_name, failure_category) (increase(klio_plugin_backup_runs_total{outcome=\"failure\",k8s_namespace_name=~\"$namespace\",cluster_name=~\"$cluster\"}[$__range])), 1)", "instant": false, - "legendFormat": "{{failure_category}}", + "legendFormat": "{{cluster_name}} / {{failure_category}}", "range": true } ], - "title": "Backup failure rate by category", - "description": "Rate of failed base backup runs, broken down by failure category, to show why backups are failing.", + "title": "Failed backup runs by reason", + "description": "Failed base backup runs, grouped by cluster and failure reason.", "transparent": false, "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { - "h": 6, + "h": 8, "w": 8, - "x": 16, - "y": 7 + "x": 8, + "y": 15 }, "repeatDirection": "h", "options": { @@ -583,7 +598,7 @@ }, "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "none", "custom": { "gradientMode": "opacity", "fillOpacity": 10 @@ -600,9 +615,9 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "histogram_quantile(0.50, sum by (le) (increase(klio_plugin_backup_duration_seconds_bucket{k8s_namespace_name=~\"$namespace\"}[$__range])))", + "expr": "histogram_quantile(0.50, sum by (le, cluster_name) (klio_plugin_backup_duration_seconds_bucket{outcome=\"success\",k8s_namespace_name=~\"$namespace\",cluster_name=~\"$cluster\"}))", "instant": false, - "legendFormat": "p50", + "legendFormat": "{{cluster_name}} p50", "range": true }, { @@ -610,9 +625,9 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "histogram_quantile(0.95, sum by (le) (increase(klio_plugin_backup_duration_seconds_bucket{k8s_namespace_name=~\"$namespace\"}[$__range])))", + "expr": "histogram_quantile(0.90, sum by (le, cluster_name) (klio_plugin_backup_duration_seconds_bucket{outcome=\"success\",k8s_namespace_name=~\"$namespace\",cluster_name=~\"$cluster\"}))", "instant": false, - "legendFormat": "p95", + "legendFormat": "{{cluster_name}} p90", "range": true }, { @@ -620,24 +635,24 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "histogram_quantile(0.99, sum by (le) (increase(klio_plugin_backup_duration_seconds_bucket{k8s_namespace_name=~\"$namespace\"}[$__range])))", + "expr": "histogram_quantile(0.99, sum by (le, cluster_name) (klio_plugin_backup_duration_seconds_bucket{outcome=\"success\",k8s_namespace_name=~\"$namespace\",cluster_name=~\"$cluster\"}))", "instant": false, - "legendFormat": "p99", + "legendFormat": "{{cluster_name}} p99", "range": true } ], - "title": "Backup duration (p50/p95/p99)", - "description": "Percentile wall-clock duration of base backup runs (across all outcomes), computed over the whole selected range so the lines stay populated between runs. Widen the dashboard range to span several backups for a stable reading; if the range contains no backup, the panel is empty.", + "title": "Backup duration percentiles", + "description": "50th/90th/99th-percentile duration of successful base backup runs, per cluster.", "transparent": false, "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { - "h": 6, + "h": 8, "w": 8, - "x": 0, - "y": 13 + "x": 16, + "y": 15 }, "repeatDirection": "h", "options": { @@ -671,24 +686,24 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum by (outcome) (increase(klio_plugin_backup_runs_total{k8s_namespace_name=~\"$namespace\"}[$__interval]))", + "expr": "max by (cluster_name) (klio_client_wal_timeline{k8s_namespace_name=~\"$namespace\",cluster_name=~\"$cluster\"})", "instant": false, - "legendFormat": "{{outcome}}", + "legendFormat": "{{cluster_name}}", "range": true } ], - "title": "Backups by outcome", - "description": "Count of base backup runs per time bucket, split by outcome. Bars aggregate the number of backups over each interval (per day on a multi-day range).", + "title": "WAL streaming timeline", + "description": "PostgreSQL timeline of the WAL streaming client.", "transparent": false, "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { - "h": 6, + "h": 8, "w": 8, - "x": 8, - "y": 13 + "x": 0, + "y": 23 }, "repeatDirection": "h", "options": { @@ -705,74 +720,85 @@ }, "fieldConfig": { "defaults": { - "unit": "short", + "unit": "none", "decimals": 0, "custom": { - "drawStyle": "bars", "gradientMode": "opacity", - "fillOpacity": 80, - "stacking": { - "mode": "normal" - } + "lineWidth": 2, + "lineInterpolation": "stepAfter", + "fillOpacity": 0 } }, "overrides": [] } }, { - "type": "bargauge", + "type": "timeseries", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "max by (cluster_name) (klio_client_wal_timeline{k8s_namespace_name=~\"$namespace\",cluster_name=~\"$cluster\"})", + "expr": "histogram_quantile(0.50, sum by (le, cluster_name) (rate(klio_client_wal_block_duration_nanoseconds_bucket{k8s_namespace_name=~\"$namespace\",cluster_name=~\"$cluster\"}[$__rate_interval])))", "instant": false, - "legendFormat": "{{cluster_name}}", + "legendFormat": "{{cluster_name}} p50", + "range": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.90, sum by (le, cluster_name) (rate(klio_client_wal_block_duration_nanoseconds_bucket{k8s_namespace_name=~\"$namespace\",cluster_name=~\"$cluster\"}[$__rate_interval])))", + "instant": false, + "legendFormat": "{{cluster_name}} p90", + "range": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.99, sum by (le, cluster_name) (rate(klio_client_wal_block_duration_nanoseconds_bucket{k8s_namespace_name=~\"$namespace\",cluster_name=~\"$cluster\"}[$__rate_interval])))", + "instant": false, + "legendFormat": "{{cluster_name}} p99", "range": true } ], - "title": "Streaming timeline by cluster", - "description": "PostgreSQL timeline the WAL streaming client is currently streaming, per cluster.", + "title": "WAL block send duration percentiles (rate)", + "description": "50th/90th/99th-percentile duration of the client's gRPC send of a WAL block to the server, per cluster. Reflects recent activity, over a rolling few-minute window.", "transparent": false, "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { - "h": 6, + "h": 8, "w": 8, - "x": 16, - "y": 13 + "x": 8, + "y": 23 }, "repeatDirection": "h", "options": { - "displayMode": "gradient", - "valueMode": "color", - "namePlacement": "auto", - "showUnfilled": true, - "sizing": "auto", - "minVizWidth": 8, - "minVizHeight": 16, "legend": { "displayMode": "list", "placement": "bottom", - "showLegend": false, + "showLegend": true, "calcs": [] }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "maxVizHeight": 300, - "orientation": "horizontal" + "tooltip": { + "mode": "", + "sort": "" + } }, "fieldConfig": { "defaults": { - "unit": "none", - "decimals": 0 + "unit": "ns", + "custom": { + "gradientMode": "opacity", + "fillOpacity": 10 + } }, "overrides": [] } @@ -785,9 +811,9 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "histogram_quantile(0.50, sum by (le, cluster_name) (rate(klio_client_wal_block_duration_nanoseconds_bucket{k8s_namespace_name=~\"$namespace\",cluster_name=~\"$cluster\"}[$__rate_interval])))", + "expr": "histogram_quantile(0.50, sum by (le, cluster_name) (klio_client_wal_block_duration_nanoseconds_bucket{k8s_namespace_name=~\"$namespace\",cluster_name=~\"$cluster\"}))", "instant": false, - "legendFormat": "p50 {{cluster_name}}", + "legendFormat": "{{cluster_name}} p50", "range": true }, { @@ -795,9 +821,9 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "histogram_quantile(0.95, sum by (le, cluster_name) (rate(klio_client_wal_block_duration_nanoseconds_bucket{k8s_namespace_name=~\"$namespace\",cluster_name=~\"$cluster\"}[$__rate_interval])))", + "expr": "histogram_quantile(0.90, sum by (le, cluster_name) (klio_client_wal_block_duration_nanoseconds_bucket{k8s_namespace_name=~\"$namespace\",cluster_name=~\"$cluster\"}))", "instant": false, - "legendFormat": "p95 {{cluster_name}}", + "legendFormat": "{{cluster_name}} p90", "range": true }, { @@ -805,24 +831,24 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "histogram_quantile(0.99, sum by (le, cluster_name) (rate(klio_client_wal_block_duration_nanoseconds_bucket{k8s_namespace_name=~\"$namespace\",cluster_name=~\"$cluster\"}[$__rate_interval])))", + "expr": "histogram_quantile(0.99, sum by (le, cluster_name) (klio_client_wal_block_duration_nanoseconds_bucket{k8s_namespace_name=~\"$namespace\",cluster_name=~\"$cluster\"}))", "instant": false, - "legendFormat": "p99 {{cluster_name}}", + "legendFormat": "{{cluster_name}} p99", "range": true } ], - "title": "WAL block send duration (p50/p95/p99) by cluster", - "description": "Percentile latency of the client's gRPC send of a WAL block to the server, per cluster. Most meaningful under active write load; on an idle or low-write cluster, WAL blocks are sent too infrequently for the underlying histogram_quantile to be reliable, so the line may look sparse or noisy rather than absent.", + "title": "WAL block send duration percentiles (total)", + "description": "50th/90th/99th-percentile duration of the client's gRPC send of a WAL block to the server, per cluster. Reflects all activity since the server last restarted.", "transparent": false, "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 19 + "h": 8, + "w": 8, + "x": 16, + "y": 23 }, "repeatDirection": "h", "options": { @@ -856,11 +882,26 @@ "h": 1, "w": 24, "x": 0, - "y": 25 + "y": 31 }, "id": 0, "panels": [] }, + { + "type": "text", + "transparent": true, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 32 + }, + "repeatDirection": "h", + "options": { + "mode": "markdown", + "content": "State of the Klio server: WAL ingest, retained backups and snapshots, and the internal queue." + } + }, { "type": "stat", "targets": [ @@ -869,14 +910,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "max(klio_server_uptime_seconds{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\"})", + "expr": "max by (service_name) (klio_server_uptime_seconds{service_name=~\"$server\"})", "instant": false, - "legendFormat": "uptime", + "legendFormat": "{{service_name}}", "range": true } ], "title": "Server uptime", - "description": "Time since the Klio server process started. A sudden drop means the server StatefulSet restarted.", + "description": "Time since the Klio server process started, per server. A sudden drop means that server's StatefulSet restarted.", "transparent": false, "datasource": { "type": "prometheus", @@ -884,9 +925,9 @@ }, "gridPos": { "h": 6, - "w": 4, + "w": 8, "x": 0, - "y": 26 + "y": 34 }, "repeatDirection": "h", "options": { @@ -906,11 +947,11 @@ "valueSize": 28 }, "percentChangeColorMode": "standard", - "orientation": "" + "orientation": "horizontal" }, "fieldConfig": { "defaults": { - "unit": "dtdurations", + "unit": "dtdhms", "color": { "mode": "palette-classic" } @@ -919,79 +960,21 @@ } }, { - "type": "bargauge", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "expr": "max by (tier) (klio_server_wal_latest_written_timeline{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\",cluster_name=~\"$cluster\"})", - "instant": false, - "legendFormat": "{{tier}}", - "range": true - } - ], - "title": "Latest WAL timeline by tier", - "description": "Current PostgreSQL timeline of the latest WAL written per tier. A change reflects a promotion or failover.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 4, - "y": 26 - }, - "repeatDirection": "h", - "options": { - "displayMode": "gradient", - "valueMode": "color", - "namePlacement": "auto", - "showUnfilled": true, - "sizing": "auto", - "minVizWidth": 8, - "minVizHeight": 16, - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": false, - "calcs": [] - }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "maxVizHeight": 300, - "orientation": "horizontal" - }, - "fieldConfig": { - "defaults": { - "unit": "none", - "decimals": 0 - }, - "overrides": [] - } - }, - { - "type": "bargauge", + "type": "stat", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum by (tier) (klio_server_backup_snapshots{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\"})", + "expr": "sum by (cluster_name, tier) (klio_server_backup_backups{service_name=~\"$server\",cluster_name=~\"$cluster\"})", "instant": false, - "legendFormat": "{{tier}}", + "legendFormat": "{{cluster_name}} {{tier}}", "range": true } ], - "title": "Base snapshots by tier", - "description": "Base backup snapshots currently retained per tier.", + "title": "Retained backups", + "description": "Number of PostgreSQL backups currently retained per cluster and tier.", "transparent": false, "datasource": { "type": "prometheus", @@ -999,37 +982,37 @@ }, "gridPos": { "h": 6, - "w": 4, + "w": 8, "x": 8, - "y": 26 + "y": 34 }, "repeatDirection": "h", "options": { - "displayMode": "gradient", - "valueMode": "color", - "namePlacement": "auto", - "showUnfilled": true, - "sizing": "auto", - "minVizWidth": 8, - "minVizHeight": 16, - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": false, - "calcs": [] - }, + "graphMode": "none", + "colorMode": "value", + "justifyMode": "auto", + "textMode": "auto", + "wideLayout": true, + "showPercentChange": false, "reduceOptions": { "calcs": [ "lastNotNull" ] }, - "maxVizHeight": 300, + "text": { + "titleSize": 14, + "valueSize": 28 + }, + "percentChangeColorMode": "standard", "orientation": "horizontal" }, "fieldConfig": { "defaults": { "unit": "none", - "decimals": 0 + "decimals": 0, + "color": { + "mode": "palette-classic" + } }, "overrides": [] } @@ -1042,14 +1025,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "max(klio_server_backup_latest_snapshot_size_bytes{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\"})", + "expr": "min by (cluster_name, tier) (klio_server_backup_oldest_backup_completion_time_seconds{service_name=~\"$server\",cluster_name=~\"$cluster\"}) * 1000", "instant": false, - "legendFormat": "latest size", + "legendFormat": "{{cluster_name}} {{tier}}", "range": true } ], - "title": "Latest snapshot size", - "description": "Size on the backend of the most recent base backup snapshot.", + "title": "Oldest restorable point", + "description": "Timestamp at which the oldest retained PostgreSQL backup completed, per cluster and tier. It is the earliest point in time a restore can target, as long as all the WAL files since the backup started are still retained.", "transparent": false, "datasource": { "type": "prometheus", @@ -1057,9 +1040,9 @@ }, "gridPos": { "h": 6, - "w": 4, - "x": 12, - "y": 26 + "w": 8, + "x": 16, + "y": 34 }, "repeatDirection": "h", "options": { @@ -1079,11 +1062,11 @@ "valueSize": 28 }, "percentChangeColorMode": "standard", - "orientation": "" + "orientation": "horizontal" }, "fieldConfig": { "defaults": { - "unit": "bytes", + "unit": "dateTimeAsIso", "color": { "mode": "palette-classic" } @@ -1099,14 +1082,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "max(klio_server_backup_latest_snapshot_files{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\"})", + "expr": "time() - max by (cluster_name, tier) (klio_server_backup_latest_backup_completion_time_seconds{service_name=~\"$server\",cluster_name=~\"$cluster\"})", "instant": false, - "legendFormat": "latest files", + "legendFormat": "{{cluster_name}} {{tier}}", "range": true } ], - "title": "Latest snapshot files", - "description": "Number of files in the most recent base backup snapshot.", + "title": "Latest backup age (completion)", + "description": "Elapsed time since the most recently retained PostgreSQL backup completed, per cluster and tier.", "transparent": false, "datasource": { "type": "prometheus", @@ -1114,9 +1097,9 @@ }, "gridPos": { "h": 6, - "w": 4, - "x": 16, - "y": 26 + "w": 8, + "x": 0, + "y": 40 }, "repeatDirection": "h", "options": { @@ -1136,12 +1119,11 @@ "valueSize": 28 }, "percentChangeColorMode": "standard", - "orientation": "" + "orientation": "horizontal" }, "fieldConfig": { "defaults": { - "unit": "short", - "decimals": 0, + "unit": "dtdhms", "color": { "mode": "palette-classic" } @@ -1157,14 +1139,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "max(klio_server_backup_latest_snapshot_dirs{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\"})", + "expr": "time() - max by (cluster_name, tier) (klio_server_backup_latest_backup_start_time_seconds{service_name=~\"$server\",cluster_name=~\"$cluster\"})", "instant": false, - "legendFormat": "latest dirs", + "legendFormat": "{{cluster_name}} {{tier}}", "range": true } ], - "title": "Latest snapshot dirs", - "description": "Number of directories in the most recent base backup snapshot.", + "title": "Latest backup age (start)", + "description": "Elapsed time since the most recently retained PostgreSQL backup started, per cluster and tier.", "transparent": false, "datasource": { "type": "prometheus", @@ -1172,9 +1154,9 @@ }, "gridPos": { "h": 6, - "w": 4, - "x": 20, - "y": 26 + "w": 8, + "x": 8, + "y": 40 }, "repeatDirection": "h", "options": { @@ -1194,12 +1176,11 @@ "valueSize": 28 }, "percentChangeColorMode": "standard", - "orientation": "" + "orientation": "horizontal" }, "fieldConfig": { "defaults": { - "unit": "short", - "decimals": 0, + "unit": "dtdhms", "color": { "mode": "palette-classic" } @@ -1215,14 +1196,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "time() - max(klio_server_backup_latest_snapshot_timestamp_seconds{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\"})", + "expr": "max by (cluster_name, tier) (klio_server_backup_latest_backup_timeline{service_name=~\"$server\",cluster_name=~\"$cluster\"})", "instant": false, - "legendFormat": "latest age", + "legendFormat": "{{cluster_name}} {{tier}}", "range": true } ], - "title": "Latest snapshot age", - "description": "Age of the most recent base backup snapshot. Should stay below the backup interval.", + "title": "Latest backup timeline", + "description": "PostgreSQL timeline of the latest retained backup, per cluster and tier. Differing from the oldest timeline means the retention window spans a promotion or failover.", "transparent": false, "datasource": { "type": "prometheus", @@ -1230,9 +1211,9 @@ }, "gridPos": { "h": 6, - "w": 4, - "x": 0, - "y": 32 + "w": 8, + "x": 16, + "y": 40 }, "repeatDirection": "h", "options": { @@ -1252,11 +1233,12 @@ "valueSize": 28 }, "percentChangeColorMode": "standard", - "orientation": "" + "orientation": "horizontal" }, "fieldConfig": { "defaults": { - "unit": "dtdurations", + "unit": "none", + "decimals": 0, "color": { "mode": "palette-classic" } @@ -1265,31 +1247,455 @@ } }, { - "type": "stat", + "type": "table", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "time() - min(klio_server_backup_oldest_snapshot_timestamp_seconds{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\"})", - "instant": false, - "legendFormat": "oldest age", - "range": true - } - ], - "title": "Oldest snapshot age", - "description": "Age of the oldest retained base backup snapshot, reflecting the effective retention horizon.", - "transparent": false, - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, + "expr": "floor(max by (cluster_name, tier) (klio_server_backup_latest_backup_start_lsn_bytes{service_name=~\"$server\",cluster_name=~\"$cluster\"}) / 4294967296)", + "format": "table", + "hide": true, + "instant": true, + "range": false, + "refId": "h0" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "max by (cluster_name, tier) (klio_server_backup_latest_backup_start_lsn_bytes{service_name=~\"$server\",cluster_name=~\"$cluster\"}) % 4294967296", + "format": "table", + "hide": true, + "instant": true, + "range": false, + "refId": "l0" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "floor(max by (cluster_name, tier) (klio_server_backup_latest_backup_end_lsn_bytes{service_name=~\"$server\",cluster_name=~\"$cluster\"}) / 4294967296)", + "format": "table", + "hide": true, + "instant": true, + "range": false, + "refId": "h1" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "max by (cluster_name, tier) (klio_server_backup_latest_backup_end_lsn_bytes{service_name=~\"$server\",cluster_name=~\"$cluster\"}) % 4294967296", + "format": "table", + "hide": true, + "instant": true, + "range": false, + "refId": "l1" + }, + { + "datasource": { + "type": "__expr__", + "uid": "__expr__" + }, + "expression": "SELECT\n h0.cluster_name AS \"Cluster Name\",\n h0.tier AS \"Tier\",\n concat(CONV(h0.__value__, 10, 16), \"/\", CONV(l0.__value__, 10, 16)) as \"start LSN\",\n concat(CONV(h1.__value__, 10, 16), \"/\", CONV(l1.__value__, 10, 16)) as \"end LSN\"\nFROM h0\nJOIN l0 ON l0.cluster_name = h0.cluster_name AND l0.tier = h0.tier\nJOIN h1 ON h1.cluster_name = h0.cluster_name AND h1.tier = h0.tier\nJOIN l1 ON l1.cluster_name = h0.cluster_name AND l1.tier = h0.tier", + "hide": false, + "refId": "sql", + "type": "sql" + } + ], + "title": "Latest backup LSN", + "description": "Start and end LSN of the latest retained PostgreSQL backup, per cluster and tier.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 46 + }, + "repeatDirection": "h", + "options": { + "frameIndex": 0, + "showHeader": true, + "showTypeIcons": false, + "footer": { + "show": false, + "reducer": null, + "countRows": false + }, + "cellHeight": "sm" + }, + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "inspect": false + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "time() - min by (cluster_name, tier) (klio_server_backup_oldest_backup_completion_time_seconds{service_name=~\"$server\",cluster_name=~\"$cluster\"})", + "instant": false, + "legendFormat": "{{cluster_name}} {{tier}}", + "range": true + } + ], + "title": "Oldest backup age (completion)", + "description": "Elapsed time since the oldest retained PostgreSQL backup completed, per cluster and tier, reflecting each tier's effective retention horizon.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 52 + }, + "repeatDirection": "h", + "options": { + "graphMode": "none", + "colorMode": "value", + "justifyMode": "auto", + "textMode": "auto", + "wideLayout": true, + "showPercentChange": false, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "text": { + "titleSize": 14, + "valueSize": 28 + }, + "percentChangeColorMode": "standard", + "orientation": "horizontal" + }, + "fieldConfig": { + "defaults": { + "unit": "dtdhms", + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "time() - min by (cluster_name, tier) (klio_server_backup_oldest_backup_start_time_seconds{service_name=~\"$server\",cluster_name=~\"$cluster\"})", + "instant": false, + "legendFormat": "{{cluster_name}} {{tier}}", + "range": true + } + ], + "title": "Oldest backup age (start)", + "description": "Elapsed time since the oldest retained PostgreSQL backup started, per cluster and tier, reflecting each tier's effective retention horizon.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 52 + }, + "repeatDirection": "h", + "options": { + "graphMode": "none", + "colorMode": "value", + "justifyMode": "auto", + "textMode": "auto", + "wideLayout": true, + "showPercentChange": false, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "text": { + "titleSize": 14, + "valueSize": 28 + }, + "percentChangeColorMode": "standard", + "orientation": "horizontal" + }, + "fieldConfig": { + "defaults": { + "unit": "dtdhms", + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "max by (cluster_name, tier) (klio_server_backup_oldest_backup_timeline{service_name=~\"$server\",cluster_name=~\"$cluster\"})", + "instant": false, + "legendFormat": "{{cluster_name}} {{tier}}", + "range": true + } + ], + "title": "Oldest backup timeline", + "description": "PostgreSQL timeline of the oldest retained backup, per cluster and tier. Differing from the latest timeline means the retention window spans a promotion or failover.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 52 + }, + "repeatDirection": "h", + "options": { + "graphMode": "none", + "colorMode": "value", + "justifyMode": "auto", + "textMode": "auto", + "wideLayout": true, + "showPercentChange": false, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "text": { + "titleSize": 14, + "valueSize": 28 + }, + "percentChangeColorMode": "standard", + "orientation": "horizontal" + }, + "fieldConfig": { + "defaults": { + "unit": "none", + "decimals": 0, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + } + }, + { + "type": "table", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "floor(max by (cluster_name, tier) (klio_server_backup_oldest_backup_start_lsn_bytes{service_name=~\"$server\",cluster_name=~\"$cluster\"}) / 4294967296)", + "format": "table", + "hide": true, + "instant": true, + "range": false, + "refId": "h0" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "max by (cluster_name, tier) (klio_server_backup_oldest_backup_start_lsn_bytes{service_name=~\"$server\",cluster_name=~\"$cluster\"}) % 4294967296", + "format": "table", + "hide": true, + "instant": true, + "range": false, + "refId": "l0" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "floor(max by (cluster_name, tier) (klio_server_backup_oldest_backup_end_lsn_bytes{service_name=~\"$server\",cluster_name=~\"$cluster\"}) / 4294967296)", + "format": "table", + "hide": true, + "instant": true, + "range": false, + "refId": "h1" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "max by (cluster_name, tier) (klio_server_backup_oldest_backup_end_lsn_bytes{service_name=~\"$server\",cluster_name=~\"$cluster\"}) % 4294967296", + "format": "table", + "hide": true, + "instant": true, + "range": false, + "refId": "l1" + }, + { + "datasource": { + "type": "__expr__", + "uid": "__expr__" + }, + "expression": "SELECT\n h0.cluster_name AS \"Cluster Name\",\n h0.tier AS \"Tier\",\n concat(CONV(h0.__value__, 10, 16), \"/\", CONV(l0.__value__, 10, 16)) as \"start LSN\",\n concat(CONV(h1.__value__, 10, 16), \"/\", CONV(l1.__value__, 10, 16)) as \"end LSN\"\nFROM h0\nJOIN l0 ON l0.cluster_name = h0.cluster_name AND l0.tier = h0.tier\nJOIN h1 ON h1.cluster_name = h0.cluster_name AND h1.tier = h0.tier\nJOIN l1 ON l1.cluster_name = h0.cluster_name AND l1.tier = h0.tier", + "hide": false, + "refId": "sql", + "type": "sql" + } + ], + "title": "Oldest backup LSN", + "description": "Start and end LSN of the oldest retained PostgreSQL backup, per cluster and tier.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 58 + }, + "repeatDirection": "h", + "options": { + "frameIndex": 0, + "showHeader": true, + "showTypeIcons": false, + "footer": { + "show": false, + "reducer": null, + "countRows": false + }, + "cellHeight": "sm" + }, + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "inspect": false + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "sum by (cluster, tier) (label_replace(klio_server_backup_snapshots{service_name=~\"$server\"}, \"cluster\", \"$1\", \"snapshot_source\", \"[^@]*@([^:]+):.*\"))", + "instant": false, + "legendFormat": "{{cluster}} {{tier}}", + "range": true + } + ], + "title": "Retained snapshots", + "description": "Total kopia snapshots currently retained per cluster and tier.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 64 + }, + "repeatDirection": "h", + "options": { + "graphMode": "none", + "colorMode": "value", + "justifyMode": "auto", + "textMode": "auto", + "wideLayout": true, + "showPercentChange": false, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "text": { + "titleSize": 14, + "valueSize": 28 + }, + "percentChangeColorMode": "standard", + "orientation": "horizontal" + }, + "fieldConfig": { + "defaults": { + "unit": "none", + "decimals": 0, + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + } + }, + { + "type": "stat", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "max by (cluster, tier) (label_replace(klio_server_backup_latest_snapshot_size_bytes{service_name=~\"$server\"}, \"cluster\", \"$1\", \"snapshot_source\", \"[^@]*@([^:]+):.*\"))", + "instant": false, + "legendFormat": "{{cluster}} {{tier}}", + "range": true + } + ], + "title": "Latest snapshot size", + "description": "Size of the most recent base backup snapshot on Kopia, not accounting for deduplication.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, "gridPos": { "h": 6, "w": 4, "x": 4, - "y": 32 + "y": 64 }, "repeatDirection": "h", "options": { @@ -1309,11 +1715,11 @@ "valueSize": 28 }, "percentChangeColorMode": "standard", - "orientation": "" + "orientation": "horizontal" }, "fieldConfig": { "defaults": { - "unit": "dtdurations", + "unit": "bytes", "color": { "mode": "palette-classic" } @@ -1329,14 +1735,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "time() - max(klio_server_backup_latest_backup_start_time_seconds{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\",cluster_name=~\"$cluster\"})", + "expr": "max by (cluster, tier) (label_replace(klio_server_backup_latest_snapshot_files{service_name=~\"$server\"}, \"cluster\", \"$1\", \"snapshot_source\", \"[^@]*@([^:]+):.*\"))", "instant": false, - "legendFormat": "latest start age", + "legendFormat": "{{cluster}} {{tier}}", "range": true } ], - "title": "Latest backup age (start)", - "description": "Elapsed time since the most recently retained PostgreSQL backup started.", + "title": "Latest snapshot files", + "description": "Number of files in the most recent base backup snapshot on Kopia.", "transparent": false, "datasource": { "type": "prometheus", @@ -1346,7 +1752,7 @@ "h": 6, "w": 4, "x": 8, - "y": 32 + "y": 64 }, "repeatDirection": "h", "options": { @@ -1366,11 +1772,12 @@ "valueSize": 28 }, "percentChangeColorMode": "standard", - "orientation": "" + "orientation": "horizontal" }, "fieldConfig": { "defaults": { - "unit": "dtdurations", + "unit": "none", + "decimals": 0, "color": { "mode": "palette-classic" } @@ -1386,14 +1793,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "time() - max(klio_server_backup_latest_backup_completion_time_seconds{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\",cluster_name=~\"$cluster\"})", + "expr": "max by (cluster, tier) (label_replace(klio_server_backup_latest_snapshot_dirs{service_name=~\"$server\"}, \"cluster\", \"$1\", \"snapshot_source\", \"[^@]*@([^:]+):.*\"))", "instant": false, - "legendFormat": "latest completion age", + "legendFormat": "{{cluster}} {{tier}}", "range": true } ], - "title": "Latest backup age (completion)", - "description": "Elapsed time since the most recently retained PostgreSQL backup completed.", + "title": "Latest snapshot dirs", + "description": "Number of directories in the most recent base backup snapshot on Kopia.", "transparent": false, "datasource": { "type": "prometheus", @@ -1403,7 +1810,7 @@ "h": 6, "w": 4, "x": 12, - "y": 32 + "y": 64 }, "repeatDirection": "h", "options": { @@ -1423,11 +1830,12 @@ "valueSize": 28 }, "percentChangeColorMode": "standard", - "orientation": "" + "orientation": "horizontal" }, "fieldConfig": { "defaults": { - "unit": "dtdurations", + "unit": "none", + "decimals": 0, "color": { "mode": "palette-classic" } @@ -1443,14 +1851,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "time() - min(klio_server_backup_oldest_backup_start_time_seconds{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\",cluster_name=~\"$cluster\"})", + "expr": "time() - max by (cluster, tier) (label_replace(klio_server_backup_latest_snapshot_timestamp_seconds{service_name=~\"$server\"}, \"cluster\", \"$1\", \"snapshot_source\", \"[^@]*@([^:]+):.*\"))", "instant": false, - "legendFormat": "oldest start age", + "legendFormat": "{{cluster}} {{tier}}", "range": true } ], - "title": "Oldest backup age (start)", - "description": "Elapsed time since the oldest retained PostgreSQL backup started, reflecting the effective retention horizon.", + "title": "Latest snapshot age", + "description": "Age of the most recent base backup snapshot on Kopia, per cluster and tier.", "transparent": false, "datasource": { "type": "prometheus", @@ -1460,7 +1868,7 @@ "h": 6, "w": 4, "x": 16, - "y": 32 + "y": 64 }, "repeatDirection": "h", "options": { @@ -1480,11 +1888,11 @@ "valueSize": 28 }, "percentChangeColorMode": "standard", - "orientation": "" + "orientation": "horizontal" }, "fieldConfig": { "defaults": { - "unit": "dtdurations", + "unit": "dtdhms", "color": { "mode": "palette-classic" } @@ -1500,14 +1908,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "time() - min(klio_server_backup_oldest_backup_completion_time_seconds{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\",cluster_name=~\"$cluster\"})", + "expr": "time() - min by (cluster, tier) (label_replace(klio_server_backup_oldest_snapshot_timestamp_seconds{service_name=~\"$server\"}, \"cluster\", \"$1\", \"snapshot_source\", \"[^@]*@([^:]+):.*\"))", "instant": false, - "legendFormat": "oldest completion age", + "legendFormat": "{{cluster}} {{tier}}", "range": true } ], - "title": "Oldest backup age (completion)", - "description": "Elapsed time since the oldest retained PostgreSQL backup completed, reflecting the effective retention horizon.", + "title": "Oldest snapshot age", + "description": "Age of the oldest retained base backup snapshot on Kopia, per cluster and tier.", "transparent": false, "datasource": { "type": "prometheus", @@ -1517,7 +1925,7 @@ "h": 6, "w": 4, "x": 20, - "y": 32 + "y": 64 }, "repeatDirection": "h", "options": { @@ -1536,35 +1944,164 @@ "titleSize": 14, "valueSize": 28 }, - "percentChangeColorMode": "standard", - "orientation": "" + "percentChangeColorMode": "standard", + "orientation": "horizontal" + }, + "fieldConfig": { + "defaults": { + "unit": "dtdhms", + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "max by (cluster_name, tier) (klio_server_wal_latest_written_timeline{service_name=~\"$server\",cluster_name=~\"$cluster\"})", + "instant": false, + "legendFormat": "{{cluster_name}} {{tier}}", + "range": true + } + ], + "title": "WAL timeline", + "description": "PostgreSQL timeline of the latest WAL written per cluster and tier", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 70 + }, + "repeatDirection": "h", + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "", + "sort": "" + } + }, + "fieldConfig": { + "defaults": { + "unit": "none", + "decimals": 0, + "custom": { + "gradientMode": "opacity", + "lineWidth": 2, + "lineInterpolation": "stepAfter", + "fillOpacity": 0 + } + }, + "overrides": [] + } + }, + { + "type": "timeseries", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "time() - max by (cluster_name, tier) (klio_server_wal_latest_written_time_seconds{service_name=~\"$server\",cluster_name=~\"$cluster\"})", + "instant": false, + "legendFormat": "{{cluster_name}} {{tier}}", + "range": true + } + ], + "title": "Time since last WAL written", + "description": "Elapsed time since the server last wrote a WAL file for each cluster and tier.", + "transparent": false, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 70 + }, + "repeatDirection": "h", + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "", + "sort": "" + } }, "fieldConfig": { "defaults": { "unit": "dtdurations", - "color": { - "mode": "palette-classic" + "custom": { + "gradientMode": "opacity", + "fillOpacity": 10 } }, "overrides": [] } }, { - "type": "bargauge", + "type": "table", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum by (tier) (klio_server_backup_backups{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\",cluster_name=~\"$cluster\"})", - "instant": false, - "legendFormat": "{{tier}}", - "range": true + "expr": "floor(max by (cluster_name, tier) (klio_server_wal_latest_written_lsn_bytes{service_name=~\"$server\",cluster_name=~\"$cluster\"}) / 4294967296)", + "format": "table", + "hide": true, + "instant": true, + "range": false, + "refId": "h0" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "max by (cluster_name, tier) (klio_server_wal_latest_written_lsn_bytes{service_name=~\"$server\",cluster_name=~\"$cluster\"}) % 4294967296", + "format": "table", + "hide": true, + "instant": true, + "range": false, + "refId": "l0" + }, + { + "datasource": { + "type": "__expr__", + "uid": "__expr__" + }, + "expression": "SELECT\n h0.cluster_name AS \"Cluster Name\",\n h0.tier AS \"Tier\",\n concat(CONV(h0.__value__, 10, 16), \"/\", CONV(l0.__value__, 10, 16)) as \"written LSN\"\nFROM h0\nJOIN l0 ON l0.cluster_name = h0.cluster_name AND l0.tier = h0.tier", + "hide": false, + "refId": "sql", + "type": "sql" } ], - "title": "PostgreSQL backups retained by tier", - "description": "Number of PostgreSQL backups currently retained per tier, across clusters.", + "title": "Latest written LSN", + "description": "Most recent WAL LSN the server has written for each cluster and tier", "transparent": false, "datasource": { "type": "prometheus", @@ -1573,36 +2110,27 @@ "gridPos": { "h": 6, "w": 8, - "x": 0, - "y": 38 + "x": 16, + "y": 70 }, "repeatDirection": "h", "options": { - "displayMode": "gradient", - "valueMode": "color", - "namePlacement": "auto", - "showUnfilled": true, - "sizing": "auto", - "minVizWidth": 8, - "minVizHeight": 16, - "legend": { - "displayMode": "list", - "placement": "bottom", - "showLegend": false, - "calcs": [] - }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] + "frameIndex": 0, + "showHeader": true, + "showTypeIcons": false, + "footer": { + "show": false, + "reducer": null, + "countRows": false }, - "maxVizHeight": 300, - "orientation": "horizontal" + "cellHeight": "sm" }, "fieldConfig": { "defaults": { - "unit": "none", - "decimals": 0 + "custom": { + "align": "auto", + "inspect": false + } }, "overrides": [] } @@ -1615,24 +2143,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "max by (cluster_name) (klio_server_backup_latest_backup_start_lsn_bytes{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\",cluster_name=~\"$cluster\"})", - "instant": false, - "legendFormat": "{{cluster_name}} start", - "range": true - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "expr": "max by (cluster_name) (klio_server_backup_latest_backup_end_lsn_bytes{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\",cluster_name=~\"$cluster\"})", + "expr": "sum by (cluster_name, tier) (rate(klio_server_wal_written_total{service_name=~\"$server\",cluster_name=~\"$cluster\"}[$__rate_interval]))", "instant": false, - "legendFormat": "{{cluster_name}} end", + "legendFormat": "{{cluster_name}} {{tier}}", "range": true } ], - "title": "Latest backup LSN by cluster", - "description": "Start and end LSN of the latest retained PostgreSQL backup, per cluster.", + "title": "WAL files written (rate)", + "description": "Rate of WAL files written by the server, split by cluster and storage tier.", "transparent": false, "datasource": { "type": "prometheus", @@ -1640,9 +2158,9 @@ }, "gridPos": { "h": 6, - "w": 8, - "x": 8, - "y": 38 + "w": 12, + "x": 0, + "y": 76 }, "repeatDirection": "h", "options": { @@ -1659,7 +2177,7 @@ }, "fieldConfig": { "defaults": { - "unit": "bytes", + "unit": "ops", "custom": { "gradientMode": "opacity", "fillOpacity": 10 @@ -1676,24 +2194,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "max by (cluster_name) (klio_server_backup_oldest_backup_start_lsn_bytes{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\",cluster_name=~\"$cluster\"})", - "instant": false, - "legendFormat": "{{cluster_name}} start", - "range": true - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "expr": "max by (cluster_name) (klio_server_backup_oldest_backup_end_lsn_bytes{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\",cluster_name=~\"$cluster\"})", + "expr": "sum by (cluster_name, tier) (klio_server_wal_written_total{service_name=~\"$server\",cluster_name=~\"$cluster\"})", "instant": false, - "legendFormat": "{{cluster_name}} end", + "legendFormat": "{{cluster_name}} {{tier}}", "range": true } ], - "title": "Oldest backup LSN by cluster", - "description": "Start and end LSN of the oldest retained PostgreSQL backup, per cluster.", + "title": "WAL files written (total)", + "description": "Total WAL files written by the server since it last restarted, split by cluster and storage tier.", "transparent": false, "datasource": { "type": "prometheus", @@ -1701,9 +2209,9 @@ }, "gridPos": { "h": 6, - "w": 8, - "x": 16, - "y": 38 + "w": 12, + "x": 12, + "y": 76 }, "repeatDirection": "h", "options": { @@ -1720,7 +2228,7 @@ }, "fieldConfig": { "defaults": { - "unit": "bytes", + "unit": "none", "custom": { "gradientMode": "opacity", "fillOpacity": 10 @@ -1730,31 +2238,21 @@ } }, { - "type": "bargauge", + "type": "timeseries", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "max by (cluster_name) (klio_server_backup_latest_backup_timeline{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\",cluster_name=~\"$cluster\"})", - "instant": false, - "legendFormat": "{{cluster_name}} latest", - "range": true - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "expr": "max by (cluster_name) (klio_server_backup_oldest_backup_timeline{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\",cluster_name=~\"$cluster\"})", + "expr": "sum by (cluster_name, tier) (rate(klio_server_wal_written_size_bytes_total{service_name=~\"$server\",cluster_name=~\"$cluster\"}[$__rate_interval]))", "instant": false, - "legendFormat": "{{cluster_name}} oldest", + "legendFormat": "{{cluster_name}} {{tier}}", "range": true } ], - "title": "Backup timeline by cluster", - "description": "PostgreSQL timeline of the latest and oldest retained backup, per cluster. A mismatch means the retention window spans a promotion or failover.", + "title": "WAL bytes written (rate)", + "description": "Rate of WAL bytes written by the server, split by cluster and storage tier.", "transparent": false, "datasource": { "type": "prometheus", @@ -1762,37 +2260,30 @@ }, "gridPos": { "h": 6, - "w": 8, + "w": 12, "x": 0, - "y": 44 + "y": 82 }, "repeatDirection": "h", "options": { - "displayMode": "gradient", - "valueMode": "color", - "namePlacement": "auto", - "showUnfilled": true, - "sizing": "auto", - "minVizWidth": 8, - "minVizHeight": 16, "legend": { "displayMode": "list", "placement": "bottom", - "showLegend": false, + "showLegend": true, "calcs": [] }, - "reduceOptions": { - "calcs": [ - "lastNotNull" - ] - }, - "maxVizHeight": 300, - "orientation": "horizontal" + "tooltip": { + "mode": "", + "sort": "" + } }, "fieldConfig": { "defaults": { - "unit": "none", - "decimals": 0 + "unit": "binBps", + "custom": { + "gradientMode": "opacity", + "fillOpacity": 10 + } }, "overrides": [] } @@ -1805,14 +2296,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum by (tier) (rate(klio_server_wal_written_total{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\",cluster_name=~\"$cluster\"}[$__rate_interval]))", + "expr": "sum by (cluster_name, tier) (klio_server_wal_written_size_bytes_total{service_name=~\"$server\",cluster_name=~\"$cluster\"})", "instant": false, - "legendFormat": "{{tier}}", + "legendFormat": "{{cluster_name}} {{tier}}", "range": true } ], - "title": "WAL files written rate by tier", - "description": "Rate of WAL files written by the server, split by storage tier.", + "title": "WAL bytes written (total)", + "description": "Total WAL bytes written by the server since it last restarted, split by cluster and storage tier.", "transparent": false, "datasource": { "type": "prometheus", @@ -1820,9 +2311,9 @@ }, "gridPos": { "h": 6, - "w": 8, - "x": 8, - "y": 44 + "w": 12, + "x": 12, + "y": 82 }, "repeatDirection": "h", "options": { @@ -1839,7 +2330,7 @@ }, "fieldConfig": { "defaults": { - "unit": "wps", + "unit": "bytes", "custom": { "gradientMode": "opacity", "fillOpacity": 10 @@ -1856,24 +2347,44 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum by (tier) (rate(klio_server_wal_written_size_bytes_total{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\",cluster_name=~\"$cluster\"}[$__rate_interval]))", + "expr": "histogram_quantile(0.50, sum by (le, path, stage) (rate(klio_server_wal_block_duration_nanoseconds_bucket{service_name=~\"$server\",cluster_name=~\"$cluster\"}[$__rate_interval])))", + "instant": false, + "legendFormat": "{{path}}/{{stage}} p50", + "range": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.90, sum by (le, path, stage) (rate(klio_server_wal_block_duration_nanoseconds_bucket{service_name=~\"$server\",cluster_name=~\"$cluster\"}[$__rate_interval])))", + "instant": false, + "legendFormat": "{{path}}/{{stage}} p90", + "range": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.99, sum by (le, path, stage) (rate(klio_server_wal_block_duration_nanoseconds_bucket{service_name=~\"$server\",cluster_name=~\"$cluster\"}[$__rate_interval])))", "instant": false, - "legendFormat": "{{tier}}", + "legendFormat": "{{path}}/{{stage}} p99", "range": true } ], - "title": "WAL bytes written rate by tier", - "description": "Rate of WAL bytes written by the server, split by storage tier.", + "title": "WAL block operation duration percentiles (rate)", + "description": "Percentile per-block WAL processing duration on the server, split by `path` (put ingest / get serve) and `stage`, aggregated across the selected clusters. Reflects recent activity, over the rate interval window.", "transparent": false, "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 44 + "h": 8, + "w": 12, + "x": 0, + "y": 88 }, "repeatDirection": "h", "options": { @@ -1890,7 +2401,7 @@ }, "fieldConfig": { "defaults": { - "unit": "Bps", + "unit": "ns", "custom": { "gradientMode": "opacity", "fillOpacity": 10 @@ -1907,24 +2418,44 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "time() - max by (tier) (klio_server_wal_latest_written_time_seconds{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\",cluster_name=~\"$cluster\"})", + "expr": "histogram_quantile(0.50, sum by (le, path, stage) (klio_server_wal_block_duration_nanoseconds_bucket{service_name=~\"$server\",cluster_name=~\"$cluster\"}))", + "instant": false, + "legendFormat": "{{path}}/{{stage}} p50", + "range": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.90, sum by (le, path, stage) (klio_server_wal_block_duration_nanoseconds_bucket{service_name=~\"$server\",cluster_name=~\"$cluster\"}))", + "instant": false, + "legendFormat": "{{path}}/{{stage}} p90", + "range": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.99, sum by (le, path, stage) (klio_server_wal_block_duration_nanoseconds_bucket{service_name=~\"$server\",cluster_name=~\"$cluster\"}))", "instant": false, - "legendFormat": "{{tier}}", + "legendFormat": "{{path}}/{{stage}} p99", "range": true } ], - "title": "Time since last WAL written by tier", - "description": "Elapsed time since the server last wrote a WAL file for each tier. A stale tier-1 value means PostgreSQL stopped shipping WALs; a stale tier-2 value means the remote backend stopped receiving them.", + "title": "WAL block operation duration percentiles (total)", + "description": "Percentile per-block WAL processing duration on the server, split by `path` (put ingest / get serve) and `stage`, aggregated across the selected clusters. Reflects all activity since the server last restarted.", "transparent": false, "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 50 + "h": 8, + "w": 12, + "x": 12, + "y": 88 }, "repeatDirection": "h", "options": { @@ -1941,7 +2472,7 @@ }, "fieldConfig": { "defaults": { - "unit": "dtdurations", + "unit": "ns", "custom": { "gradientMode": "opacity", "fillOpacity": 10 @@ -1958,24 +2489,44 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "max by (tier) (klio_server_wal_latest_written_lsn_bytes{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\",cluster_name=~\"$cluster\"})", + "expr": "histogram_quantile(0.50, sum by (le, tier) (rate(klio_server_wal_get_duration_nanoseconds_bucket{service_name=~\"$server\",cluster_name=~\"$cluster\"}[$__rate_interval])))", + "instant": false, + "legendFormat": "{{tier}} p50", + "range": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.90, sum by (le, tier) (rate(klio_server_wal_get_duration_nanoseconds_bucket{service_name=~\"$server\",cluster_name=~\"$cluster\"}[$__rate_interval])))", + "instant": false, + "legendFormat": "{{tier}} p90", + "range": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.99, sum by (le, tier) (rate(klio_server_wal_get_duration_nanoseconds_bucket{service_name=~\"$server\",cluster_name=~\"$cluster\"}[$__rate_interval])))", "instant": false, - "legendFormat": "{{tier}}", + "legendFormat": "{{tier}} p99", "range": true } ], - "title": "Latest written LSN by tier", - "description": "Most recent WAL LSN the server has written for each tier, as a byte offset.", + "title": "WAL file get duration percentiles (rate)", + "description": "Percentile duration of a complete WAL file gRPC get, split by the tier that served it, aggregated across the selected clusters. Reflects recent activity, over the rate interval window.", "transparent": false, "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 50 + "h": 8, + "w": 12, + "x": 0, + "y": 96 }, "repeatDirection": "h", "options": { @@ -1992,7 +2543,7 @@ }, "fieldConfig": { "defaults": { - "unit": "bytes", + "unit": "ns", "custom": { "gradientMode": "opacity", "fillOpacity": 10 @@ -2009,24 +2560,44 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum by (outcome, tier) (rate(klio_server_backup_verifications_total{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\"}[$__rate_interval]))", + "expr": "histogram_quantile(0.50, sum by (le, tier) (klio_server_wal_get_duration_nanoseconds_bucket{service_name=~\"$server\",cluster_name=~\"$cluster\"}))", + "instant": false, + "legendFormat": "{{tier}} p50", + "range": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.90, sum by (le, tier) (klio_server_wal_get_duration_nanoseconds_bucket{service_name=~\"$server\",cluster_name=~\"$cluster\"}))", + "instant": false, + "legendFormat": "{{tier}} p90", + "range": true + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "histogram_quantile(0.99, sum by (le, tier) (klio_server_wal_get_duration_nanoseconds_bucket{service_name=~\"$server\",cluster_name=~\"$cluster\"}))", "instant": false, - "legendFormat": "{{tier}} / {{outcome}}", + "legendFormat": "{{tier}} p99", "range": true } ], - "title": "Backup verification rate by outcome and tier", - "description": "Rate of base backup verification checks, broken down by outcome and tier.", + "title": "WAL file get duration percentiles (total)", + "description": "Percentile duration of a complete WAL file gRPC get, split by the tier that served it, aggregated across the selected clusters. Reflects all activity since the server last restarted.", "transparent": false, "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 50 + "h": 8, + "w": 12, + "x": 12, + "y": 96 }, "repeatDirection": "h", "options": { @@ -2043,7 +2614,7 @@ }, "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "ns", "custom": { "gradientMode": "opacity", "fillOpacity": 10 @@ -2060,9 +2631,9 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "histogram_quantile(0.50, sum by (le, path, stage) (rate(klio_server_wal_block_duration_nanoseconds_bucket{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\",cluster_name=~\"$cluster\"}[$__rate_interval])))", + "expr": "histogram_quantile(0.50, sum by (le, cluster_name) (rate(klio_server_wal_upload_duration_nanoseconds_bucket{service_name=~\"$server\",cluster_name=~\"$cluster\"}[$__rate_interval])))", "instant": false, - "legendFormat": "p50 {{path}}/{{stage}}", + "legendFormat": "{{cluster_name}} p50", "range": true }, { @@ -2070,9 +2641,9 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "histogram_quantile(0.95, sum by (le, path, stage) (rate(klio_server_wal_block_duration_nanoseconds_bucket{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\",cluster_name=~\"$cluster\"}[$__rate_interval])))", + "expr": "histogram_quantile(0.90, sum by (le, cluster_name) (rate(klio_server_wal_upload_duration_nanoseconds_bucket{service_name=~\"$server\",cluster_name=~\"$cluster\"}[$__rate_interval])))", "instant": false, - "legendFormat": "p95 {{path}}/{{stage}}", + "legendFormat": "{{cluster_name}} p90", "range": true }, { @@ -2080,24 +2651,24 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "histogram_quantile(0.99, sum by (le, path, stage) (rate(klio_server_wal_block_duration_nanoseconds_bucket{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\",cluster_name=~\"$cluster\"}[$__rate_interval])))", + "expr": "histogram_quantile(0.99, sum by (le, cluster_name) (rate(klio_server_wal_upload_duration_nanoseconds_bucket{service_name=~\"$server\",cluster_name=~\"$cluster\"}[$__rate_interval])))", "instant": false, - "legendFormat": "p99 {{path}}/{{stage}}", + "legendFormat": "{{cluster_name}} p99", "range": true } ], - "title": "WAL block duration (p50/p95/p99) by path and stage", - "description": "Percentile per-block WAL processing duration on the server, split by `path` (put ingest / get serve) and `stage`.", + "title": "WAL tier-2 upload duration percentiles (rate)", + "description": "Percentile duration of the tier-2 archival upload to remote storage, per cluster. Reflects recent activity, over the rate interval window.", "transparent": false, "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { - "h": 6, - "w": 8, + "h": 8, + "w": 12, "x": 0, - "y": 56 + "y": 104 }, "repeatDirection": "h", "options": { @@ -2131,9 +2702,9 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "histogram_quantile(0.50, sum by (le, tier) (rate(klio_server_wal_get_duration_nanoseconds_bucket{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\",cluster_name=~\"$cluster\"}[$__rate_interval])))", + "expr": "histogram_quantile(0.50, sum by (le, cluster_name) (klio_server_wal_upload_duration_nanoseconds_bucket{service_name=~\"$server\",cluster_name=~\"$cluster\"}))", "instant": false, - "legendFormat": "p50 {{tier}}", + "legendFormat": "{{cluster_name}} p50", "range": true }, { @@ -2141,9 +2712,9 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "histogram_quantile(0.95, sum by (le, tier) (rate(klio_server_wal_get_duration_nanoseconds_bucket{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\",cluster_name=~\"$cluster\"}[$__rate_interval])))", + "expr": "histogram_quantile(0.90, sum by (le, cluster_name) (klio_server_wal_upload_duration_nanoseconds_bucket{service_name=~\"$server\",cluster_name=~\"$cluster\"}))", "instant": false, - "legendFormat": "p95 {{tier}}", + "legendFormat": "{{cluster_name}} p90", "range": true }, { @@ -2151,24 +2722,24 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "histogram_quantile(0.99, sum by (le, tier) (rate(klio_server_wal_get_duration_nanoseconds_bucket{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\",cluster_name=~\"$cluster\"}[$__rate_interval])))", + "expr": "histogram_quantile(0.99, sum by (le, cluster_name) (klio_server_wal_upload_duration_nanoseconds_bucket{service_name=~\"$server\",cluster_name=~\"$cluster\"}))", "instant": false, - "legendFormat": "p99 {{tier}}", + "legendFormat": "{{cluster_name}} p99", "range": true } ], - "title": "WAL file get duration (p50/p95/p99) by tier", - "description": "Percentile duration of a complete WAL file gRPC get, split by the tier that served it.", + "title": "WAL tier-2 upload duration percentiles (total)", + "description": "Percentile duration of the tier-2 archival upload to remote storage, per cluster. Reflects all activity since the server last restarted.", "transparent": false, "datasource": { "type": "prometheus", "uid": "${datasource}" }, "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 56 + "h": 8, + "w": 12, + "x": 12, + "y": 104 }, "repeatDirection": "h", "options": { @@ -2202,34 +2773,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "histogram_quantile(0.50, sum by (le, cluster_name) (rate(klio_server_wal_upload_duration_nanoseconds_bucket{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\",cluster_name=~\"$cluster\"}[$__rate_interval])))", - "instant": false, - "legendFormat": "p50 {{cluster_name}}", - "range": true - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "expr": "histogram_quantile(0.95, sum by (le, cluster_name) (rate(klio_server_wal_upload_duration_nanoseconds_bucket{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\",cluster_name=~\"$cluster\"}[$__rate_interval])))", - "instant": false, - "legendFormat": "p95 {{cluster_name}}", - "range": true - }, - { - "datasource": { - "type": "prometheus", - "uid": "${datasource}" - }, - "expr": "histogram_quantile(0.99, sum by (le, cluster_name) (rate(klio_server_wal_upload_duration_nanoseconds_bucket{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\",cluster_name=~\"$cluster\"}[$__rate_interval])))", + "expr": "sum by (cluster_name, outcome) (klio_server_backup_relay_total{service_name=~\"$server\",cluster_name=~\"$cluster\"})", "instant": false, - "legendFormat": "p99 {{cluster_name}}", + "legendFormat": "{{cluster_name}} / {{outcome}}", "range": true } ], - "title": "WAL tier-2 upload duration (p50/p95/p99) by cluster", - "description": "Percentile duration of the tier-2 archival upload to remote storage, per cluster.", + "title": "Tier-2 backup relay attempts", + "description": "Total tier-2 relay attempts since the server last restarted, per cluster and outcome.", "transparent": false, "datasource": { "type": "prometheus", @@ -2238,8 +2789,8 @@ "gridPos": { "h": 6, "w": 8, - "x": 16, - "y": 56 + "x": 0, + "y": 112 }, "repeatDirection": "h", "options": { @@ -2256,7 +2807,7 @@ }, "fieldConfig": { "defaults": { - "unit": "ns", + "unit": "none", "custom": { "gradientMode": "opacity", "fillOpacity": 10 @@ -2273,14 +2824,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum by (outcome) (rate(klio_server_backup_relay_total{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\",cluster_name=~\"$cluster\"}[$__rate_interval]))", + "expr": "sum by (cluster_name, outcome, tier) (klio_server_backup_verifications_total{service_name=~\"$server\",cluster_name=~\"$cluster\"})", "instant": false, - "legendFormat": "{{outcome}}", + "legendFormat": "{{cluster_name}} {{tier}} / {{outcome}}", "range": true } ], - "title": "Tier-2 relay rate by outcome", - "description": "Rate of tier-2 relay attempts after a backup (migration and verification), by outcome.", + "title": "Backup verification runs", + "description": "Total base backup verification checks since the server last restarted, per cluster, broken down by outcome and tier.", "transparent": false, "datasource": { "type": "prometheus", @@ -2288,9 +2839,9 @@ }, "gridPos": { "h": 6, - "w": 12, - "x": 0, - "y": 62 + "w": 8, + "x": 8, + "y": 112 }, "repeatDirection": "h", "options": { @@ -2307,7 +2858,7 @@ }, "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "none", "custom": { "gradientMode": "opacity", "fillOpacity": 10 @@ -2324,14 +2875,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum by (tier, outcome) (rate(klio_server_backup_maintenance_total{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\",cluster_name=~\"$cluster\"}[$__rate_interval]))", + "expr": "sum by (cluster_name, tier, outcome) (klio_server_backup_maintenance_total{service_name=~\"$server\",cluster_name=~\"$cluster\"})", "instant": false, - "legendFormat": "{{tier}} / {{outcome}}", + "legendFormat": "{{cluster_name}} {{tier}}/{{outcome}}", "range": true } ], - "title": "Maintenance run rate by tier and outcome", - "description": "Rate of post-backup maintenance runs (base-snapshot retention and WAL cleanup), by tier and outcome.", + "title": "Maintenance runs", + "description": "Total post-backup maintenance runs since the server last restarted, per cluster, tier and outcome.", "transparent": false, "datasource": { "type": "prometheus", @@ -2339,9 +2890,9 @@ }, "gridPos": { "h": 6, - "w": 12, - "x": 12, - "y": 62 + "w": 8, + "x": 16, + "y": 112 }, "repeatDirection": "h", "options": { @@ -2358,7 +2909,7 @@ }, "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "none", "custom": { "gradientMode": "opacity", "fillOpacity": 10 @@ -2375,14 +2926,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum by (stream) (klio_server_queue_messages{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\"})", + "expr": "sum by (service_name, stream) (klio_server_queue_messages{service_name=~\"$server\"})", "instant": false, - "legendFormat": "{{stream}}", + "legendFormat": "{{service_name}} / {{stream}}", "range": true } ], - "title": "Queue messages by stream", - "description": "Messages currently held in each NATS JetStream stream of the embedded queue.", + "title": "Queue size (messages)", + "description": "Messages currently held in each NATS JetStream stream of the embedded queue, per server.", "transparent": false, "datasource": { "type": "prometheus", @@ -2392,7 +2943,7 @@ "h": 6, "w": 12, "x": 0, - "y": 68 + "y": 118 }, "repeatDirection": "h", "options": { @@ -2426,14 +2977,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum by (stream) (klio_server_queue_bytes{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\"})", + "expr": "sum by (service_name, stream) (klio_server_queue_bytes{service_name=~\"$server\"})", "instant": false, - "legendFormat": "{{stream}}", + "legendFormat": "{{service_name}} / {{stream}}", "range": true } ], - "title": "Queue bytes by stream", - "description": "Bytes currently held in each NATS JetStream stream of the embedded queue.", + "title": "Queue size (bytes)", + "description": "Bytes currently held in each NATS JetStream stream of the embedded queue, per server.", "transparent": false, "datasource": { "type": "prometheus", @@ -2443,7 +2994,7 @@ "h": 6, "w": 12, "x": 12, - "y": 68 + "y": 118 }, "repeatDirection": "h", "options": { @@ -2472,19 +3023,44 @@ { "type": "row", "collapsed": false, - "title": "WAL Replication Lag", + "title": "WAL replication lag", "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 74 + "y": 124 }, "id": 0, "panels": [] }, + { + "type": "text", + "transparent": true, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 125 + }, + "repeatDirection": "h", + "options": { + "mode": "markdown", + "content": "How far the WAL stored in tier 1 and tier 2 trails behind the PostgreSQL primary." + } + }, { "type": "timeseries", "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "cnpg_pg_stat_replication_write_diff_bytes{namespace=~\"$namespace\",application_name=\"klio\"}", + "instant": false, + "legendFormat": "{{pod}} write", + "range": true + }, { "datasource": { "type": "prometheus", @@ -2492,12 +3068,12 @@ }, "expr": "cnpg_pg_stat_replication_flush_diff_bytes{namespace=~\"$namespace\",application_name=\"klio\"}", "instant": false, - "legendFormat": "{{pod}}", + "legendFormat": "{{pod}} flush", "range": true } ], - "title": "Tier-1 Replication lag (Bytes)", - "description": "Byte distance between the primary's current WAL LSN and the LSN Klio's streaming client has flushed, from CloudNativePG's pg_stat_replication.", + "title": "Tier-1 replication lag (bytes)", + "description": "Byte distance between the primary's current WAL LSN and the LSN Klio's streaming client has written to disk and flushed.", "transparent": false, "datasource": { "type": "prometheus", @@ -2507,7 +3083,7 @@ "h": 6, "w": 8, "x": 0, - "y": 75 + "y": 127 }, "repeatDirection": "h", "options": { @@ -2536,6 +3112,16 @@ { "type": "timeseries", "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "expr": "cnpg_pg_stat_replication_write_lag_seconds{namespace=~\"$namespace\",application_name=\"klio\"}", + "instant": false, + "legendFormat": "{{pod}} write", + "range": true + }, { "datasource": { "type": "prometheus", @@ -2543,12 +3129,12 @@ }, "expr": "cnpg_pg_stat_replication_flush_lag_seconds{namespace=~\"$namespace\",application_name=\"klio\"}", "instant": false, - "legendFormat": "{{pod}}", + "legendFormat": "{{pod}} flush", "range": true } ], - "title": "Tier-1 Replication lag (Seconds)", - "description": "Time between a commit on the primary and Klio's streaming client flushing the corresponding WAL, from CloudNativePG's pg_stat_replication.", + "title": "Tier-1 replication lag (seconds)", + "description": "Time between a commit on the primary and Klio's streaming client writing to disk and flushing the corresponding WAL.", "transparent": false, "datasource": { "type": "prometheus", @@ -2558,7 +3144,7 @@ "h": 6, "w": 8, "x": 8, - "y": 75 + "y": 127 }, "repeatDirection": "h", "options": { @@ -2592,14 +3178,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "(klio_server_wal_latest_written_lsn_bytes{tier=\"tier1\",k8s_namespace_name=~\"$namespace\",host_name=~\"$server\",cluster_name=~\"$cluster\"} - on (cluster_name) klio_server_wal_latest_written_lsn_bytes{tier=\"tier2\",k8s_namespace_name=~\"$namespace\",host_name=~\"$server\",cluster_name=~\"$cluster\"}) / 1024 / 1024", + "expr": "(max by (cluster_name) (klio_server_wal_latest_written_lsn_bytes{tier=\"tier1\",service_name=~\"$server\",cluster_name=~\"$cluster\"}) - max by (cluster_name) (klio_server_wal_latest_written_lsn_bytes{tier=\"tier2\",service_name=~\"$server\",cluster_name=~\"$cluster\"})) ", "instant": false, "legendFormat": "{{cluster_name}}", "range": true } ], - "title": "Tier-2 archival lag (Tier-1 LSN gap)", - "description": "LSN distance between tier 1 (local disk) and tier 2 (remote storage) per cluster. A growing gap means remote archival is falling behind.", + "title": "Tier-2 archival lag (bytes)", + "description": "Bytes difference between the WALs in tier 1 and tier 2 per cluster. ", "transparent": false, "datasource": { "type": "prometheus", @@ -2609,7 +3195,7 @@ "h": 6, "w": 8, "x": 16, - "y": 75 + "y": 127 }, "repeatDirection": "h", "options": { @@ -2626,7 +3212,7 @@ }, "fieldConfig": { "defaults": { - "unit": "mbytes", + "unit": "bytes", "custom": { "gradientMode": "opacity", "fillOpacity": 10 @@ -2685,7 +3271,7 @@ "name": "server", "label": "Server", "skipUrlSync": false, - "query": "label_values(klio_server_uptime_seconds{k8s_namespace_name=~\"$namespace\"}, host_name)", + "query": "label_values(klio_server_uptime_seconds, service_name)", "datasource": { "type": "prometheus", "uid": "${datasource}" @@ -2710,7 +3296,7 @@ "name": "cluster", "label": "Cluster", "skipUrlSync": false, - "query": "label_values(klio_server_wal_written_total{k8s_namespace_name=~\"$namespace\",host_name=~\"$server\"}, cluster_name)", + "query": "label_values(klio_server_wal_written_total{service_name=~\"$server\"}, cluster_name)", "datasource": { "type": "prometheus", "uid": "${datasource}" diff --git a/observability/grafana/main.go b/observability/grafana/main.go index 0f0e1fa0d..88ecf8bdb 100644 --- a/observability/grafana/main.go +++ b/observability/grafana/main.go @@ -38,37 +38,64 @@ import ( "os" "strings" - "github.com/grafana/grafana-foundation-sdk/go/bargauge" "github.com/grafana/grafana-foundation-sdk/go/cog" "github.com/grafana/grafana-foundation-sdk/go/cog/variants" "github.com/grafana/grafana-foundation-sdk/go/common" "github.com/grafana/grafana-foundation-sdk/go/dashboard" "github.com/grafana/grafana-foundation-sdk/go/prometheus" "github.com/grafana/grafana-foundation-sdk/go/stat" + "github.com/grafana/grafana-foundation-sdk/go/table" + "github.com/grafana/grafana-foundation-sdk/go/text" "github.com/grafana/grafana-foundation-sdk/go/timeseries" + "github.com/grafana/grafana-foundation-sdk/go/units" ) +// lsnHalf is 2^32: a PostgreSQL LSN is a 64-bit byte offset that PostgreSQL +// prints as two 32-bit halves in hexadecimal separated by a slash (e.g. +// `16/B374D848`). Dividing by lsnHalf yields the high half, the remainder is +// the low half. +const lsnHalf = 4294967296 + const ( // datasourceVar is the name of the Prometheus data source template // variable that every panel queries through, so the dashboard is portable // across Grafana installations. datasourceVar = "datasource" - // panelHeight is the shared grid height (in rows) of every panel, so the + // mediumPanelHeight is the shared grid height (in rows) of every panel, so the // dashboard grid packs flush without vertical gaps. - panelHeight = 6 - - // nsMatcher is the namespace label selector every klio metric carries - // (the Prometheus export of the k8s.namespace.name resource attribute). - nsMatcher = `k8s_namespace_name=~"$namespace"` - // serverMatcher additionally selects on host_name, which the klio_server_* - // series carry as the Klio server host (the StatefulSet pod), and combines - // it with the namespace selector. - serverMatcher = `k8s_namespace_name=~"$namespace",host_name=~"$server"` - // walMatcher additionally selects on cluster_name, which only the - // klio_server_wal_* series carry, and combines it with the namespace and - // server selectors. - walMatcher = `k8s_namespace_name=~"$namespace",host_name=~"$server",cluster_name=~"$cluster"` + mediumPanelHeight = 6 + largePanelHeight = 8 + // descriptionPanelHeight is the grid height (in rows) of the text panel + // that explains each row section. + descriptionPanelHeight = 2 + + gridWidth = 24 + smallestPanelWidth = 3 + smallPanelWidth = 4 + mediumPanelWidth = 6 + largePanelWidth = 8 + largestPanelWidth = 12 + + // clientMatcher selects the plugin sidecar's per-cluster series + // (klio_plugin_backup_* and klio_client_wal_*). These carry the PostgreSQL + // pod's k8s.namespace.name and a cluster_name, so they are scoped by + // $namespace and $cluster. Because they identify their cluster directly, + // several clusters sharing one namespace never fold into a single series. + clientMatcher = `k8s_namespace_name=~"$namespace",cluster_name=~"$cluster"` + // serverMatcher selects the server-level klio_server_* series that carry no + // cluster_name (uptime, the embedded queue, verifications and the Kopia base + // snapshots) by the Klio server's OpenTelemetry service.name. service.name + // identifies a server uniquely even when two servers share a pod host name + // (host_name collides when two Servers have the same name in different + // namespaces); k8s.namespace.name is deliberately NOT used here because a + // server tags every series with its OWN namespace, which would hide the + // metrics of a cluster it serves cross-namespace. + serverMatcher = `service_name=~"$server"` + // walMatcher additionally selects on cluster_name, which the per-cluster + // klio_server_wal_* and klio_server_backup_* (backups / latest_backup_* / + // oldest_backup_*) series carry, and combines it with the server selector. + walMatcher = `service_name=~"$server",cluster_name=~"$cluster"` ) func main() { @@ -103,7 +130,7 @@ type sizedPanel struct { place func(dashboard.GridPos) cog.Builder[dashboard.Panel] } -// sized pairs a panel builder (stat, timeseries, bargauge, ...) with its grid +// sized pairs a panel builder (stat, timeseries, ...) with its grid // width and height so layoutSection can assign it an explicit position. func sized[B interface { GridPos(gridPos dashboard.GridPos) B @@ -131,26 +158,99 @@ func query(expr, legend string) *prometheus.DataqueryBuilder { Range() } +// instantTableTarget builds an instant Prometheus query in table format, used +// by the LSN tables where only the current value per cluster/tier matters. +func instantTableTarget(expr, refID string) *prometheus.DataqueryBuilder { + return prometheus.NewDataqueryBuilder(). + Datasource(datasourceRef()). + Expr(expr). + Format(prometheus.PromQueryFormatTable). + Instant(). + RefId(refID) +} + +// lsnColumn is one LSN value rendered as a pair of hexadecimal columns (high +// and low 32-bit halves) in an lsnTablePanel. agg is the PromQL that reduces +// the LSN metric to one series per cluster_name and tier. +type lsnColumn struct { + label string + agg string +} + +// lsnTablePanel renders one or more PostgreSQL LSNs per cluster and tier as a +// table. Each 64-bit byte offset is split into its high and low 32-bit halves +// in PromQL (floor(lsn/2^32) and lsn%2^32, hidden targets h/l), then a +// Grafana SQL expression (the __expr__ datasource) joins the halves on +// cluster_name/tier and renders PostgreSQL's own X/Y hex notation as a single +// visible column per LSN. Neither PromQL nor a Grafana transform can do this +// join+format: PromQL has no decimal-to-hex formatting, and no transform +// combines two numeric fields into one templated string. __value__ is the +// numeric column exposed for an instant/table-format query result. +func lsnTablePanel(title string, cols ...lsnColumn) *table.PanelBuilder { + panel := table.NewPanelBuilder(). + Title(title). + Datasource(datasourceRef()). + Span(8). + Height(mediumPanelHeight) + + joins := []string{"JOIN l0 ON l0.cluster_name = h0.cluster_name AND l0.tier = h0.tier"} + columns := []string{ + `h0.cluster_name AS "Cluster Name"`, + `h0.tier AS "Tier"`, + fmt.Sprintf(`concat(CONV(h0.__value__, 10, 16), "/", CONV(l0.__value__, 10, 16)) as "%s LSN"`, + cols[0].label), + } + for i, c := range cols { + hiRef := fmt.Sprintf("h%d", i) + loRef := fmt.Sprintf("l%d", i) + panel = panel. + WithTarget(instantTableTarget(fmt.Sprintf("floor(%s / %d)", c.agg, lsnHalf), hiRef).Hide(true)). + WithTarget(instantTableTarget(fmt.Sprintf("%s %% %d", c.agg, lsnHalf), loRef).Hide(true)) + if i == 0 { + continue + } + joins = append(joins, + fmt.Sprintf("JOIN %s ON %s.cluster_name = h0.cluster_name AND %s.tier = h0.tier", hiRef, hiRef, hiRef), + fmt.Sprintf("JOIN %s ON %s.cluster_name = h0.cluster_name AND %s.tier = h0.tier", loRef, loRef, loRef)) + columns = append(columns, fmt.Sprintf( + `concat(CONV(%s.__value__, 10, 16), "/", CONV(%s.__value__, 10, 16)) as "%s LSN"`, + hiRef, loRef, c.label)) + } + sql := fmt.Sprintf("SELECT\n %s\nFROM h0\n%s", strings.Join(columns, ",\n "), strings.Join(joins, "\n")) + + return panel.WithTarget(variants.NewUnknownDataqueryBuilderFromObject(variants.UnknownDataquery{ + "refId": "sql", + "hide": false, + "datasource": map[string]any{"type": "__expr__", "uid": "__expr__"}, + "type": "sql", + "expression": sql, + })) +} + +// quantileSpec is one percentile rendered by a latency panel: the quantile +// value passed to histogram_quantile and the legend label for its series. +type quantileSpec struct { + q float64 + label string +} + // quantiles are the percentiles every latency panel renders together, so the -// median, the tail and the extreme tail always appear side by side rather than -// a single percentile hiding the shape of the distribution. +// median, the tail and the extreme tail always appear side by side rather +// than a single percentile hiding the shape of the distribution. // //nolint:gochecknoglobals -var quantiles = []struct { - q float64 - label string -}{ +var quantiles = []quantileSpec{ {0.50, "p50"}, - {0.95, "p95"}, + {0.90, "p90"}, {0.99, "p99"}, } -// quantileTargetsWindow builds one p50/p95/p99 target for a histogram, applying -// counterFn (`rate` or `increase`) over window (e.g. `$__rate_interval` or -// `$__range`). It groups the _bucket series by groupBy (which MUST include -// `le`, or histogram_quantile cannot find the bucket boundaries and the panel -// renders no data) and labels each series " ", dropping the -// trailing space when legend is empty. +// quantileTargetsWindow builds one target per entry in quantiles for a +// histogram, applying counterFn (`rate` or `increase`) over window (e.g. +// `$__rate_interval` or `$__range`). It groups the _bucket series by groupBy +// (which MUST include `le`, or histogram_quantile cannot find the bucket +// boundaries and the panel renders no data) and labels each series +// " ", dropping the leading space when legend is empty. func quantileTargetsWindow( bucketMetric, groupBy, matcher, legend, counterFn, window string, ) []cog.Builder[variants.Dataquery] { @@ -159,27 +259,34 @@ func quantileTargetsWindow( targets = append(targets, query( fmt.Sprintf("histogram_quantile(%.2f, sum by (%s) (%s(%s{%s}[%s])))", p.q, groupBy, counterFn, bucketMetric, matcher, window), - strings.TrimSpace(p.label+" "+legend))) + strings.TrimSpace(legend+" "+p.label))) } return targets } -// quantileTargets builds p50/p95/p99 targets for a high-frequency histogram, -// using rate() over $__rate_interval so the percentiles track the dashboard's -// selected range and zoom. +// quantileTargets builds one target per entry in quantiles for a +// high-frequency histogram, using rate() over $__rate_interval so the +// percentiles track the dashboard's selected range and zoom. func quantileTargets(bucketMetric, groupBy, matcher, legend string) []cog.Builder[variants.Dataquery] { return quantileTargetsWindow(bucketMetric, groupBy, matcher, legend, "rate", "$__rate_interval") } -// quantileTargetsRange builds p50/p95/p99 targets for an infrequent-event -// histogram (e.g. backups), using increase() over the whole visible range -// ($__range). A short rate() window almost never catches a rare event, so the -// percentiles would otherwise collapse to no data except at the instant the -// event fires; the range window keeps them populated whenever the selected -// range spans at least one event. -func quantileTargetsRange(bucketMetric, groupBy, matcher, legend string) []cog.Builder[variants.Dataquery] { - return quantileTargetsWindow(bucketMetric, groupBy, matcher, legend, "increase", "$__range") +// quantileTargetsAbsolute builds one target per entry in quantiles straight +// off the raw cumulative bucket counters (no rate()/increase()), so a single +// observation still yields a value instead of NaN. The tradeoff: each point +// is a since-restart quantile that dilutes as more observations accrue and +// resets on a server restart, rather than one scoped to the dashboard's +// selected range or zoom. +func quantileTargetsAbsolute(bucketMetric, groupBy, matcher, legend string) []cog.Builder[variants.Dataquery] { + targets := make([]cog.Builder[variants.Dataquery], 0, len(quantiles)) + for _, p := range quantiles { + targets = append(targets, query( + fmt.Sprintf("histogram_quantile(%.2f, sum by (%s) (%s{%s}))", p.q, groupBy, bucketMetric, matcher), + strings.TrimSpace(legend+" "+p.label))) + } + + return targets } // tableLegend renders a compact table legend at the bottom of a panel. @@ -201,11 +308,11 @@ func timeseriesPanel(title, unit string, targets ...cog.Builder[variants.Dataque GradientMode(common.GraphGradientModeOpacity). Legend(tableLegend()). // 8/24 columns => three timeseries per row for a dense layout. - Span(8). + Span(largePanelWidth). // Uniform height across stat and timeseries panels so the grid packs // flush with no vertical gaps (the auto-layout starts each new row below // the tallest panel of the previous one). - Height(panelHeight) + Height(mediumPanelHeight) for _, target := range targets { panel = panel.WithTarget(target) } @@ -213,34 +320,25 @@ func timeseriesPanel(title, unit string, targets ...cog.Builder[variants.Dataque return panel } -// barPanel builds a stacked bar chart (a timeseries in bar draw style) for -// counting discrete events over time, such as backups per bucket. -func barPanel(title, unit string, targets ...cog.Builder[variants.Dataquery]) *timeseries.PanelBuilder { - return timeseriesPanel(title, unit, targets...). - DrawStyle(common.GraphDrawStyleBars). - FillOpacity(80). - Stacking(common.NewStackingConfigBuilder().Mode(common.StackingModeNormal)). +// timelinePanel renders an integer PostgreSQL timeline ID over time as a +// stepped line, so an operator sees not only the current timeline but exactly +// when it changed (a promotion or failover), which a single current value (a +// stat or bar gauge) cannot convey. The unit is a plain count with no decimals. +func timelinePanel(title string, targets ...cog.Builder[variants.Dataquery]) *timeseries.PanelBuilder { + return timeseriesPanel(title, units.Number, targets...). + FillOpacity(0). + LineInterpolation(common.LineInterpolationStepAfter). + LineWidth(2). Decimals(0) } -// barGaugePanel builds a horizontal bar gauge that renders one labeled bar per -// series, so multi-series "by tier / by stream" values always show every series -// (a narrow stat tile can hide all but the first). Every caller renders a -// label-only classification (tier, cluster, stream), so the unit is always -// "none". -func barGaugePanel(title string, targets ...cog.Builder[variants.Dataquery]) *bargauge.PanelBuilder { - panel := bargauge.NewPanelBuilder(). - Title(title). - Datasource(datasourceRef()). - Unit("none"). - ReduceOptions(common.NewReduceDataOptionsBuilder().Calcs([]string{"lastNotNull"})). - Orientation(common.VizOrientationHorizontal). - Decimals(0) - for _, target := range targets { - panel = panel.WithTarget(target) - } - - return panel +// descriptionPanel builds a transparent markdown text panel that explains a +// row section (a Grafana row has no description field of its own). +func descriptionPanel(content string) *text.PanelBuilder { + return text.NewPanelBuilder(). + Transparent(true). + Mode(text.TextModeMarkdown). + Content(content) } // statPanel builds a stat panel showing the last value of its query targets, @@ -262,8 +360,8 @@ func statPanel(title, unit string, targets ...cog.Builder[variants.Dataquery]) * // overflow and clip in the dense tiles. Text(common.NewVizTextDisplayOptionsBuilder().TitleSize(14).ValueSize(28)). // 4/24 columns => six stat tiles per row for a dense layout. - Span(4). - Height(panelHeight) + Span(smallPanelWidth). + Height(mediumPanelHeight) for _, target := range targets { panel = panel.WithTarget(target) } diff --git a/observability/grafana/replication.go b/observability/grafana/replication.go index 42f9ccd33..0e423fa98 100644 --- a/observability/grafana/replication.go +++ b/observability/grafana/replication.go @@ -21,6 +21,8 @@ package main import ( "fmt" + + "github.com/grafana/grafana-foundation-sdk/go/units" ) // cnpgMatcher selects the CloudNativePG replication metrics for Klio's WAL @@ -37,38 +39,36 @@ const cnpgMatcher = `namespace=~"$namespace",application_name="klio"` // scraped into the same Prometheus as Klio's metrics. func replicationPanels() []sizedPanel { return []sizedPanel{ - // WAL lag as the byte distance between the primary's current LSN and - // the LSN Klio's streaming client has flushed. Reported in bytes; - // Grafana scales the unit for display. - sized(8, panelHeight, timeseriesPanel("Tier-1 Replication lag (Bytes)", "bytes", + sized(gridWidth, descriptionPanelHeight, descriptionPanel( + "How far the WAL stored in tier 1 and tier 2 trails behind the PostgreSQL primary.")), + sized(largePanelWidth, mediumPanelHeight, timeseriesPanel("Tier-1 replication lag (bytes)", units.BytesIEC, + query( + fmt.Sprintf("cnpg_pg_stat_replication_write_diff_bytes{%s}", cnpgMatcher), + "{{pod}} write"), query( fmt.Sprintf("cnpg_pg_stat_replication_flush_diff_bytes{%s}", cnpgMatcher), - "{{pod}}"), + "{{pod}} flush"), ).Description("Byte distance between the primary's current WAL LSN and the LSN Klio's streaming "+ - "client has flushed, from CloudNativePG's pg_stat_replication.")), - // Flush lag in seconds: time between a commit on the primary and Klio's - // client flushing the corresponding WAL. - sized(8, panelHeight, timeseriesPanel("Tier-1 Replication lag (Seconds)", "s", + "client has written to disk and flushed.")), + + sized(largePanelWidth, mediumPanelHeight, timeseriesPanel("Tier-1 replication lag (seconds)", units.Seconds, + query( + fmt.Sprintf("cnpg_pg_stat_replication_write_lag_seconds{%s}", cnpgMatcher), + "{{pod}} write"), query( fmt.Sprintf("cnpg_pg_stat_replication_flush_lag_seconds{%s}", cnpgMatcher), - "{{pod}}"), - ).Description("Time between a commit on the primary and Klio's streaming client flushing the "+ - "corresponding WAL, from CloudNativePG's pg_stat_replication.")), - // Derived: tier-2 archival backlog as the LSN gap between what the - // server has on local disk (tier1) and what has been archived remotely - // (tier2), in MiB. max by (cluster_name) collapses each tier to one - // series per cluster so the subtraction is one-to-one even when several - // namespaces or server instances export the metric. - sized(8, panelHeight, timeseriesPanel("Tier-2 archival lag (Tier-1 LSN gap)", "mbytes", + "{{pod}} flush"), + ).Description("Time between a commit on the primary and Klio's streaming client writing to disk "+ + "and flushing the corresponding WAL.")), + + sized(largePanelWidth, mediumPanelHeight, timeseriesPanel("Tier-2 archival lag (bytes)", units.BytesIEC, query( fmt.Sprintf( - "(klio_server_wal_latest_written_lsn_bytes{tier=\"tier1\",%s} - "+ - "on (cluster_name) klio_server_wal_latest_written_lsn_bytes{tier=\"tier2\",%s}) "+ - "/ 1024 / 1024", + "(max by (cluster_name) (klio_server_wal_latest_written_lsn_bytes{tier=\"tier1\",%s}) - "+ + "max by (cluster_name) (klio_server_wal_latest_written_lsn_bytes{tier=\"tier2\",%s})) ", walMatcher, walMatcher), "{{cluster_name}}", ), - ).Description("LSN distance between tier 1 (local disk) and tier 2 (remote storage) per cluster. "+ - "A growing gap means remote archival is falling behind.")), + ).Description("Bytes difference between the WALs in tier 1 and tier 2 per cluster. ")), } } diff --git a/observability/grafana/server.go b/observability/grafana/server.go index 723babc23..6d89c0846 100644 --- a/observability/grafana/server.go +++ b/observability/grafana/server.go @@ -21,168 +21,301 @@ package main import ( "fmt" + + "github.com/grafana/grafana-foundation-sdk/go/common" + "github.com/grafana/grafana-foundation-sdk/go/units" ) +// snapshotCluster wraps a Kopia base-snapshot selector in a label_replace that +// derives a `cluster` label from the snapshot_source attribute (formatted +// `userName@clusterName:path`). The per-source snapshot gauges carry +// snapshot_source and tier but no cluster_name, so this lets them be grouped +// per PostgreSQL cluster instead of folding every cluster on a server into one +// value. +func snapshotCluster(selector string) string { + return fmt.Sprintf(`label_replace(%s, "cluster", "$1", "snapshot_source", "[^@]*@([^:]+):.*")`, selector) +} + // serverPanels returns the "Server" section panels. These metrics are emitted // by the Klio server StatefulSet (the `klio.server.*` family, exported to // Prometheus as `klio_server_*`): WAL ingest, backup verification, base // snapshots, the retention window of physical PostgreSQL backups, and the -// embedded NATS JetStream queue. Queries are scoped by $namespace and -// $server; the WAL and PostgreSQL-backup series additionally carry -// cluster_name and are scoped by $cluster. +// embedded NATS JetStream queue. Server-level series (uptime, snapshots, +// queue) carry no cluster_name and are grouped by service_name (the server +// identity, scoped by $server); the per-cluster WAL, PostgreSQL-backup and +// verification series carry cluster_name and are additionally scoped by +// $cluster. +// +//nolint:maintidx func serverPanels() []sizedPanel { return []sizedPanel{ - // Compact single/low-cardinality values first (stat tiles and bar - // gauges), then the wider time series. - sized(4, panelHeight, statPanel("Server uptime", "dtdurations", - query(fmt.Sprintf("max(klio_server_uptime_seconds{%s})", serverMatcher), "uptime"), - ).Description("Time since the Klio server process started. A sudden drop means the server "+ - "StatefulSet restarted.")), - // A bar gauge shows one labeled bar per tier, so both tiers are always - // visible (a narrow stat tile hides all but the first series). - sized(4, panelHeight, barGaugePanel("Latest WAL timeline by tier", - query(fmt.Sprintf("max by (tier) (klio_server_wal_latest_written_timeline{%s})", walMatcher), "{{tier}}"), - ).Description("Current PostgreSQL timeline of the latest WAL written per tier. A change reflects "+ - "a promotion or failover.")), - sized(4, panelHeight, barGaugePanel("Base snapshots by tier", - query(fmt.Sprintf("sum by (tier) (klio_server_backup_snapshots{%s})", serverMatcher), "{{tier}}"), - ).Description("Base backup snapshots currently retained per tier.")), - sized(4, panelHeight, statPanel("Latest snapshot size", "bytes", - query(fmt.Sprintf("max(klio_server_backup_latest_snapshot_size_bytes{%s})", serverMatcher), "latest size"), - ).Description("Size on the backend of the most recent base backup snapshot.")), - sized(4, panelHeight, statPanel("Latest snapshot files", "short", - query(fmt.Sprintf("max(klio_server_backup_latest_snapshot_files{%s})", serverMatcher), "latest files"), + sized(gridWidth, descriptionPanelHeight, descriptionPanel( + "State of the Klio server: WAL ingest, retained backups and snapshots, and the internal queue.")), + // Overview of the server and of its retention window. Stat tiles are laid out + // horizontally so that, with several clusters or tiers selected, every series + // stays visible instead of stacking vertically and clipping below the fixed + // panel height. Server-level values are grouped by service_name so two servers + // (even two with the same pod host name in different namespaces) never + // collapse into one number. + sized(largePanelWidth, mediumPanelHeight, statPanel("Server uptime", units.DurationInDaysHoursMinutesSeconds, + query(fmt.Sprintf("max by (service_name) (klio_server_uptime_seconds{%s})", serverMatcher), + "{{service_name}}"), + ).Orientation(common.VizOrientationHorizontal). + Description("Time since the Klio server process started, per server. A sudden drop means that "+ + "server's StatefulSet restarted.")), + + sized(largePanelWidth, mediumPanelHeight, statPanel("Retained backups", units.Number, + query(fmt.Sprintf("sum by (cluster_name, tier) (klio_server_backup_backups{%s})", walMatcher), + "{{cluster_name}} {{tier}}"), ).Decimals(0). - Description("Number of files in the most recent base backup snapshot.")), - sized(4, panelHeight, statPanel("Latest snapshot dirs", "short", - query(fmt.Sprintf("max(klio_server_backup_latest_snapshot_dirs{%s})", serverMatcher), "latest dirs"), + Orientation(common.VizOrientationHorizontal). + Description("Number of PostgreSQL backups currently retained per cluster and tier.")), + + sized(largePanelWidth, mediumPanelHeight, + statPanel("Oldest restorable point", units.DatetimeISO, + query( + fmt.Sprintf("min by (cluster_name, tier) "+ + "(klio_server_backup_oldest_backup_completion_time_seconds{%s}) * 1000", walMatcher), + "{{cluster_name}} {{tier}}"), + ).Orientation(common.VizOrientationHorizontal). + Description("Timestamp at which the oldest retained PostgreSQL backup completed, per cluster and "+ + "tier. It is the earliest point in time a restore can target, as long as all the WAL files "+ + "since the backup started are still retained.")), + + // Latest retained backup. + sized(largePanelWidth, mediumPanelHeight, + statPanel("Latest backup age (completion)", units.DurationInDaysHoursMinutesSeconds, + query( + fmt.Sprintf("time() - max by (cluster_name, tier) "+ + "(klio_server_backup_latest_backup_completion_time_seconds{%s})", walMatcher), + "{{cluster_name}} {{tier}}"), + ).Orientation(common.VizOrientationHorizontal). + Description("Elapsed time since the most recently retained PostgreSQL backup completed, per cluster "+ + "and tier.")), + + sized(largePanelWidth, mediumPanelHeight, + statPanel("Latest backup age (start)", units.DurationInDaysHoursMinutesSeconds, + query(fmt.Sprintf("time() - max by (cluster_name, tier) "+ + "(klio_server_backup_latest_backup_start_time_seconds{%s})", walMatcher), "{{cluster_name}} {{tier}}"), + ).Orientation(common.VizOrientationHorizontal). + Description("Elapsed time since the most recently retained PostgreSQL backup started, per cluster "+ + "and tier.")), + + sized(largePanelWidth, mediumPanelHeight, statPanel("Latest backup timeline", units.Number, + query(fmt.Sprintf("max by (cluster_name, tier) (klio_server_backup_latest_backup_timeline{%s})", + walMatcher), "{{cluster_name}} {{tier}}"), ).Decimals(0). - Description("Number of directories in the most recent base backup snapshot.")), - sized(4, panelHeight, statPanel("Latest snapshot age", "dtdurations", - query(fmt.Sprintf("time() - max(klio_server_backup_latest_snapshot_timestamp_seconds{%s})", serverMatcher), - "latest age"), - ).Description("Age of the most recent base backup snapshot. Should stay below the backup interval.")), - sized(4, panelHeight, statPanel("Oldest snapshot age", "dtdurations", - query(fmt.Sprintf("time() - min(klio_server_backup_oldest_snapshot_timestamp_seconds{%s})", serverMatcher), - "oldest age"), - ).Description("Age of the oldest retained base backup snapshot, reflecting the effective "+ - "retention horizon.")), - - // Retention window of the physical PostgreSQL backups (distinct from - // the Kopia base-snapshot gauges above): the klio.server.backup.backups - // / latest_backup_* / oldest_backup_* family, scoped by cluster_name - // via walMatcher. - sized(4, panelHeight, statPanel("Latest backup age (start)", "dtdurations", - query(fmt.Sprintf("time() - max(klio_server_backup_latest_backup_start_time_seconds{%s})", walMatcher), - "latest start age"), - ).Description("Elapsed time since the most recently retained PostgreSQL backup started.")), - sized(4, panelHeight, statPanel("Latest backup age (completion)", "dtdurations", - query( - fmt.Sprintf("time() - max(klio_server_backup_latest_backup_completion_time_seconds{%s})", walMatcher), - "latest completion age"), - ).Description("Elapsed time since the most recently retained PostgreSQL backup completed.")), - sized(4, panelHeight, statPanel("Oldest backup age (start)", "dtdurations", - query(fmt.Sprintf("time() - min(klio_server_backup_oldest_backup_start_time_seconds{%s})", walMatcher), - "oldest start age"), - ).Description("Elapsed time since the oldest retained PostgreSQL backup started, reflecting the "+ - "effective retention horizon.")), - sized(4, panelHeight, statPanel("Oldest backup age (completion)", "dtdurations", - query( - fmt.Sprintf("time() - min(klio_server_backup_oldest_backup_completion_time_seconds{%s})", walMatcher), - "oldest completion age"), - ).Description("Elapsed time since the oldest retained PostgreSQL backup completed, reflecting the "+ - "effective retention horizon.")), - sized(8, panelHeight, barGaugePanel("PostgreSQL backups retained by tier", - query(fmt.Sprintf("sum by (tier) (klio_server_backup_backups{%s})", walMatcher), "{{tier}}"), + Orientation(common.VizOrientationHorizontal). + Description("PostgreSQL timeline of the latest retained backup, per cluster and tier. Differing "+ + "from the oldest timeline means the retention window spans a promotion or failover.")), + + sized(gridWidth, mediumPanelHeight, lsnTablePanel("Latest backup LSN", + lsnColumn{"start", fmt.Sprintf( + "max by (cluster_name, tier) (klio_server_backup_latest_backup_start_lsn_bytes{%s})", walMatcher)}, + lsnColumn{"end", fmt.Sprintf( + "max by (cluster_name, tier) (klio_server_backup_latest_backup_end_lsn_bytes{%s})", walMatcher)}, + ).Description("Start and end LSN of the latest retained PostgreSQL backup, per cluster and tier.")), + + // Oldest retained backup, i.e. the edge of the retention window. + sized(largePanelWidth, mediumPanelHeight, + statPanel("Oldest backup age (completion)", units.DurationInDaysHoursMinutesSeconds, + query( + fmt.Sprintf("time() - min by (cluster_name, tier) "+ + "(klio_server_backup_oldest_backup_completion_time_seconds{%s})", walMatcher), + "{{cluster_name}} {{tier}}"), + ).Orientation(common.VizOrientationHorizontal). + Description("Elapsed time since the oldest retained PostgreSQL backup completed, per cluster and "+ + "tier, reflecting each tier's effective retention horizon.")), + + sized(largePanelWidth, mediumPanelHeight, + statPanel("Oldest backup age (start)", units.DurationInDaysHoursMinutesSeconds, + query(fmt.Sprintf("time() - min by (cluster_name, tier) "+ + "(klio_server_backup_oldest_backup_start_time_seconds{%s})", walMatcher), "{{cluster_name}} {{tier}}"), + ).Orientation(common.VizOrientationHorizontal). + Description("Elapsed time since the oldest retained PostgreSQL backup started, per cluster and tier, "+ + "reflecting each tier's effective retention horizon.")), + + sized(largePanelWidth, mediumPanelHeight, statPanel("Oldest backup timeline", units.Number, + query(fmt.Sprintf("max by (cluster_name, tier) (klio_server_backup_oldest_backup_timeline{%s})", + walMatcher), "{{cluster_name}} {{tier}}"), + ).Decimals(0). + Orientation(common.VizOrientationHorizontal). + Description("PostgreSQL timeline of the oldest retained backup, per cluster and tier. Differing "+ + "from the latest timeline means the retention window spans a promotion or failover.")), + + sized(gridWidth, mediumPanelHeight, lsnTablePanel("Oldest backup LSN", + lsnColumn{"start", fmt.Sprintf( + "max by (cluster_name, tier) (klio_server_backup_oldest_backup_start_lsn_bytes{%s})", walMatcher)}, + lsnColumn{"end", fmt.Sprintf( + "max by (cluster_name, tier) (klio_server_backup_oldest_backup_end_lsn_bytes{%s})", walMatcher)}, + ).Description("Start and end LSN of the oldest retained PostgreSQL backup, per cluster and tier.")), + + // Base snapshots on Kopia. + sized(smallPanelWidth, mediumPanelHeight, statPanel("Retained snapshots", units.Number, + query(fmt.Sprintf("sum by (cluster, tier) (%s)", + snapshotCluster(fmt.Sprintf("klio_server_backup_snapshots{%s}", serverMatcher))), "{{cluster}} {{tier}}"), + ).Decimals(0). + Orientation(common.VizOrientationHorizontal). + Description("Total kopia snapshots currently retained per cluster and tier.")), + + sized(smallPanelWidth, mediumPanelHeight, statPanel("Latest snapshot size", units.BytesIEC, + query(fmt.Sprintf("max by (cluster, tier) (%s)", + snapshotCluster(fmt.Sprintf("klio_server_backup_latest_snapshot_size_bytes{%s}", serverMatcher))), + "{{cluster}} {{tier}}"), + ).Orientation(common.VizOrientationHorizontal). + Description("Size of the most recent base backup snapshot on Kopia, not accounting for deduplication.")), + + sized(smallPanelWidth, mediumPanelHeight, statPanel("Latest snapshot files", units.Number, + query(fmt.Sprintf("max by (cluster, tier) (%s)", + snapshotCluster(fmt.Sprintf("klio_server_backup_latest_snapshot_files{%s}", serverMatcher))), + "{{cluster}} {{tier}}"), + ).Decimals(0). + Orientation(common.VizOrientationHorizontal). + Description("Number of files in the most recent base backup snapshot on Kopia.")), + + sized(smallPanelWidth, mediumPanelHeight, statPanel("Latest snapshot dirs", units.Number, + query(fmt.Sprintf("max by (cluster, tier) (%s)", + snapshotCluster(fmt.Sprintf("klio_server_backup_latest_snapshot_dirs{%s}", serverMatcher))), + "{{cluster}} {{tier}}"), ).Decimals(0). - Description("Number of PostgreSQL backups currently retained per tier, across clusters.")), - sized(8, panelHeight, timeseriesPanel("Latest backup LSN by cluster", "bytes", - query(fmt.Sprintf("max by (cluster_name) (klio_server_backup_latest_backup_start_lsn_bytes{%s})", - walMatcher), "{{cluster_name}} start"), - query(fmt.Sprintf("max by (cluster_name) (klio_server_backup_latest_backup_end_lsn_bytes{%s})", - walMatcher), "{{cluster_name}} end"), - ).Description("Start and end LSN of the latest retained PostgreSQL backup, per cluster.")), - sized(8, panelHeight, timeseriesPanel("Oldest backup LSN by cluster", "bytes", - query(fmt.Sprintf("max by (cluster_name) (klio_server_backup_oldest_backup_start_lsn_bytes{%s})", - walMatcher), "{{cluster_name}} start"), - query(fmt.Sprintf("max by (cluster_name) (klio_server_backup_oldest_backup_end_lsn_bytes{%s})", - walMatcher), "{{cluster_name}} end"), - ).Description("Start and end LSN of the oldest retained PostgreSQL backup, per cluster.")), - sized(8, panelHeight, barGaugePanel("Backup timeline by cluster", - query(fmt.Sprintf("max by (cluster_name) (klio_server_backup_latest_backup_timeline{%s})", walMatcher), - "{{cluster_name}} latest"), - query(fmt.Sprintf("max by (cluster_name) (klio_server_backup_oldest_backup_timeline{%s})", walMatcher), - "{{cluster_name}} oldest"), - ).Description("PostgreSQL timeline of the latest and oldest retained backup, per cluster. A "+ - "mismatch means the retention window spans a promotion or failover.")), - - // WAL ingest, unified across tiers via the `tier` label. - sized(8, panelHeight, timeseriesPanel("WAL files written rate by tier", "wps", - query(fmt.Sprintf("sum by (tier) (rate(klio_server_wal_written_total{%s}[$__rate_interval]))", walMatcher), - "{{tier}}"), - ).Description("Rate of WAL files written by the server, split by storage tier.")), - sized(8, panelHeight, timeseriesPanel("WAL bytes written rate by tier", "Bps", + Orientation(common.VizOrientationHorizontal). + Description("Number of directories in the most recent base backup snapshot on Kopia.")), + + sized(smallPanelWidth, mediumPanelHeight, statPanel("Latest snapshot age", units.DurationInDaysHoursMinutesSeconds, + query(fmt.Sprintf("time() - max by (cluster, tier) (%s)", + snapshotCluster(fmt.Sprintf("klio_server_backup_latest_snapshot_timestamp_seconds{%s}", serverMatcher))), + "{{cluster}} {{tier}}"), + ).Orientation(common.VizOrientationHorizontal). + Description("Age of the most recent base backup snapshot on Kopia, per cluster and tier.")), + + sized(smallPanelWidth, mediumPanelHeight, statPanel("Oldest snapshot age", units.DurationInDaysHoursMinutesSeconds, + query(fmt.Sprintf("time() - min by (cluster, tier) (%s)", + snapshotCluster(fmt.Sprintf("klio_server_backup_oldest_snapshot_timestamp_seconds{%s}", serverMatcher))), + "{{cluster}} {{tier}}"), + ).Orientation(common.VizOrientationHorizontal). + Description("Age of the oldest retained base backup snapshot on Kopia, per cluster and tier.")), + + // WAL ingest. + sized(largePanelWidth, mediumPanelHeight, timelinePanel("WAL timeline", + query(fmt.Sprintf("max by (cluster_name, tier) (klio_server_wal_latest_written_timeline{%s})", walMatcher), + "{{cluster_name}} {{tier}}"), + ).Description("PostgreSQL timeline of the latest WAL written per cluster and tier")), + + sized(largePanelWidth, mediumPanelHeight, timeseriesPanel("Time since last WAL written", units.DurationSeconds, + query(fmt.Sprintf("time() - max by (cluster_name, tier) (klio_server_wal_latest_written_time_seconds{%s})", + walMatcher), "{{cluster_name}} {{tier}}"), + ).Description("Elapsed time since the server last wrote a WAL file for each cluster and tier.")), + + sized(largePanelWidth, mediumPanelHeight, lsnTablePanel("Latest written LSN", + lsnColumn{"written", fmt.Sprintf( + "max by (cluster_name, tier) (klio_server_wal_latest_written_lsn_bytes{%s})", walMatcher)}, + ).Description("Most recent WAL LSN the server has written for each cluster and tier")), + + sized(largestPanelWidth, mediumPanelHeight, timeseriesPanel("WAL files written (rate)", units.OpsPerSecond, + query(fmt.Sprintf("sum by (cluster_name, tier) (rate(klio_server_wal_written_total{%s}[$__rate_interval]))", + walMatcher), "{{cluster_name}} {{tier}}"), + ).Description("Rate of WAL files written by the server, split by cluster and storage tier.")), + + sized(largestPanelWidth, mediumPanelHeight, timeseriesPanel("WAL files written (total)", units.Number, + query(fmt.Sprintf("sum by (cluster_name, tier) (klio_server_wal_written_total{%s})", + walMatcher), "{{cluster_name}} {{tier}}"), + ).Description("Total WAL files written by the server since it last restarted, split by cluster and "+ + "storage tier.")), + + sized(largestPanelWidth, mediumPanelHeight, timeseriesPanel("WAL bytes written (rate)", units.BytesPerSecondIEC, query( - fmt.Sprintf("sum by (tier) (rate(klio_server_wal_written_size_bytes_total{%s}[$__rate_interval]))", - walMatcher), - "{{tier}}"), - ).Description("Rate of WAL bytes written by the server, split by storage tier.")), - sized(8, panelHeight, timeseriesPanel("Time since last WAL written by tier", "dtdurations", - query(fmt.Sprintf("time() - max by (tier) (klio_server_wal_latest_written_time_seconds{%s})", walMatcher), - "{{tier}}"), - ).Description("Elapsed time since the server last wrote a WAL file for each tier. A stale tier-1 "+ - "value means PostgreSQL stopped shipping WALs; a stale tier-2 value means the remote backend "+ - "stopped receiving them.")), - sized(8, panelHeight, timeseriesPanel("Latest written LSN by tier", "bytes", - query(fmt.Sprintf("max by (tier) (klio_server_wal_latest_written_lsn_bytes{%s})", walMatcher), "{{tier}}"), - ).Description("Most recent WAL LSN the server has written for each tier, as a byte offset.")), - sized(8, panelHeight, timeseriesPanel("Backup verification rate by outcome and tier", "ops", + fmt.Sprintf("sum by (cluster_name, tier) "+ + "(rate(klio_server_wal_written_size_bytes_total{%s}[$__rate_interval]))", walMatcher), + "{{cluster_name}} {{tier}}"), + ).Description("Rate of WAL bytes written by the server, split by cluster and storage tier.")), + + sized(largestPanelWidth, mediumPanelHeight, timeseriesPanel("WAL bytes written (total)", units.BytesIEC, query( - fmt.Sprintf("sum by (outcome, tier) (rate(klio_server_backup_verifications_total{%s}[$__rate_interval]))", - serverMatcher), - "{{tier}} / {{outcome}}", + fmt.Sprintf("sum by (cluster_name, tier) (klio_server_wal_written_size_bytes_total{%s})", walMatcher), + "{{cluster_name}} {{tier}}"), + ).Description("Total WAL bytes written by the server since it last restarted, split by cluster and "+ + "storage tier.")), + + // WAL processing duration, each (rate) panel next to its (total) counterpart. + sized(largestPanelWidth, largePanelHeight, + timeseriesPanel("WAL block operation duration percentiles (rate)", units.Nanoseconds, + quantileTargets("klio_server_wal_block_duration_nanoseconds_bucket", "le, path, stage", + walMatcher, "{{path}}/{{stage}}")..., + ).Description("Percentile per-block WAL processing duration on the server, split by `path` "+ + "(put ingest / get serve) and `stage`, aggregated across the selected clusters. Reflects recent "+ + "activity, over the rate interval window.")), + + sized(largestPanelWidth, largePanelHeight, + timeseriesPanel("WAL block operation duration percentiles (total)", units.Nanoseconds, + quantileTargetsAbsolute("klio_server_wal_block_duration_nanoseconds_bucket", "le, path, stage", + walMatcher, "{{path}}/{{stage}}")..., + ).Description("Percentile per-block WAL processing duration on the server, split by `path` "+ + "(put ingest / get serve) and `stage`, aggregated across the selected clusters. Reflects all "+ + "activity since the server last restarted.")), + + sized(largestPanelWidth, largePanelHeight, + timeseriesPanel("WAL file get duration percentiles (rate)", units.Nanoseconds, + quantileTargets("klio_server_wal_get_duration_nanoseconds_bucket", "le, tier", + walMatcher, "{{tier}}")..., + ).Description("Percentile duration of a complete WAL file gRPC get, split by the tier that "+ + "served it, aggregated across the selected clusters. Reflects recent activity, over the rate interval window.")), + + sized(largestPanelWidth, largePanelHeight, + timeseriesPanel("WAL file get duration percentiles (total)", units.Nanoseconds, + quantileTargetsAbsolute("klio_server_wal_get_duration_nanoseconds_bucket", "le, tier", + walMatcher, "{{tier}}")..., + ).Description("Percentile duration of a complete WAL file gRPC get, split by the tier that "+ + "served it, aggregated across the selected clusters. Reflects all activity since the server "+ + "last restarted.")), + + sized(largestPanelWidth, largePanelHeight, + timeseriesPanel("WAL tier-2 upload duration percentiles (rate)", units.Nanoseconds, + quantileTargets("klio_server_wal_upload_duration_nanoseconds_bucket", "le, cluster_name", + walMatcher, "{{cluster_name}}")..., + ).Description("Percentile duration of the tier-2 archival upload to remote storage, per cluster. "+ + "Reflects recent activity, over the rate interval window.")), + + sized(largestPanelWidth, largePanelHeight, + timeseriesPanel("WAL tier-2 upload duration percentiles (total)", units.Nanoseconds, + quantileTargetsAbsolute("klio_server_wal_upload_duration_nanoseconds_bucket", "le, cluster_name", + walMatcher, "{{cluster_name}}")..., + ).Description("Percentile duration of the tier-2 archival upload to remote storage, per cluster. "+ + "Reflects all activity since the server last restarted.")), + + // Async operations. + sized(largePanelWidth, mediumPanelHeight, timeseriesPanel("Tier-2 backup relay attempts", units.Number, + query( + fmt.Sprintf("sum by (cluster_name, outcome) (klio_server_backup_relay_total{%s})", walMatcher), + "{{cluster_name}} / {{outcome}}", ), - ).Description("Rate of base backup verification checks, broken down by outcome and tier.")), - - // WAL processing latency, from the per-block and per-file duration - // histograms introduced alongside the unified WAL ingest series. - sized(8, panelHeight, timeseriesPanel("WAL block duration (p50/p95/p99) by path and stage", "ns", - quantileTargets("klio_server_wal_block_duration_nanoseconds_bucket", "le, path, stage", - walMatcher, "{{path}}/{{stage}}")..., - ).Description("Percentile per-block WAL processing duration on the server, split by `path` "+ - "(put ingest / get serve) and `stage`.")), - sized(8, panelHeight, timeseriesPanel("WAL file get duration (p50/p95/p99) by tier", "ns", - quantileTargets("klio_server_wal_get_duration_nanoseconds_bucket", "le, tier", - walMatcher, "{{tier}}")..., - ).Description("Percentile duration of a complete WAL file gRPC get, split by the tier that "+ - "served it.")), - sized(8, panelHeight, timeseriesPanel("WAL tier-2 upload duration (p50/p95/p99) by cluster", "ns", - quantileTargets("klio_server_wal_upload_duration_nanoseconds_bucket", "le, cluster_name", - walMatcher, "{{cluster_name}}")..., - ).Description("Percentile duration of the tier-2 archival upload to remote storage, per cluster.")), - - // Post-backup processing: tier-2 relay and per-tier maintenance runs. - sized(12, panelHeight, timeseriesPanel("Tier-2 relay rate by outcome", "ops", + ).Description("Total tier-2 relay attempts since the server last restarted, per cluster and outcome.")), + + sized(largePanelWidth, mediumPanelHeight, timeseriesPanel("Backup verification runs", units.Number, query( - fmt.Sprintf("sum by (outcome) (rate(klio_server_backup_relay_total{%s}[$__rate_interval]))", walMatcher), - "{{outcome}}", + fmt.Sprintf("sum by (cluster_name, outcome, tier) (klio_server_backup_verifications_total{%s})", walMatcher), + "{{cluster_name}} {{tier}} / {{outcome}}", ), - ).Description("Rate of tier-2 relay attempts after a backup (migration and verification), by outcome.")), - sized(12, panelHeight, timeseriesPanel("Maintenance run rate by tier and outcome", "ops", + ).Description("Total base backup verification checks since the server last restarted, per cluster, "+ + "broken down by outcome and tier.")), + + sized(largePanelWidth, mediumPanelHeight, timeseriesPanel("Maintenance runs", units.Number, query( - fmt.Sprintf("sum by (tier, outcome) (rate(klio_server_backup_maintenance_total{%s}[$__rate_interval]))", - walMatcher), - "{{tier}} / {{outcome}}", + fmt.Sprintf("sum by (cluster_name, tier, outcome) (klio_server_backup_maintenance_total{%s})", walMatcher), + "{{cluster_name}} {{tier}}/{{outcome}}", ), - ).Description("Rate of post-backup maintenance runs (base-snapshot retention and WAL cleanup), "+ - "by tier and outcome.")), - - // Embedded NATS JetStream queue, broken down by stream. - sized(8, panelHeight, timeseriesPanel("Queue messages by stream", "none", - query(fmt.Sprintf("sum by (stream) (klio_server_queue_messages{%s})", serverMatcher), "{{stream}}"), - ).Description("Messages currently held in each NATS JetStream stream of the embedded queue.")), - sized(8, panelHeight, timeseriesPanel("Queue bytes by stream", "bytes", - query(fmt.Sprintf("sum by (stream) (klio_server_queue_bytes{%s})", serverMatcher), "{{stream}}"), - ).Description("Bytes currently held in each NATS JetStream stream of the embedded queue.")), + ).Description("Total post-backup maintenance runs since the server last restarted, per cluster, "+ + "tier and outcome.")), + + // Embedded NATS JetStream queue, per server and stream. + sized(largestPanelWidth, mediumPanelHeight, timeseriesPanel("Queue size (messages)", units.Number, + query(fmt.Sprintf("sum by (service_name, stream) (klio_server_queue_messages{%s})", serverMatcher), + "{{service_name}} / {{stream}}"), + ).Description("Messages currently held in each NATS JetStream stream of the embedded queue, per "+ + "server.")), + + sized(largestPanelWidth, mediumPanelHeight, timeseriesPanel("Queue size (bytes)", units.BytesIEC, + query(fmt.Sprintf("sum by (service_name, stream) (klio_server_queue_bytes{%s})", serverMatcher), + "{{service_name}} / {{stream}}"), + ).Description("Bytes currently held in each NATS JetStream stream of the embedded queue, per server.")), } }