chore: add opt-in retry for rate-limited and transport-failed requests

README.md

@@ -89,7 +89,7 @@ Pass `Logger` on `StreamOptions` (or `ClientBuilder.Logger(...)`) to receive str
 | `client.initialized` | INFO | Once, at `BaseClient` construction |
 | `http.request.sent` | DEBUG | Before every request is sent |
 | `http.response.received` | DEBUG | After any HTTP response, including 4xx/5xx |
-| `http.request.failed` | ERROR | Only on a transport failure (no HTTP response received) |
+| `http.request.failed` | ERROR or DEBUG | ERROR on a final transport failure (no HTTP response received); DEBUG instead, with a `{RetryAttempt}` field, on any attempt that [Retry](#retry-policy) will retry. Never emitted for a final/non-retried 429 (already covered by `http.response.received`). |
 
 .NET's `ILogger` uses PascalCase message-template placeholders (`{Method}`, `{StatusCode}`, ...). Each maps to a canonical snake_case field name used identically across all GetStream SDKs:
 
@@ -116,16 +116,37 @@ Pass `Logger` on `StreamOptions` (or `ClientBuilder.Logger(...)`) to receive str
 | | `{Body}` | `body` (redacted; only present when `LogBodies=true`) |
 | `http.request.failed` | `{Method}` | `method` |
 | | `{Path}` | `path` |
-| | `{ErrorType}` | `error.type` |
-| | `{DurationMs}` | `duration_ms` |
-| | `{Message}` | `error.message` (redacted) |
+| | `{ErrorType}` | `error.type` (transport failures only; never present on a retried 429 — see below) |
+| | `{DurationMs}` | `duration_ms` (transport failures only) |
+| | `{Message}` | `error.message` (redacted; transport failures only) |
+| | `{RetryAttempt}` | `retry.attempt` (1-indexed; present only when this attempt will be retried) |
 
-`error.type` is one of `connection_reset`, `timeout`, `dns_failure`, `tls_handshake_failed`, `unknown` (see `GetStreamTransportException.ErrorType`).
+`error.type` is one of `connection_reset`, `timeout`, `dns_failure`, `tls_handshake_failed`, `unknown` (see `GetStreamTransportException.ErrorType`) — a closed transport-only enum. A retried HTTP 429 has no transport error at all, so its `http.request.failed` DEBUG line carries only `{Method}`, `{Path}`, `{RetryAttempt}`: never `{ErrorType}`.
 
 **Redaction (always on, no opt-out):** query values for `api_key`/`api_secret`/`token` (case-insensitive) become `<redacted>`; top-level JSON body keys `api_secret`/`token`/`password` become `<redacted>` (shallow, key names are preserved). No header values are ever logged. `error.message` is additionally scrubbed for any `api_key=`/`api_secret=`/`token=` value appearing anywhere in the free-form transport-exception text.
 
 **Bodies are not logged by default.** Set `StreamOptions.LogBodies = true` (or `ClientBuilder.LogBodies(true)`) to opt in; body content is still key-redacted as above. Enabling it emits exactly one WARN line at construction.
 
+## Retry Policy
+
+Auto-retry is opt-in and off by default: with no `Retry` configured, the client performs exactly one attempt and errors surface unchanged.
+
+```csharp
+var client = new StreamClient(new StreamOptions
+{
+    ApiKey = "...",
+    ApiSecret = "...",
+    Retry = new RetryConfig { Enabled = true, MaxAttempts = 3, MaxBackoff = TimeSpan.FromSeconds(30) },
+});
+```
+
+When enabled:
+- Only `GET`/`HEAD` requests are retried. Writes (`POST`/`PUT`/`PATCH`/`DELETE`) never are.
+- Only HTTP 429 (rate limit) and transport-layer failures (connection reset, timeout, DNS, TLS) are retried; any other 4xx/5xx is never retried.
+- A 429 marked `unrecoverable` by the backend is never retried.
+- `MaxAttempts` is the total attempt budget including the initial request (default `3`: 1 initial + 2 retries).
+- The delay before each retry honors a `Retry-After` response header when present, clamped to `MaxBackoff`; otherwise it's full jitter over `[0, min(MaxBackoff, 2^attempt seconds)]`.
+
 ## Release Process
 
 Releases use two paths, both handled by `.github/workflows/release.yml`:

src/Client.cs

@@ -22,6 +22,7 @@ public class BaseClient : IClient
         private readonly Microsoft.Extensions.Logging.ILogger? _logger;
         private readonly bool _userHttpClient;
         private readonly bool _logBodies;
+        private readonly RetryConfig? _retry;
         protected string ApiKey;
         protected string ApiSecret;
         protected string BaseUrl;
@@ -60,6 +61,7 @@ public BaseClient(StreamOptions opts)
             BaseUrl = opts.BaseUrl;
             _logger = opts.Logger;
             _logBodies = opts.LogBodies;
+            _retry = opts.Retry;
 
             if (opts.HttpClient != null)
             {
@@ -129,6 +131,11 @@ private void LogInitialized(StreamOptions opts)
             }
         }
 
+        /// <summary>
+        /// CHA-2959 retry loop wrapping <see cref="SendOnceAsync{TRequest,TResponse}"/>. Retries are opt-in
+        /// via <see cref="StreamOptions.Retry"/> (disabled by default: exactly one attempt, errors surface
+        /// unchanged). See <see cref="ShouldRetry"/> / <see cref="RetryDelay"/> for the policy.
+        /// </summary>
         public async Task<StreamResponse<TResponse>> MakeRequestAsync<TRequest, TResponse>(
             string method,
             string path,
@@ -138,6 +145,98 @@ public async Task<StreamResponse<TResponse>> MakeRequestAsync<TRequest, TRespons
             CancellationToken cancellationToken = default)
         {
             var url = BuildUrl(path, queryParams, pathParams);
+            // Uri parses the same absolute URL BuildUrl just produced, so the path used for
+            // logging never drifts from what is actually sent on the wire.
+            var logPath = new Uri(url).AbsolutePath;
+
+            for (var attempt = 0; ; attempt++)
+            {
+                var stopwatch = System.Diagnostics.Stopwatch.StartNew();
+                try
+                {
+                    return await SendOnceAsync<TRequest, TResponse>(method, url, requestBody, cancellationToken);
+                }
+                catch (GetStreamException ex)
+                {
+                    stopwatch.Stop();
+                    var willRetry = ShouldRetry(ex, method, attempt);
+
+                    // CHA-2957/CHA-2959: failure logging lives here (not in SendOnceAsync) because only the
+                    // loop knows whether this attempt will retry (DEBUG) or is final (ERROR). CROSS-SDK RULE:
+                    // ErrorType is a closed transport-only enum, so a retried 429 logs through a separate
+                    // template that omits the {ErrorType} placeholder entirely; a final/non-retried 429 logs
+                    // nothing here (unchanged: it was already logged via http.response.received).
+                    if (ex is GetStreamTransportException transportEx)
+                    {
+                        if (willRetry)
+                        {
+                            _logger?.LogDebug("http.request.failed {Method} {Path} {ErrorType} {DurationMs} {Message} {RetryAttempt}",
+                                method, logPath, transportEx.ErrorType, stopwatch.ElapsedMilliseconds,
+                                LogRedaction.RedactMessage(transportEx.Message), attempt + 1);
+                        }
+                        else
+                        {
+                            _logger?.LogError("http.request.failed {Method} {Path} {ErrorType} {DurationMs} {Message}",
+                                method, logPath, transportEx.ErrorType, stopwatch.ElapsedMilliseconds,
+                                LogRedaction.RedactMessage(transportEx.Message));
+                        }
+                    }
+                    else if (willRetry && ex is GetStreamRateLimitException)
+                    {
+                        _logger?.LogDebug("http.request.failed {Method} {Path} {RetryAttempt}", method, logPath, attempt + 1);
+                    }
+
+                    if (!willRetry) throw;
+
+                    await Task.Delay(RetryDelay(ex, attempt), cancellationToken);
+                }
+            }
+        }
+
+        /// <summary>CHA-2959: true when the retry policy is enabled and this failed attempt should be retried.</summary>
+        internal bool ShouldRetry(GetStreamException ex, string method, int attempt)
+        {
+            if (_retry is not { Enabled: true }) return false;
+            if (method != "GET" && method != "HEAD") return false;
+            if (attempt + 1 >= _retry.MaxAttempts) return false;
+
+            return ex switch
+            {
+                GetStreamRateLimitException rl => !rl.Unrecoverable,
+                GetStreamTransportException => true,
+                _ => false,
+            };
+        }
+
+        /// <summary>
+        /// CHA-2959: delay before the next attempt. Honors <c>Retry-After</c> (clamped to <see cref="RetryConfig.MaxBackoff"/>)
+        /// when present and positive; otherwise full jitter over <c>[0, min(MaxBackoff, 2^attempt seconds)]</c>.
+        /// Only called after <see cref="ShouldRetry"/> returned true, so <c>_retry</c> is non-null here.
+        /// </summary>
+        internal TimeSpan RetryDelay(GetStreamException ex, int attempt)
+        {
+            if (ex is GetStreamRateLimitException { RetryAfter: { } retryAfter } && retryAfter > TimeSpan.Zero)
+            {
+                return retryAfter < _retry!.MaxBackoff ? retryAfter : _retry.MaxBackoff;
+            }
+
+            // Exponent clamped to 30 so the shift never wraps (long shift counts are masked mod 64 in C#).
+            var ceilTicks = Math.Min(_retry!.MaxBackoff.Ticks, TimeSpan.TicksPerSecond << Math.Min(attempt, 30));
+            return ceilTicks <= 0 ? TimeSpan.Zero : TimeSpan.FromTicks((long)(Random.Shared.NextDouble() * ceilTicks));
+        }
+
+        /// <summary>
+        /// Builds and sends a single HTTP attempt: fresh <see cref="HttpRequestMessage"/> (a sent request
+        /// cannot be reused across retries), logs <c>http.request.sent</c>/<c>http.response.received</c>,
+        /// and throws on transport failure or non-2xx without logging <c>http.request.failed</c> itself
+        /// -- that is the retry loop's job (see <see cref="MakeRequestAsync{TRequest,TResponse}"/>).
+        /// </summary>
+        private async Task<StreamResponse<TResponse>> SendOnceAsync<TRequest, TResponse>(
+            string method,
+            string url,
+            TRequest? requestBody,
+            CancellationToken cancellationToken)
+        {
             var request = new HttpRequestMessage(new HttpMethod(method), url);
 
             // Add authentication headers
@@ -168,8 +267,6 @@ public async Task<StreamResponse<TResponse>> MakeRequestAsync<TRequest, TRespons
                 }
             }
 
-            // Uri parses the same absolute URL BuildUrl just produced, so path/query for
-            // logging never drift from what is actually sent on the wire.
             var uri = new Uri(url);
             var logPath = uri.AbsolutePath;
             var logQuery = LogRedaction.RedactQuery(uri.Query.TrimStart('?'));
@@ -197,12 +294,7 @@ public async Task<StreamResponse<TResponse>> MakeRequestAsync<TRequest, TRespons
             }
             catch (Exception ex) when (IsTransportException(ex, cancellationToken))
             {
-                stopwatch.Stop();
-                var transportException = WrapTransportException(ex, cancellationToken);
-                _logger?.LogError("http.request.failed {Method} {Path} {ErrorType} {DurationMs} {Message}",
-                    method, logPath, transportException.ErrorType, stopwatch.ElapsedMilliseconds,
-                    LogRedaction.RedactMessage(transportException.Message));
-                throw transportException;
+                throw WrapTransportException(ex, cancellationToken);
             }
             stopwatch.Stop();
 
@@ -239,6 +331,7 @@ public async Task<StreamResponse<TResponse>> MakeRequestAsync<TRequest, TRespons
         }
 
         // CHA-2957: near-duplicate send path, not exposed on IClient; intentionally not instrumented with structured logging.
+        // CHA-2959: for the same reason (not on IClient, no generated code calls it), intentionally excluded from the retry policy.
         public async Task<StreamResponse<TResponse>> MakeRequestAsyncDebug<TRequest, TResponse>(
             string method,
             string path,

src/ClientBuilder.cs

@@ -38,6 +38,7 @@ public class ClientBuilder
         private System.Net.Http.HttpClient? _httpClient;
         private ILogger? _logger;
         private bool _logBodies;
+        private RetryConfig? _retry;
 
         /// <summary>
         /// Set the API key
@@ -105,6 +106,9 @@ public ClientBuilder EnvPath(string path)
         /// <summary>Opt-in body logging (DEBUG events); shallow secret-key redaction still applies. Default false.</summary>
         public ClientBuilder LogBodies(bool enabled) { _logBodies = enabled; return this; }
 
+        /// <summary>Opt-in auto-retry policy (default: disabled, no retries).</summary>
+        public ClientBuilder Retry(RetryConfig retry) { _retry = retry; return this; }
+
         /// <summary>
         /// Create a client builder that loads configuration from environment variables
         /// </summary>
@@ -216,6 +220,7 @@ private StreamOptions BuildStreamOptions()
                 HttpClient = _httpClient,
                 Logger = _logger,
                 LogBodies = _logBodies,
+                Retry = _retry,
             };
         }
 

src/RetryConfig.cs

@@ -0,0 +1,20 @@
+using System;
+
+namespace GetStream
+{
+    /// <summary>
+    /// Opt-in auto-retry policy. Disabled by default: the client performs exactly one attempt and surfaces errors unchanged.
+    /// When enabled, only GET/HEAD requests failing with HTTP 429 or a transport error are retried, and never when the backend marked the error unrecoverable.
+    /// </summary>
+    public class RetryConfig
+    {
+        /// <summary>Master switch. Default false.</summary>
+        public bool Enabled { get; set; }
+
+        /// <summary>Total attempt budget including the initial request. Default 3 (1 initial + 2 retries).</summary>
+        public int MaxAttempts { get; set; } = 3;
+
+        /// <summary>Cap applied to every wait between attempts, including Retry-After hints. Default 30s.</summary>
+        public TimeSpan MaxBackoff { get; set; } = TimeSpan.FromSeconds(30);
+    }
+}

src/StreamOptions.cs

@@ -41,5 +41,8 @@ public class StreamOptions
 
         /// <summary>Opt-in body logging on the DEBUG events; known-secret keys stay redacted. Default false.</summary>
         public bool LogBodies { get; set; }
+
+        /// <summary>Opt-in auto-retry policy. Null (or Enabled=false) means no retries.</summary>
+        public RetryConfig? Retry { get; set; }
     }
 }