fix(billing): let operators resolve billing dead letters and stop re-paging old ones (#8869)

A billing Stripe sync that exhausted its retries stayed dead-lettered forever:
requeue only sends it back through the same failure, retention never prunes a
dead letter, and the hourly billing reconcile cron logged every one at ERROR
on every run.

- POST /api/v1/admin/outbox/[id]/resolve closes a dead-lettered billing Stripe
  event (dead_letter -> completed). last_error keeps who resolved it, why, and
  the last failure; the action is audited. Other domains count completed rows
  as delivered, so only billing Stripe events can be resolved.
- The reconcile cron logs dead letters from the last two hourly runs at ERROR
  and older unresolved ones at WARN with the same identifiers; the dead-letter
  scan returns newest first so its cap never hides a new one.
- The admin dashboard reports a resolved cancellation as resolved, not
  applied.

Author: waleedlatif1

apps/sim/app/api/cron/reconcile-billing-seats/route.ts

@@ -1,5 +1,7 @@
 import { createLogger } from '@sim/logger'
+import { toStringOrNull } from '@sim/utils/coerce'
 import { toError } from '@sim/utils/errors'
+import { toRecord } from '@sim/utils/object'
 import { type NextRequest, NextResponse } from 'next/server'
 import { verifyCronAuth } from '@/lib/auth/internal'
 import { reconcileTeamSeatDrift } from '@/lib/billing/organizations/seat-drift'
@@ -17,6 +19,22 @@ const BILLING_SYNC_EVENT_TYPES = [
   OUTBOX_EVENT_TYPES.STRIPE_SYNC_CANCEL_AT_PERIOD_END,
 ]
 
+/**
+ * Dead letters newer than two hourly runs log at ERROR, so one missed run cannot hide one; older
+ * ones repeat at WARN until requeued or resolved through the admin outbox API.
+ */
+const NEW_DEAD_LETTER_WINDOW_MS = 2 * 60 * 60 * 1000
+
+function describeDeadLetter(event: Awaited<ReturnType<typeof findDeadLetteredEvents>>[number]) {
+  return {
+    id: event.id,
+    eventType: event.eventType,
+    subscriptionId: toStringOrNull(toRecord(event.payload).subscriptionId),
+    deadLetteredAt: event.processedAt?.toISOString() ?? null,
+    lastError: event.lastError,
+  }
+}
+
 /**
  * Periodic billing-seat reconciliation. Self-heals Team organizations whose
  * stored seat count drifted from their member count, and reports any
@@ -38,18 +56,28 @@ export const GET = withRouteHandler(async (request: NextRequest) => {
     const drift = await reconcileTeamSeatDrift()
 
     const deadLettered = await findDeadLetteredEvents(BILLING_SYNC_EVENT_TYPES)
-    if (deadLettered.length > 0) {
+    const newSince = Date.now() - NEW_DEAD_LETTER_WINDOW_MS
+    const isNew = (event: (typeof deadLettered)[number]) =>
+      event.processedAt !== null && event.processedAt.getTime() >= newSince
+    const newlyDeadLettered = deadLettered.filter(isNew)
+    const previouslyReported = deadLettered.filter((event) => !isNew(event))
+    if (newlyDeadLettered.length > 0) {
       logger.error(
         'Dead-lettered billing sync events require manual remediation — a billing state change (seat charge or cancellation) never reached Stripe',
         {
           requestId,
-          count: deadLettered.length,
-          events: deadLettered.map((event) => ({
-            id: event.id,
-            eventType: event.eventType,
-            subscriptionId: (event.payload as { subscriptionId?: string } | null)?.subscriptionId,
-            lastError: event.lastError,
-          })),
+          count: newlyDeadLettered.length,
+          events: newlyDeadLettered.map(describeDeadLetter),
+        }
+      )
+    }
+    if (previouslyReported.length > 0) {
+      logger.warn(
+        'Previously reported billing sync dead letters are still unresolved — requeue or resolve them through the admin outbox API',
+        {
+          requestId,
+          count: previouslyReported.length,
+          events: previouslyReported.map(describeDeadLetter),
         }
       )
     }

apps/sim/app/api/v1/admin/outbox/[id]/resolve/route.ts

@@ -0,0 +1,61 @@
+import { createLogger } from '@sim/logger'
+import { toError } from '@sim/utils/errors'
+import { NextResponse } from 'next/server'
+import { resolveBillingSyncDeadLetter } from '@/lib/admin/billing-sync-resolution'
+import {
+  type AdminV1ResolveOutboxEventResponse,
+  adminV1ResolveOutboxEventContract,
+} from '@/lib/api/contracts/v1/admin'
+import { getValidationErrorMessage, parseRequest } from '@/lib/api/server'
+import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
+import { withAdminAuthParams } from '@/app/api/v1/admin/middleware'
+
+const logger = createLogger('AdminOutboxResolveAPI')
+
+export const dynamic = 'force-dynamic'
+
+const invalidOutboxEventResponse = (message: string) =>
+  NextResponse.json({ success: false, error: message }, { status: 400 })
+
+/**
+ * POST /api/v1/admin/outbox/[id]/resolve
+ *
+ * Close a dead-lettered billing Stripe event an operator has remediated by hand or decided needs
+ * nothing (see `resolveBillingSyncDeadLetter`). Use `/requeue` to retry one instead.
+ *
+ * Body: `{ "reason": string, "resolvedBy": string }`
+ */
+export const POST = withRouteHandler(
+  withAdminAuthParams<{ id: string }>(async (request, context) => {
+    const parsed = await parseRequest(adminV1ResolveOutboxEventContract, request, context, {
+      validationErrorResponse: (error) =>
+        invalidOutboxEventResponse(getValidationErrorMessage(error, 'Invalid resolve request')),
+      invalidJsonResponse: () => invalidOutboxEventResponse('Request body must be valid JSON'),
+    })
+    if (!parsed.success) return parsed.response
+
+    const { id } = parsed.data.params
+
+    try {
+      const resolved = await resolveBillingSyncDeadLetter(id, parsed.data.body, request)
+      if (!resolved) {
+        return NextResponse.json(
+          {
+            success: false,
+            error:
+              'Event not found, not in dead_letter status, or not a billing Stripe event. Only dead-lettered billing Stripe events can be resolved.',
+          },
+          { status: 404 }
+        )
+      }
+
+      return NextResponse.json<AdminV1ResolveOutboxEventResponse>({ success: true, resolved })
+    } catch (error) {
+      logger.error('Failed to resolve outbox event', { eventId: id, error: toError(error).message })
+      return NextResponse.json(
+        { success: false, error: 'Failed to resolve outbox event' },
+        { status: 500 }
+      )
+    }
+  })
+)

apps/sim/lib/admin/billing-sync-resolution.ts

@@ -0,0 +1,36 @@
+import { AuditAction, type AuditLogParams, AuditResourceType, recordAudit } from '@sim/audit'
+import type { AdminV1ResolveOutboxEventBody } from '@/lib/api/contracts/v1/admin'
+import { OUTBOX_EVENT_TYPES } from '@/lib/billing/webhooks/outbox-events'
+import { resolveDeadLetteredOutboxEvent } from '@/lib/core/outbox/service'
+
+/**
+ * Billing Stripe events only: their readers tell a resolved row from delivered work (the admin
+ * dashboard reports a completed cancellation with a `lastError` as `resolved`). Other domains
+ * count every `completed` row as delivered and retry through their own admin actions.
+ */
+const RESOLVABLE_EVENT_TYPES = Object.values(OUTBOX_EVENT_TYPES)
+
+/**
+ * Closes a dead-lettered billing Stripe event for the Admin API and audits who resolved it and
+ * why. Null when the event does not exist, is not dead-lettered, or is not a billing Stripe event.
+ */
+export async function resolveBillingSyncDeadLetter(
+  eventId: string,
+  resolution: AdminV1ResolveOutboxEventBody,
+  request: AuditLogParams['request']
+) {
+  const resolved = await resolveDeadLetteredOutboxEvent(eventId, RESOLVABLE_EVENT_TYPES, resolution)
+  if (!resolved) return null
+  recordAudit({
+    workspaceId: null,
+    actorId: null,
+    actorName: `Admin API (${resolution.resolvedBy})`,
+    action: AuditAction.BILLING_SYNC_RESOLVED,
+    resourceType: AuditResourceType.BILLING,
+    resourceId: resolved.id,
+    description: `Admin API resolved a dead-lettered ${resolved.eventType} event`,
+    metadata: { eventType: resolved.eventType, ...resolution },
+    request,
+  })
+  return resolved
+}

apps/sim/lib/admin/subscription-lifecycle.ts

@@ -177,6 +177,24 @@ async function listRecentSubscriptionPayments(stripe: Stripe, stripeSubscription
   }
 }
 
+/**
+ * Where a dashboard cancellation event stands. A completed event with a `lastError` is a dead
+ * letter an operator resolved without the cancellation reaching Stripe, so it reads as `resolved`
+ * with that note, never as `applied`.
+ */
+function cancellationSyncOutcome(status: string, lastError: string | null) {
+  if (status === 'completed') {
+    return lastError
+      ? { status: 'resolved' as const, error: lastError }
+      : { status: 'applied' as const, error: null }
+  }
+  if (status === 'dead_letter') return { status: 'failed' as const, error: lastError }
+  return {
+    status: status === 'processing' ? ('processing' as const) : ('pending' as const),
+    error: null,
+  }
+}
+
 async function getDashboardCancellationSync(organizationId: string, subscriptionId: string) {
   const [row] = await db
     .select({
@@ -205,15 +223,7 @@ async function getDashboardCancellationSync(organizationId: string, subscription
       row.eventType === OUTBOX_EVENT_TYPES.STRIPE_CANCEL_SUBSCRIPTION_IMMEDIATELY
         ? ('immediate' as const)
         : ('period_end' as const),
-    status:
-      row.status === 'completed'
-        ? ('applied' as const)
-        : row.status === 'dead_letter'
-          ? ('failed' as const)
-          : row.status === 'processing'
-            ? ('processing' as const)
-            : ('pending' as const),
-    error: row.status === 'dead_letter' ? row.error : null,
+    ...cancellationSyncOutcome(row.status, row.error),
   }
 }
 
@@ -286,6 +296,7 @@ export async function requestDashboardSubscriptionCancellation({
         id: outboxEvent.id,
         eventType: outboxEvent.eventType,
         status: outboxEvent.status,
+        lastError: outboxEvent.lastError,
         subscriptionId: sql<string>`${outboxEvent.payload} ->> 'subscriptionId'`,
         reason: sql<string | null>`${outboxEvent.payload} ->> 'reason'`,
       })
@@ -351,12 +362,8 @@ export async function requestDashboardSubscriptionCancellation({
         operationId,
         outboxEventId: existingOperation.id,
         subscriptionId: existingOperation.subscriptionId,
-        status:
-          existingOperation.status === 'completed'
-            ? ('applied' as const)
-            : existingOperation.status === 'processing'
-              ? ('processing' as const)
-              : ('pending' as const),
+        status: cancellationSyncOutcome(existingOperation.status, existingOperation.lastError)
+          .status,
       }
     }
 

apps/sim/lib/api/contracts/v1/admin/dashboard.ts

@@ -663,7 +663,7 @@ const adminDashboardBillingActionsSchema = z.object({
     .object({
       operationId: z.string().uuid(),
       timing: z.enum(['period_end', 'immediate']),
-      status: z.enum(['pending', 'processing', 'applied', 'failed']),
+      status: z.enum(['pending', 'processing', 'applied', 'resolved', 'failed']),
       error: z.string().nullable(),
     })
     .nullable(),
@@ -684,7 +684,7 @@ const adminDashboardBillingActionsSchema = z.object({
 const adminDashboardCancellationResultSchema = z.object({
   success: z.literal(true),
   operationId: z.string(),
-  status: z.enum(['pending', 'processing', 'applied', 'failed']),
+  status: z.enum(['pending', 'processing', 'applied', 'resolved', 'failed']),
 })
 const adminDashboardRefundResultSchema = z.object({
   success: z.literal(true),

apps/sim/lib/api/contracts/v1/admin/outbox.ts

@@ -1,5 +1,6 @@
 import { z } from 'zod'
 import {
+  type ContractBody,
   type ContractJsonResponse,
   type ContractParams,
   type ContractQuery,
@@ -62,6 +63,29 @@ const adminV1OutboxRequeueResultSchema = z.object({
   }),
 })
 
+/** Bounded so the 500-character `last_error` keeps at least 100 characters of the last failure. */
+const adminV1ResolveOutboxEventBodySchema = z.object({
+  reason: z
+    .string({ error: 'reason is required' })
+    .trim()
+    .min(1, 'reason is required')
+    .max(300, 'reason must be at most 300 characters'),
+  resolvedBy: z
+    .string({ error: 'resolvedBy is required' })
+    .trim()
+    .min(1, 'resolvedBy is required')
+    .max(60, 'resolvedBy must be at most 60 characters'),
+})
+
+const adminV1OutboxResolveResultSchema = z.object({
+  success: z.literal(true),
+  resolved: z.object({
+    id: z.string(),
+    eventType: z.string(),
+    lastError: z.string().nullable(),
+  }),
+})
+
 export const adminV1ListOutboxContract = defineRouteContract({
   method: 'GET',
   path: '/api/v1/admin/outbox',
@@ -82,6 +106,17 @@ export const adminV1RequeueOutboxEventContract = defineRouteContract({
   },
 })
 
+export const adminV1ResolveOutboxEventContract = defineRouteContract({
+  method: 'POST',
+  path: '/api/v1/admin/outbox/[id]/resolve',
+  params: adminV1IdParamsSchema,
+  body: adminV1ResolveOutboxEventBodySchema,
+  response: {
+    mode: 'json',
+    schema: adminV1OutboxResolveResultSchema,
+  },
+})
+
 export type AdminV1ListOutboxQueryInput = ContractQueryInput<typeof adminV1ListOutboxContract>
 export type AdminV1ListOutboxQuery = ContractQuery<typeof adminV1ListOutboxContract>
 export type AdminV1RequeueOutboxEventParams = ContractParams<
@@ -91,3 +126,7 @@ export type AdminV1ListOutboxResponse = ContractJsonResponse<typeof adminV1ListO
 export type AdminV1RequeueOutboxEventResponse = ContractJsonResponse<
   typeof adminV1RequeueOutboxEventContract
 >
+export type AdminV1ResolveOutboxEventBody = ContractBody<typeof adminV1ResolveOutboxEventContract>
+export type AdminV1ResolveOutboxEventResponse = ContractJsonResponse<
+  typeof adminV1ResolveOutboxEventContract
+>