feat(tables): write order_key on insert, flag-gate delete reindex + query ordering, add backfill

Flag off (default) = identical behavior. Single-insert assigns a fractional
order_key; queryRows orders by order_key when the flag is on; deletes skip the
O(N) reindex when on. Per-table-atomic backfill script populates existing rows.

Author: TheodoreSpeaks

apps/sim/lib/table/service.ts

@@ -18,13 +18,15 @@ import { createLogger } from '@sim/logger'
 import { getPostgresErrorCode } from '@sim/utils/errors'
 import { generateId } from '@sim/utils/id'
 import { and, count, eq, gt, gte, inArray, isNull, ne, or, type SQL, sql } from 'drizzle-orm'
+import { isTablesFractionalOrderingEnabled } from '@/lib/core/config/feature-flags'
 import { MATERIALIZE_CONCURRENCY, mapWithConcurrency } from '@/lib/core/utils/concurrency'
 import { generateRestoreName } from '@/lib/core/utils/restore-name'
 import type { DbOrTx } from '@/lib/db/types'
 import { materializeExecutionData } from '@/lib/logs/execution/trace-store'
 import { COLUMN_TYPES, NAME_PATTERN, TABLE_LIMITS, USER_TABLE_ROWS_SQL_NAME } from './constants'
 import { areGroupDepsSatisfied } from './deps'
 import { CSV_MAX_BATCH_SIZE } from './import'
+import { keyBetween } from './order-key'
 import { buildFilterClause, buildSortClause } from './sql'
 import { fireTableTrigger } from './trigger'
 import type {
@@ -1148,9 +1150,48 @@ export function buildOrderedRowValues(opts: {
 }
 
 /**
- * Inserts a single row in its own transaction: sets timeouts, reserves the
- * position, and inserts. Validation and side-effect dispatch stay with the
- * caller. Capacity is enforced by the `increment_user_table_row_count` trigger.
+ * Computes the fractional `order_key` for a row being inserted at
+ * `requestedPosition` (or appended when omitted). Neighbors are resolved by the
+ * current `position` order — valid because keys are kept consistent with
+ * position order while the flag is off. Caller holds the row-order lock.
+ *
+ * NOTE: flag-on insert-*at a position* will resolve neighbors by `order_key`
+ * (via beforeRowId/afterRowId) once the wire contract carries them; until then
+ * append (the common path) is exact and at-position is position-derived.
+ */
+async function resolveInsertOrderKey(
+  trx: DbTransaction,
+  tableId: string,
+  requestedPosition?: number
+): Promise<string> {
+  const orderKeyAtPosition = async (pos: number): Promise<string | null> => {
+    if (pos < 0) return null
+    const [r] = await trx
+      .select({ orderKey: userTableRows.orderKey })
+      .from(userTableRows)
+      .where(and(eq(userTableRows.tableId, tableId), eq(userTableRows.position, pos)))
+      .limit(1)
+    return r?.orderKey ?? null
+  }
+  if (requestedPosition === undefined) {
+    const [{ maxKey }] = await trx
+      .select({ maxKey: sql<string | null>`max(${userTableRows.orderKey})` })
+      .from(userTableRows)
+      .where(eq(userTableRows.tableId, tableId))
+    return keyBetween(maxKey ?? null, null)
+  }
+  const lo = await orderKeyAtPosition(requestedPosition - 1)
+  const hi = await orderKeyAtPosition(requestedPosition)
+  return keyBetween(lo, hi)
+}
+
+/**
+ * Inserts a single row in its own transaction. Always assigns a fractional
+ * `order_key`. When the fractional-ordering flag is on, `order_key` is
+ * authoritative and `position` is a best-effort append (no O(N) shift); when
+ * off, `position` is reserved as before (shifting to open the slot). Validation
+ * and side-effect dispatch stay with the caller; capacity is enforced by the
+ * `increment_user_table_row_count` trigger.
  */
 async function insertOrderedRow(params: {
   tableId: string
@@ -1164,7 +1205,25 @@ async function insertOrderedRow(params: {
   const { tableId, workspaceId, data, rowId, position, createdBy, now } = params
   const [row] = await db.transaction(async (trx) => {
     await setTableTxTimeouts(trx)
-    const targetPosition = await reserveInsertPosition(trx, tableId, position)
+    await acquireRowOrderLock(trx, tableId)
+    const orderKey = await resolveInsertOrderKey(trx, tableId, position)
+
+    let targetPosition: number
+    if (isTablesFractionalOrderingEnabled) {
+      // order_key is authoritative — keep a best-effort, no-shift position.
+      targetPosition = await nextRowPosition(trx, tableId)
+    } else if (position !== undefined) {
+      const [existing] = await trx
+        .select({ id: userTableRows.id })
+        .from(userTableRows)
+        .where(and(eq(userTableRows.tableId, tableId), eq(userTableRows.position, position)))
+        .limit(1)
+      if (existing) await shiftRowsUpFrom(trx, tableId, position)
+      targetPosition = position
+    } else {
+      targetPosition = await nextRowPosition(trx, tableId)
+    }
+
     return trx
       .insert(userTableRows)
       .values({
@@ -1173,6 +1232,7 @@ async function insertOrderedRow(params: {
         workspaceId,
         data,
         position: targetPosition,
+        orderKey,
         createdAt: now,
         updatedAt: now,
         ...(createdBy ? { createdBy } : {}),
@@ -1211,7 +1271,11 @@ async function deleteOrderedRow(params: {
       )
       .returning({ position: userTableRows.position })
     if (!deleted) return false
-    await shiftRowsDownAfter(trx, tableId, deleted.position)
+    // Fractional ordering: deleting a row never changes another row's order_key,
+    // so the O(N) position reshift is skipped entirely.
+    if (!isTablesFractionalOrderingEnabled) {
+      await shiftRowsDownAfter(trx, tableId, deleted.position)
+    }
     return true
   })
 }
@@ -1246,7 +1310,8 @@ async function deleteOrderedRowsByIds(params: {
         .returning({ id: userTableRows.id, position: userTableRows.position })
       deleted.push(...rows)
     }
-    if (deleted.length > 0) {
+    // Fractional ordering: deletes leave order_key untouched, so no recompaction.
+    if (!isTablesFractionalOrderingEnabled && deleted.length > 0) {
       const minDeletedPos = deleted.reduce(
         (min, r) => (r.position < min ? r.position : min),
         deleted[0].position
@@ -2154,7 +2219,14 @@ export async function queryRows(
     .from(userTableRows)
     .where(whereClause ?? baseConditions)
   if (orderByClause) {
-    query = query.orderBy(orderByClause) as typeof query
+    // Explicit data-column sort: tiebreak by the default order for stability.
+    query = query.orderBy(
+      orderByClause,
+      isTablesFractionalOrderingEnabled ? userTableRows.orderKey : userTableRows.position,
+      userTableRows.id
+    ) as typeof query
+  } else if (isTablesFractionalOrderingEnabled) {
+    query = query.orderBy(userTableRows.orderKey, userTableRows.id) as typeof query
   } else {
     query = query.orderBy(userTableRows.position) as typeof query
   }

apps/sim/scripts/backfill-table-order-keys.ts

@@ -0,0 +1,118 @@
+#!/usr/bin/env bun
+
+/**
+ * Backfills the `order_key` column on `user_table_rows`.
+ *
+ * Row ordering is moving from the contiguous integer `position` to a fractional
+ * string `order_key` (O(1) insert/delete — no reshift/recompact). This script
+ * assigns each existing row a key derived from its current `position` order, so
+ * the new ordering matches today's once the `TABLES_FRACTIONAL_ORDERING` flag is
+ * flipped on.
+ *
+ * Per-table-atomic: each table is keyed inside one transaction holding the same
+ * per-table advisory lock the app uses for inserts, so a concurrent insert can't
+ * interleave. Idempotent: tables already fully keyed are skipped; a table with
+ * any NULL key is fully re-keyed from `position` order (deterministic, so a
+ * re-run after a partial failure is safe).
+ *
+ * Usage:
+ *   DATABASE_URL=... bun run apps/sim/scripts/backfill-table-order-keys.ts
+ *   DATABASE_URL=... bun run apps/sim/scripts/backfill-table-order-keys.ts --dry-run
+ */
+
+import { userTableRows } from '@sim/db/schema'
+import { getErrorMessage } from '@sim/utils/errors'
+import { asc, eq, isNull, sql } from 'drizzle-orm'
+import { drizzle } from 'drizzle-orm/postgres-js'
+import postgres from 'postgres'
+import { nKeysBetween } from '@/lib/table/order-key'
+
+export async function runBackfill(): Promise<void> {
+  const dryRun = process.argv.includes('--dry-run')
+  const connectionString = process.env.DATABASE_URL ?? process.env.POSTGRES_URL
+  if (!connectionString) {
+    console.error('Missing DATABASE_URL or POSTGRES_URL')
+    process.exit(1)
+  }
+
+  const client = postgres(connectionString, {
+    prepare: false,
+    idle_timeout: 20,
+    connect_timeout: 30,
+    max: 5,
+    onnotice: () => {},
+  })
+  const db = drizzle(client)
+
+  const stats = { tables: 0, tablesKeyed: 0, rowsKeyed: 0, failed: 0 }
+
+  try {
+    // Tables that still have at least one un-keyed row.
+    const pending = await db
+      .selectDistinct({ tableId: userTableRows.tableId })
+      .from(userTableRows)
+      .where(isNull(userTableRows.orderKey))
+
+    console.log(
+      `Backfill starting — ${pending.length} table(s) with NULL order_key${dryRun ? ' [DRY RUN]' : ''}`
+    )
+
+    for (const { tableId } of pending) {
+      stats.tables += 1
+      try {
+        const keyed = await db.transaction(async (trx) => {
+          // Serialize with concurrent inserts on this table (same lock the app uses).
+          await trx.execute(
+            sql`SELECT pg_advisory_xact_lock(hashtextextended(${`user_table_rows_pos:${tableId}`}, 0))`
+          )
+          const rows = await trx
+            .select({ id: userTableRows.id })
+            .from(userTableRows)
+            .where(eq(userTableRows.tableId, tableId))
+            .orderBy(asc(userTableRows.position), asc(userTableRows.id))
+
+          if (rows.length === 0) return 0
+          const keys = nKeysBetween(null, null, rows.length)
+          if (dryRun) return rows.length
+
+          // One UPDATE … FROM (VALUES …) mapping id → key.
+          const values = sql.join(
+            rows.map((r, i) => sql`(${r.id}, ${keys[i]})`),
+            sql`, `
+          )
+          await trx.execute(sql`
+            UPDATE user_table_rows AS t
+            SET order_key = v.order_key
+            FROM (VALUES ${values}) AS v(id, order_key)
+            WHERE t.id = v.id AND t.table_id = ${tableId}
+          `)
+          return rows.length
+        })
+        stats.tablesKeyed += 1
+        stats.rowsKeyed += keyed
+        console.log(`  ${tableId}: keyed ${keyed} rows`)
+      } catch (error) {
+        stats.failed += 1
+        console.error(`  ${tableId}: FAILED — ${getErrorMessage(error)}`)
+      }
+    }
+
+    console.log('Backfill complete.')
+    console.log(`  tables scanned: ${stats.tables}`)
+    console.log(`  tables keyed:   ${stats.tablesKeyed}`)
+    console.log(`  rows keyed:     ${stats.rowsKeyed}`)
+    console.log(`  failed:         ${stats.failed}`)
+    if (stats.failed > 0) process.exitCode = 1
+  } finally {
+    await client.end({ timeout: 5 }).catch(() => {})
+  }
+}
+
+if ((import.meta as { main?: boolean }).main) {
+  try {
+    await runBackfill()
+  } catch (error) {
+    console.error('Backfill aborted:', getErrorMessage(error))
+    process.exitCode = 1
+  }
+}