feat(cli, migrate): add stash encrypt commands + @cipherstash/migrate

.changeset/encryption-migrations.md

@@ -0,0 +1,19 @@
+---
+'@cipherstash/cli': minor
+'@cipherstash/migrate': minor
+---
+
+Add `stash encrypt` command group and `@cipherstash/migrate` library for plaintext → encrypted column migrations.
+
+New CLI commands:
+
+- `stash encrypt status` — per-column migration status (phase, backfill progress, drift between intent and state, EQL registration).
+- `stash encrypt plan` — diff `.cipherstash/migrations.json` (intent) vs observed state.
+- `stash encrypt advance --to <phase> --table <t> --column <c>` — record a phase transition (`schema-added` / `dual-writing` / `backfilling` / `backfilled` / `cut-over` / `dropped`).
+- `stash encrypt backfill --table <t> --column <c>` — resumable, idempotent, chunked encryption of plaintext into `<col>_encrypted`. Uses the user's encryption client (Protect/Stack). SIGINT-safe; re-run to resume.
+- `stash encrypt cutover --table <t> --column <c>` — runs `eql_v2.rename_encrypted_columns()` inside a transaction; optionally forces Proxy config refresh via `CIPHERSTASH_PROXY_URL`. After cutover, apps reading `<col>` transparently receive the encrypted column.
+- `stash encrypt drop --table <t> --column <c>` — generates a migration file that drops the old plaintext column.
+
+`stash db install` now also installs a `cipherstash.cs_migrations` table used to track per-column migration runtime state (current phase, backfill cursor, rows processed). The table is append-only (event-log shape) and kept separate from `eql_v2_configuration` which remains the authoritative EQL intent store used by Proxy.
+
+The new `@cipherstash/migrate` package exposes the same primitives as a library for users who want to embed backfill in their own workers or cron jobs — all commands are thin wrappers around its exports (`runBackfill`, `appendEvent`, `latestByColumn`, `progress`, `renameEncryptedColumns`, `reloadConfig`, `readManifest`, `writeManifest`).

packages/cli/package.json

@@ -42,6 +42,7 @@
 	"dependencies": {
 		"@anthropic-ai/claude-agent-sdk": "^0.2.87",
 		"@cipherstash/auth": "catalog:repo",
+		"@cipherstash/migrate": "workspace:*",
 		"@clack/prompts": "0.10.1",
 		"dotenv": "16.4.7",
 		"jiti": "2.6.1",

packages/cli/scripts/e2e-encrypt.sh

@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+# End-to-end smoke test for `stash encrypt`.
+#
+# Requires a local Postgres you have superuser on. Creates & destroys
+# `stash_e2e_test`. Requires CipherStash credentials in the environment
+# for the actual encryption step (CS_CLIENT_ACCESS_KEY etc).
+#
+# Usage: bash packages/cli/scripts/e2e-encrypt.sh
+
+set -euo pipefail
+
+DB=${STASH_E2E_DB:-stash_e2e_test}
+HOST=${STASH_E2E_HOST:-localhost}
+DATABASE_URL="postgres://${USER}@${HOST}/${DB}"
+STASH="$(cd "$(dirname "$0")/../dist/bin" && pwd)/stash.js"
+FIXTURES="$(cd "$(dirname "$0")/fixtures" && pwd)"
+
+if [ ! -x "$STASH" ]; then
+  echo "CLI not built. Run: pnpm --filter @cipherstash/cli build" >&2
+  exit 1
+fi
+
+psql -h "$HOST" -d postgres -c "DROP DATABASE IF EXISTS ${DB}" >/dev/null
+psql -h "$HOST" -d postgres -c "CREATE DATABASE ${DB}" >/dev/null
+
+export DATABASE_URL
+
+echo "==> 1. Install EQL + cs_migrations"
+"$STASH" db install --force
+
+echo "==> 2. Seed 5000 plaintext users"
+psql "$DATABASE_URL" -f "$FIXTURES/seed-users.sql" >/dev/null
+psql "$DATABASE_URL" -c "ALTER TABLE users ADD COLUMN email_encrypted eql_v2_encrypted" >/dev/null
+
+echo "==> 3. Record dual-writing"
+"$STASH" encrypt advance --to dual-writing --table users --column email
+
+echo "==> 4. Backfill with interrupt/resume"
+"$STASH" encrypt backfill --table users --column email --chunk-size 500 &
+PID=$!
+sleep 2
+kill -INT "$PID" || true
+wait "$PID" || true
+"$STASH" encrypt backfill --table users --column email
+
+REMAINING=$(psql "$DATABASE_URL" -At -c "SELECT count(*) FROM users WHERE email_encrypted IS NULL")
+if [ "$REMAINING" != "0" ]; then
+  echo "FAIL: ${REMAINING} rows still unencrypted" >&2
+  exit 1
+fi
+echo "OK: all 5000 rows encrypted"
+
+echo "==> 5. Status"
+"$STASH" encrypt status
+
+echo "==> 6. Cutover"
+"$STASH" encrypt cutover --table users --column email
+
+echo "==> 7. Drop"
+"$STASH" encrypt drop --table users --column email --migrations-dir "$(pwd)/drizzle"
+
+echo "==> Done."

packages/cli/scripts/fixtures/seed-users.sql

@@ -0,0 +1,16 @@
+-- Seed a users table with plaintext emails for e2e backfill testing.
+--
+-- The encrypted target column must be created separately (drizzle-kit /
+-- stash db push route), after which the backfill encrypts `email` → `email_encrypted`.
+
+DROP TABLE IF EXISTS users CASCADE;
+
+CREATE TABLE users (
+  id        bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
+  email     text NOT NULL,
+  created_at timestamptz NOT NULL DEFAULT now()
+);
+
+INSERT INTO users (email)
+SELECT 'user-' || g || '@example.com'
+FROM generate_series(1, 5000) AS g;

packages/cli/src/bin/stash.ts

@@ -73,6 +73,13 @@ Commands:
 
   schema build         Build an encryption schema from your database
 
+  encrypt status       Show per-column migration status (phase, progress, drift)
+  encrypt plan         Diff intent (.cipherstash/migrations.json) vs observed state
+  encrypt advance      Record a phase transition for a column
+  encrypt backfill     Resumably encrypt plaintext into the encrypted column
+  encrypt cutover      Rename swap encrypted → primary column
+  encrypt drop         Generate a migration to drop the plaintext column
+
   env                  (experimental) Print production env vars for deployment
 
 Options:
@@ -198,6 +205,99 @@ async function runDbCommand(
   }
 }
 
+async function runEncryptCommand(
+  sub: string | undefined,
+  flags: Record<string, boolean>,
+  values: Record<string, string>,
+) {
+  switch (sub) {
+    case 'status': {
+      const { statusCommand } = await requireStack(
+        () => import('../commands/encrypt/status.js'),
+      )
+      await statusCommand()
+      break
+    }
+    case 'plan': {
+      const { planCommand } = await requireStack(
+        () => import('../commands/encrypt/plan.js'),
+      )
+      await planCommand()
+      break
+    }
+    case 'advance': {
+      const table = requireValue(values, 'table')
+      const column = requireValue(values, 'column')
+      const to = requireValue(values, 'to') as
+        | 'schema-added'
+        | 'dual-writing'
+        | 'backfilling'
+        | 'backfilled'
+        | 'cut-over'
+        | 'dropped'
+      const { advanceCommand } = await requireStack(
+        () => import('../commands/encrypt/advance.js'),
+      )
+      await advanceCommand({ table, column, to, note: values.note })
+      break
+    }
+    case 'backfill': {
+      const table = requireValue(values, 'table')
+      const column = requireValue(values, 'column')
+      const { backfillCommand } = await requireStack(
+        () => import('../commands/encrypt/backfill.js'),
+      )
+      await backfillCommand({
+        table,
+        column,
+        pkColumn: values['pk-column'],
+        chunkSize: values['chunk-size']
+          ? Number(values['chunk-size'])
+          : undefined,
+        encryptedColumn: values['encrypted-column'],
+        schemaColumnKey: values['schema-column-key'],
+      })
+      break
+    }
+    case 'cutover': {
+      const table = requireValue(values, 'table')
+      const column = requireValue(values, 'column')
+      const { cutoverCommand } = await requireStack(
+        () => import('../commands/encrypt/cutover.js'),
+      )
+      await cutoverCommand({ table, column, proxyUrl: values['proxy-url'] })
+      break
+    }
+    case 'drop': {
+      const table = requireValue(values, 'table')
+      const column = requireValue(values, 'column')
+      const { dropCommand } = await requireStack(
+        () => import('../commands/encrypt/drop.js'),
+      )
+      await dropCommand({
+        table,
+        column,
+        migrationsDir: values['migrations-dir'],
+      })
+      break
+    }
+    default:
+      p.log.error(`Unknown encrypt subcommand: ${sub ?? '(none)'}`)
+      console.log()
+      console.log(HELP)
+      process.exit(1)
+  }
+}
+
+function requireValue(values: Record<string, string>, key: string): string {
+  const v = values[key]
+  if (!v) {
+    p.log.error(`Missing required --${key} value.`)
+    process.exit(1)
+  }
+  return v
+}
+
 async function runSchemaCommand(
   sub: string | undefined,
   flags: Record<string, boolean>,
@@ -255,6 +355,9 @@ async function main() {
     case 'db':
       await runDbCommand(subcommand, flags, values)
       break
+    case 'encrypt':
+      await runEncryptCommand(subcommand, flags, values)
+      break
     case 'schema':
       await runSchemaCommand(subcommand, flags)
       break

packages/cli/src/commands/db/install.ts

@@ -8,7 +8,12 @@ import {
   downloadEqlSql,
   loadBundledEqlSql,
 } from '@/installer/index.js'
+import {
+  MIGRATIONS_SCHEMA_SQL,
+  installMigrationsSchema,
+} from '@cipherstash/migrate'
 import * as p from '@clack/prompts'
+import pg from 'pg'
 import { ensureStashConfig } from './config-scaffold.js'
 import { detectDrizzle, detectSupabase } from './detect.js'
 import { rewriteEncryptedAlterColumns } from './rewrite-migrations.js'
@@ -137,6 +142,23 @@ export async function installCommand(options: InstallOptions) {
     p.log.success('Supabase role permissions granted.')
   }
 
+  s.start('Installing cs_migrations tracking schema...')
+  const migrationsDb = new pg.Client({ connectionString: config.databaseUrl })
+  try {
+    await migrationsDb.connect()
+    await installMigrationsSchema(migrationsDb)
+    s.stop('cs_migrations schema installed.')
+  } catch (err) {
+    s.stop('Failed to install cs_migrations schema.')
+    p.log.warn(
+      err instanceof Error
+        ? err.message
+        : 'Encryption migration tracking is unavailable; `stash encrypt` commands will fail until this is resolved.',
+    )
+  } finally {
+    await migrationsDb.end()
+  }
+
   printNextSteps()
   p.outro('Done!')
 }
@@ -315,11 +337,16 @@ async function generateDrizzleMigration(
     }
   }
 
-  // Step 4: Write the EQL SQL into the migration file
+  // Step 4: Write the EQL SQL (and cs_migrations tracking schema) into
+  // the migration file. Bundling both means `drizzle-kit migrate` rolls
+  // everything needed for `stash encrypt ...` out to each environment
+  // in one go, rather than requiring an out-of-band `stash db install`.
   s.start('Writing EQL SQL into migration file...')
 
+  const migrationContents = `${eqlSql}\n\n-- CipherStash encryption-migration tracking schema.\n-- Tracks per-column phase + backfill progress for \`stash encrypt\`.\n${MIGRATIONS_SCHEMA_SQL.trim()}\n`
+
   try {
-    writeFileSync(generatedMigrationPath, eqlSql, 'utf-8')
+    writeFileSync(generatedMigrationPath, migrationContents, 'utf-8')
     s.stop('EQL SQL written to migration file.')
   } catch (error) {
     s.stop('Failed to write migration file.')

packages/cli/src/commands/encrypt/advance.ts

@@ -0,0 +1,93 @@
+import { loadStashConfig } from '@/config/index.js'
+import { type MigrationPhase, appendEvent } from '@cipherstash/migrate'
+import * as p from '@clack/prompts'
+import pg from 'pg'
+
+/**
+ * Map a user-declared target phase to the event name we write to
+ * `cs_migrations`. `backfilling` is recorded as `backfill_started`; the
+ * phase itself is set to `backfilling` regardless.
+ */
+const PHASE_TO_EVENT: Record<
+  MigrationPhase,
+  | 'schema_added'
+  | 'dual_writing'
+  | 'backfill_started'
+  | 'backfilled'
+  | 'cut_over'
+  | 'dropped'
+> = {
+  'schema-added': 'schema_added',
+  'dual-writing': 'dual_writing',
+  backfilling: 'backfill_started',
+  backfilled: 'backfilled',
+  'cut-over': 'cut_over',
+  dropped: 'dropped',
+}
+
+/**
+ * Options accepted by `stash encrypt advance`. Used to *declare* that a
+ * column has reached a new phase — especially useful for `dual-writing`,
+ * which is an app-code property that the CLI cannot detect automatically.
+ */
+export interface AdvanceCommandOptions {
+  /** Physical table name, e.g. `users`. Supports `schema.table`. */
+  table: string
+  /** Physical plaintext column, e.g. `email`. */
+  column: string
+  /**
+   * The phase the column is transitioning *to*. Records a corresponding
+   * event (see {@link PHASE_TO_EVENT}). Does not enforce an order — you
+   * can move backwards if needed, e.g. to re-run a backfill.
+   */
+  to: MigrationPhase
+  /**
+   * Optional free-form note, stored in the event's `details.note`. Useful
+   * for capturing why a phase transition is happening ("deploy 1.23
+   * introduced dual-write") so it shows up in audit queries later.
+   */
+  note?: string
+}
+
+/**
+ * CLI handler for `stash encrypt advance`. Appends a phase-transition event
+ * to `cs_migrations`. When advancing to `dual-writing`, also prints a
+ * reminder about the required persistence-layer code change.
+ */
+export async function advanceCommand(options: AdvanceCommandOptions) {
+  p.intro('npx @cipherstash/cli encrypt advance')
+
+  const config = await loadStashConfig()
+  const client = new pg.Client({ connectionString: config.databaseUrl })
+
+  try {
+    await client.connect()
+    await appendEvent(client, {
+      tableName: options.table,
+      columnName: options.column,
+      event: PHASE_TO_EVENT[options.to],
+      phase: options.to,
+      details: options.note ? { note: options.note } : null,
+    })
+
+    p.log.success(
+      `${options.table}.${options.column} is now recorded as '${options.to}'.`,
+    )
+
+    if (options.to === 'dual-writing') {
+      p.note(
+        `Update your persistence layer to write this value to both columns:\n  - ${options.column} (plaintext, existing)\n  - ${options.column}_encrypted (ciphertext, via your encryption client)\n\nThen run: stash encrypt backfill --table ${options.table} --column ${options.column}`,
+        'Next',
+      )
+    }
+
+    p.outro('Recorded.')
+  } catch (error) {
+    p.log.error(
+      error instanceof Error ? error.message : 'Failed to record transition.',
+    )
+    process.exit(1)
+  } finally {
+    await client.end()
+  }
+}