Skip to content

Commit d8a9627

Browse files
committed
feat(projects): commit membership authority before reconciliation
1 parent d0a9792 commit d8a9627

15 files changed

Lines changed: 339 additions & 11 deletions

‎.github/scripts/check-project-rollout.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
#!/usr/bin/env python3
22
"""Read-only, fail-closed ECS retirement check for the Project column contract migration.
33
4-
The expected digest acknowledges fully deployed #8830 column-only creation/reassignment
5-
and retirement of pre-8830 servers and relevant old worker jobs. Every live and supported
4+
The expected digest acknowledges fully deployed #8830 transaction-barrier and durable-authority support
5+
and retirement of every incompatible server and relevant old worker job before switching. Every live and supported
66
rollback application must tolerate project_workspace being absent.
77
AWS checks below independently verify ECS retirement, not worker drainage.
88
"""
@@ -43,7 +43,7 @@ def matches_release(execution, digest):
4343

4444
def verify(environment, region, digest):
4545
if not re.fullmatch(r'sha256:[0-9a-f]{64}', digest):
46-
raise RuntimeError('Set the environment-specific PROJECT_COLUMN_ENFORCEMENT_READY_IMAGE_DIGEST after verifying #8830 connector-independent readers, column-only writers and pre-8830 server/worker drainage')
46+
raise RuntimeError('Set the environment-specific PROJECT_COLUMN_ENFORCEMENT_READY_IMAGE_DIGEST after verifying #8830 authority-aware readers/writers, transaction barriers and all incompatible server/worker drainage')
4747
pipeline = f'sim-{environment}-{region}-app-deployment'
4848
execution = latest_execution(region, pipeline)
4949
if not matches_release(execution, digest):

‎.github/workflows/migrate.yml‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -76,10 +76,11 @@ jobs:
7676
role-to-assume: ${{ inputs.environment == 'production' && secrets.AWS_ROLE_TO_ASSUME || secrets.STAGING_AWS_ROLE_TO_ASSUME }}
7777
aws-region: ${{ inputs.environment == 'production' && secrets.AWS_REGION || secrets.STAGING_AWS_REGION }}
7878

79-
# Acknowledge #8830 only after connector-independent readers and column-only writers deploy and
80-
# pre-8830 servers and relevant old Trigger.dev runs have drained. The registered Project migration backfills, validates, and removes the connector.
79+
# Acknowledge #8830 only after authority-aware readers/writers deploy and all incompatible
80+
# servers and relevant old Trigger.dev runs drain. The runner commits the column-authority
81+
# switch before backfill, validation, and connector retirement; retries never revert it.
8182
# The preflight independently checks ECS retirement; it does not inspect worker runs.
82-
- name: Require completed column-only rollout before Project enforcement
83+
- name: Require completed authority-aware rollout before Project cutover
8384
if: steps.project-contract.outputs.required == 'true'
8485
env:
8586
ENVIRONMENT: ${{ inputs.environment }}

‎apps/sim/lib/projects/__integration__/backfill-repair.integration.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import { createServer } from 'node:http'
44
import { tmpdir } from 'node:os'
55
import { join } from 'node:path'
66
import { promisify } from 'node:util'
7+
import { switchProjectMembershipAuthority } from '@sim/db/maintenance/project-rollout'
78
import journal from '@sim/db/migrations/meta/_journal.json'
89
import { readTestDatabaseUrl, readTestRedisUrl } from '@sim/db/testing/test-infrastructure'
910
import { createDeferred } from '@sim/testing/helpers/deferred'
@@ -111,6 +112,17 @@ describeWithRedis('Operator archive repair against the full compatible schema',
111112
timeout: 120000,
112113
}
113114
)
115+
await client.unsafe(`CREATE TABLE IF NOT EXISTS project_membership_rollout (
116+
id text PRIMARY KEY CHECK (id = 'membership'),
117+
phase text NOT NULL DEFAULT 'connector' CONSTRAINT project_membership_rollout_phase CHECK (phase IN ('connector', 'column'))
118+
)`)
119+
await client`INSERT INTO project_membership_rollout (id, phase) VALUES ('membership', 'connector') ON CONFLICT DO NOTHING`
120+
await client`SELECT pg_advisory_lock(hashtextextended('sim:project-backfill-operator', 0))`
121+
try {
122+
await switchProjectMembershipAuthority(client)
123+
} finally {
124+
await client`SELECT pg_advisory_unlock(hashtextextended('sim:project-backfill-operator', 0))`
125+
}
114126
}, 120000)
115127

116128
afterAll(async () => {

‎apps/sim/lib/projects/backfill-repair.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import {
1010
completeProjectArchiveRepair,
1111
ensureProjectArchiveRepairJournal,
1212
} from '@sim/db/maintenance/project-repairs'
13+
import { assertProjectColumnAuthority } from '@sim/db/maintenance/project-rollout'
1314
import * as schema from '@sim/db/schema'
1415
import { and, eq, inArray, isNull, sql } from 'drizzle-orm'
1516
import { drizzle } from 'drizzle-orm/postgres-js'
@@ -31,6 +32,7 @@ export async function repairArchivedProjectEnvironment(
3132
repair: ProjectArchiveRepair,
3233
requestId: string
3334
): Promise<void> {
35+
await assertProjectColumnAuthority(client)
3436
await ensureProjectArchiveRepairJournal(client)
3537
const connection = drizzle({ client, schema })
3638
const effects = await connection.transaction(async (tx) => {
@@ -146,6 +148,7 @@ export async function repairProjectGrouping(
146148
client: Sql,
147149
review: ProjectGroupingReview
148150
): Promise<void> {
151+
await assertProjectColumnAuthority(client)
149152
const rootId = review.detachRootId
150153
if (
151154
review.decision !== 'detach' ||

‎apps/sim/scripts/backfill-projects.ts‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
#!/usr/bin/env bun
2+
23
import { createHash } from 'node:crypto'
34
import { link, open, readFile, rename, stat, unlink } from 'node:fs/promises'
45
import { resolve } from 'node:path'
@@ -13,6 +14,7 @@ import {
1314
readProjectGroupingReviews,
1415
verifyProjectBackfill,
1516
} from '@sim/db/maintenance/project-backfill'
17+
import { assertProjectColumnAuthority } from '@sim/db/maintenance/project-rollout'
1618
import { withUtcTimestamps } from '@sim/db/timestamps'
1719
import { createLogger } from '@sim/logger'
1820
import { describeError, getTransientDatabaseFailure } from '@sim/utils/errors'
@@ -168,7 +170,7 @@ on an authorized operator host with that private manifest. Direct invocation doe
168170
AWS preflight: freshly verify the compatible deployed release, old app/worker drainage and release
169171
coordination first. Prior failed CI is not continuing proof if deployment state changed. Once the
170172
runner succeeds and writes its normal receipt, the ordinary deployment retry needs no local file.
171-
The registered migration backfills remaining assignments, verifies, then enforces.\nExit: 0 complete, 2 running/paused/incomplete, 1 failure. Use db:migrate for routine backfill and enforcement after writer drainage.\n`)
173+
The registered migration first commits the durable column-authority switch after draining incompatible writers, then backfills remaining assignments, verifies, and enforces. A failed migration never returns to connector mode. Run db:migrate once before repair/apply; its committed switch remains effective if discovery requires remediation.\nExit: 0 complete, 2 running/paused/incomplete, 1 failure. Use db:migrate for routine backfill and enforcement after writer drainage.\n`)
172174
return
173175
}
174176
const [command] = positionals
@@ -242,6 +244,7 @@ The registered migration backfills remaining assignments, verifies, then enforce
242244
await sql`SELECT pg_try_advisory_lock(hashtextextended('sim:project-backfill-operator',0)) AS held, pg_backend_pid() AS pid`
243245
if (!lock.held) throw new Error('Another Project preparation or enforcement runner is active')
244246
lockPid = lock.pid
247+
if (live) await assertProjectColumnAuthority(sql)
245248
if (command === 'plan') {
246249
if (!manifestPath) throw new Error('plan requires --manifest')
247250
const reviews = values.review
@@ -295,6 +298,11 @@ The registered migration backfills remaining assignments, verifies, then enforce
295298
new URL('../../../packages/db/maintenance/project-backfill.ts', import.meta.url)
296299
)
297300
)
301+
.update(
302+
await readFile(
303+
new URL('../../../packages/db/maintenance/project-rollout.ts', import.meta.url)
304+
)
305+
)
298306
.update(await readFile(new URL('../lib/projects/backfill-repair.ts', import.meta.url)))
299307
.digest('hex')
300308
let progress: z.infer<typeof progressSchema>

‎packages/db/maintenance/project-backfill.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { createHash } from 'node:crypto'
22
import { readFile, stat } from 'node:fs/promises'
33
import { countPendingProjectArchiveRepairs } from '@sim/db/maintenance/project-repairs'
4+
import { assertProjectColumnAuthority } from '@sim/db/maintenance/project-rollout'
45
import { createLogger } from '@sim/logger'
56
import { generateId } from '@sim/utils/id'
67
import { isRecordLike } from '@sim/utils/object'
@@ -471,6 +472,7 @@ export async function assignProjectBackfillBatch(sql: Sql, families: ProjectBack
471472
await tx`SET LOCAL statement_timeout = '3s'`
472473
await tx`SET LOCAL lock_timeout = '250ms'`
473474
const legacy = await hasLegacyMemberships(tx)
475+
await assertProjectColumnAuthority(tx)
474476
const roots = families.map((f) => f.rootId).sort(compareStrings)
475477
const expected = families.flatMap((f) => f.members).sort((a, b) => compareStrings(a.id, b.id))
476478
const ids = expected.map((row) => row.id)

‎packages/db/maintenance/project-membership.sql‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,12 @@ COMMIT;
44
SET statement_timeout = '60s';
55
--> statement-breakpoint
66
DO $$ BEGIN
7+
IF to_regclass('public.project_membership_rollout') IS NULL THEN
8+
RAISE EXCEPTION 'Project membership rollout marker is missing' USING ERRCODE = '55000';
9+
END IF;
10+
IF NOT EXISTS (SELECT 1 FROM public.project_membership_rollout WHERE id = 'membership' AND phase = 'column') THEN
11+
RAISE EXCEPTION 'Project membership authority must switch before enforcement' USING ERRCODE = '55000';
12+
END IF;
713
IF NOT pg_try_advisory_lock(hashtextextended('sim:project-backfill-operator', 0)) THEN
814
RAISE EXCEPTION 'Project preparation is still running; finish it before enforcement' USING ERRCODE = '55000';
915
END IF;
Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
import { getTransientDatabaseFailure } from '@sim/utils/errors'
2+
import { sleep } from '@sim/utils/helpers'
3+
import { backoffWithJitter } from '@sim/utils/retry'
4+
import type { Sql, TransactionSql } from 'postgres'
5+
6+
type ProjectMembershipPhase = 'connector' | 'column'
7+
8+
/** Reads durable authority without inferring it from partially copied membership. */
9+
export async function readProjectMembershipPhase(
10+
sql: Sql | TransactionSql
11+
): Promise<ProjectMembershipPhase> {
12+
const [table] =
13+
await sql`SELECT to_regclass('public.project_membership_rollout') IS NOT NULL AS present`
14+
if (!table?.present) throw new Error('Project membership rollout marker is missing')
15+
const rows = await sql<
16+
{ id: string; phase: string }[]
17+
>`SELECT id, phase FROM public.project_membership_rollout`
18+
const row = rows[0]
19+
if (rows.length !== 1 || row?.id !== 'membership' || !['connector', 'column'].includes(row.phase))
20+
throw new Error('Project membership rollout marker is missing or invalid')
21+
return row.phase as ProjectMembershipPhase
22+
}
23+
24+
/** Mutating maintenance never writes columns while compatible applications still use the connector. */
25+
export async function assertProjectColumnAuthority(sql: Sql | TransactionSql): Promise<void> {
26+
if ((await readProjectMembershipPhase(sql)) !== 'column')
27+
throw new Error(
28+
'Project membership authority must switch through db:migrate after verified server/worker drainage before maintenance writes'
29+
)
30+
}
31+
32+
/** Commits the forward-only authority change before any discovery, backfill, or repair. */
33+
export async function switchProjectMembershipAuthority(sql: Sql): Promise<void> {
34+
for (let attempt = 0; ; attempt++) {
35+
try {
36+
await sql.begin(async (tx) => {
37+
await tx`SET LOCAL statement_timeout = '3s'`
38+
await tx`SELECT set_config(CASE WHEN current_setting('transaction_timeout', true) IS NULL
39+
THEN 'idle_in_transaction_session_timeout' ELSE 'transaction_timeout' END, '5s', true)`
40+
await tx`LOCK TABLE public.workspace IN ACCESS EXCLUSIVE MODE NOWAIT`
41+
const [operator] = await tx`SELECT EXISTS (SELECT 1 FROM pg_locks
42+
WHERE pid = pg_backend_pid() AND locktype = 'advisory' AND granted
43+
AND classid = ((hashtextextended('sim:project-backfill-operator',0) >> 32) & 4294967295)::oid
44+
AND objid = (hashtextextended('sim:project-backfill-operator',0) & 4294967295)::oid
45+
AND objsubid = 1) AS held`
46+
if (!operator?.held)
47+
throw new Error('Project authority switch requires the operator session lock')
48+
const phase = await readProjectMembershipPhase(tx)
49+
if (phase === 'column') return
50+
const [state] = await tx`SELECT
51+
to_regclass('public.project_workspace') IS NOT NULL AS connector,
52+
EXISTS (SELECT 1 FROM public.workspace WHERE project_id IS NOT NULL) AS populated`
53+
if (!state?.connector || state.populated)
54+
throw new Error(
55+
'Connector authority requires an intact connector and no populated Project columns; reconcile the rollout state before switching'
56+
)
57+
await tx`UPDATE public.project_membership_rollout SET phase = 'column' WHERE id = 'membership'`
58+
})
59+
return
60+
} catch (error) {
61+
const transient = getTransientDatabaseFailure(error)
62+
if (attempt >= 2 || (transient !== 'capacity' && transient !== 'conflict')) throw error
63+
await sleep(backoffWithJitter(attempt, null, { baseMs: 100, maxMs: 500 }))
64+
}
65+
}
66+
}
67+
68+
/** Only an empty final schema push may initialize a missing singleton directly in column mode. */
69+
export async function bootstrapProjectColumnAuthority(sql: Sql): Promise<void> {
70+
await sql.begin(async (tx) => {
71+
await tx`SET LOCAL statement_timeout = '3s'`
72+
await tx`LOCK TABLE public.workspace IN ACCESS EXCLUSIVE MODE NOWAIT`
73+
const [state] = await tx`SELECT
74+
to_regclass('public.project_workspace') IS NULL AS retired,
75+
EXISTS (SELECT 1 FROM pg_attribute WHERE attrelid = 'public.workspace'::regclass
76+
AND attname = 'project_id' AND attnotnull AND NOT attisdropped) AS required,
77+
NOT EXISTS (SELECT 1 FROM public.workspace) AND NOT EXISTS (SELECT 1 FROM public.project) AS empty`
78+
if (!state?.retired || !state.required)
79+
throw new Error('Project authority bootstrap requires the fresh final schema')
80+
const [table] =
81+
await tx`SELECT to_regclass('public.project_membership_rollout') IS NOT NULL AS present`
82+
if (!table?.present)
83+
throw new Error('Project membership rollout table is missing from the final schema')
84+
const rows = await tx`SELECT id, phase FROM public.project_membership_rollout`
85+
if (!rows.length && state.empty)
86+
await tx`INSERT INTO public.project_membership_rollout (id, phase) VALUES ('membership', 'column')`
87+
await assertProjectColumnAuthority(tx)
88+
})
89+
}
Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
1-
-- The registered 0031_project_membership script backfills and verifies before installing
1+
-- The registered 0031_project_membership script commits the column-authority switch, then backfills and verifies before installing
22
-- the final constraints and retiring the connector. SQL migrations run before scripts.
33
SELECT 1;

‎packages/db/script-migrations/0031_project_membership.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import {
1010
} from '@sim/db/maintenance/project-backfill'
1111
import { enforceProjectMembership } from '@sim/db/maintenance/project-enforcement'
1212
import { countPendingProjectArchiveRepairs } from '@sim/db/maintenance/project-repairs'
13+
import { switchProjectMembershipAuthority } from '@sim/db/maintenance/project-rollout'
1314
import type { ScriptMigration } from '@sim/db/script-migrations/types'
1415
import { createLogger } from '@sim/logger'
1516
import { describeError, getTransientDatabaseFailure } from '@sim/utils/errors'
@@ -46,6 +47,8 @@ export const projectMembershipMigration: ScriptMigration = {
4647
}
4748
try {
4849
await assertProjectBackfillDatabase(sql, true)
50+
await switchProjectMembershipAuthority(sql)
51+
await assertSession()
4952
if (await countPendingProjectArchiveRepairs(sql)) {
5053
throw new Error(
5154
'Project archive cleanup is incomplete; resume repair with the original manifest before retrying'

0 commit comments

Comments
 (0)