From e7606c792ca71a01fdcec5bdc58c0395bffea55d Mon Sep 17 00:00:00 2001 From: Alice Alexandra Moore <86723305+3mdistal@users.noreply.github.com> Date: Wed, 29 Jul 2026 17:41:42 -0400 Subject: [PATCH 01/13] Add advisory Content conformance pilot --- .../workflows/content-product-conformance.yml | 50 ++ package.json | 2 + scripts/validate-content-product-docs.ts | 6 +- ...te-content-product-impact-workflow.test.ts | 58 ++ ...alidate-content-product-impact-workflow.ts | 125 ++++ .../validate-content-product-impact.test.ts | 448 +++++++++++ scripts/validate-content-product-impact.ts | 705 ++++++++++++++++++ .../content-product-development/SKILL.md | 47 ++ 8 files changed, 1438 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/content-product-conformance.yml create mode 100644 scripts/validate-content-product-impact-workflow.test.ts create mode 100644 scripts/validate-content-product-impact-workflow.ts create mode 100644 scripts/validate-content-product-impact.test.ts create mode 100644 scripts/validate-content-product-impact.ts diff --git a/.github/workflows/content-product-conformance.yml b/.github/workflows/content-product-conformance.yml new file mode 100644 index 0000000000..a4ba0cfd1d --- /dev/null +++ b/.github/workflows/content-product-conformance.yml @@ -0,0 +1,50 @@ +name: Content product conformance + +on: + pull_request: + branches: [main] + types: + [ + opened, + synchronize, + reopened, + edited, + ready_for_review, + labeled, + unlabeled, + ] + +permissions: + contents: read + pull-requests: read + +concurrency: + group: content-product-conformance-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + check: + name: Advisory Content product impact + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 + persist-credentials: false + + - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4.4.0 + + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: "22" + cache: pnpm + + - run: pnpm install --frozen-lockfile --ignore-scripts + + - name: Check Content product impact + env: + CONTENT_IMPACT_BASE_SHA: ${{ github.event.pull_request.base.sha }} + CONTENT_IMPACT_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: pnpm content-product-impact diff --git a/package.json b/package.json index d66117ebb3..47de94fd40 100644 --- a/package.json +++ b/package.json @@ -63,6 +63,8 @@ "guard:trusted-acceptance": "tsx scripts/guard-trusted-acceptance-workflow.ts", "guard:content-product-docs": "tsx scripts/validate-content-product-docs.ts", "test:content-product-docs": "tsx --test scripts/validate-content-product-docs.test.ts", + "content-product-impact": "tsx scripts/validate-content-product-impact.ts", + "test:content-product-impact": "tsx --test scripts/validate-content-product-impact.test.ts scripts/validate-content-product-impact-workflow.test.ts", "guard:workspace-skills": "tsx scripts/sync-workspace-core-skills.ts --check", "sync:template-standard": "tsx scripts/template-standard/index.ts", "guard:template-standard": "tsx scripts/template-standard/index.ts --check", diff --git a/scripts/validate-content-product-docs.ts b/scripts/validate-content-product-docs.ts index d1b5a34217..d6c48df98c 100644 --- a/scripts/validate-content-product-docs.ts +++ b/scripts/validate-content-product-docs.ts @@ -50,9 +50,9 @@ const roadmapBoundaries = new Set([ "superseded", ]); -type RecordKind = "chapter" | "feature" | "capability"; +export type RecordKind = "chapter" | "feature" | "capability"; -type ProductRecord = { +export type ProductRecord = { file: string; body: string; data: Record; @@ -60,7 +60,7 @@ type ProductRecord = { id: string; }; -type ProductCatalog = { +export type ProductCatalog = { root: string; chapters: ProductRecord[]; features: ProductRecord[]; diff --git a/scripts/validate-content-product-impact-workflow.test.ts b/scripts/validate-content-product-impact-workflow.test.ts new file mode 100644 index 0000000000..dd6c36f4f4 --- /dev/null +++ b/scripts/validate-content-product-impact-workflow.test.ts @@ -0,0 +1,58 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { describe, it } from "node:test"; + +import { validateContentProductImpactWorkflow } from "./validate-content-product-impact-workflow.ts"; + +const workflow = readFileSync( + ".github/workflows/content-product-conformance.yml", + "utf8", +); + +describe("Content product conformance workflow boundary", () => { + it("keeps the pilot read-only, exact-revision, and broadly observable", () => { + assert.deepEqual(validateContentProductImpactWorkflow(workflow), { + ok: true, + issues: [], + }); + }); + + it("rejects pull_request_target and write permissions", () => { + const unsafe = workflow + .replace(" pull_request:", " pull_request_target:") + .replace("contents: read", "contents: write"); + const result = validateContentProductImpactWorkflow(unsafe); + assert.equal(result.ok, false); + assert( + result.issues.some((issue) => issue.includes("pull_request_target")), + ); + assert(result.issues.some((issue) => issue.includes("permissions"))); + }); + + it("rejects path filtering or a candidate checkout with credentials", () => { + const unsafe = workflow + .replace(" types:", ' paths: ["templates/content/**"]\n types:') + .replace("persist-credentials: false", "persist-credentials: true"); + const result = validateContentProductImpactWorkflow(unsafe); + assert.equal(result.ok, false); + assert(result.issues.some((issue) => issue.includes("path filters"))); + assert(result.issues.some((issue) => issue.includes("credentials"))); + }); + + it("requires declaration and label edits to rerun the pilot", () => { + const unsafe = workflow.replace(" edited,\n", ""); + const result = validateContentProductImpactWorkflow(unsafe); + assert.equal(result.ok, false); + assert(result.issues.some((issue) => issue.includes("recalibration"))); + }); + + it("does not hide checker infrastructure failures", () => { + const unsafe = workflow.replace( + " run: pnpm content-product-impact", + " continue-on-error: true\n run: pnpm content-product-impact", + ); + const result = validateContentProductImpactWorkflow(unsafe); + assert.equal(result.ok, false); + assert(result.issues.some((issue) => issue.includes("infrastructure"))); + }); +}); diff --git a/scripts/validate-content-product-impact-workflow.ts b/scripts/validate-content-product-impact-workflow.ts new file mode 100644 index 0000000000..9272c9d611 --- /dev/null +++ b/scripts/validate-content-product-impact-workflow.ts @@ -0,0 +1,125 @@ +import { parse } from "yaml"; + +export type ContentImpactWorkflowGuardResult = { + ok: boolean; + issues: string[]; +}; + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function permissionIsRead(value: unknown): boolean { + return value === "read"; +} + +export function validateContentProductImpactWorkflow( + source: string, +): ContentImpactWorkflowGuardResult { + const issues: string[] = []; + let workflow: unknown; + try { + workflow = parse(source); + } catch { + return { ok: false, issues: ["workflow must be valid YAML"] }; + } + if (!isRecord(workflow)) { + return { ok: false, issues: ["workflow must be a YAML mapping"] }; + } + + const trigger = workflow.on; + if (!isRecord(trigger) || !isRecord(trigger.pull_request)) { + issues.push("workflow must use pull_request"); + } else { + const pullRequest = trigger.pull_request; + const requiredTypes = [ + "opened", + "synchronize", + "reopened", + "edited", + "ready_for_review", + "labeled", + "unlabeled", + ]; + const eventTypes = pullRequest.types; + if ( + !Array.isArray(eventTypes) || + requiredTypes.some((type) => !eventTypes.includes(type)) + ) { + issues.push( + "pull_request must include every advisory recalibration event", + ); + } + if ("paths" in pullRequest || "paths-ignore" in pullRequest) { + issues.push("workflow must not use path filters"); + } + } + if ("pull_request_target" in (isRecord(trigger) ? trigger : {})) { + issues.push("workflow must not use pull_request_target"); + } + + const permissions = workflow.permissions; + if ( + !isRecord(permissions) || + !permissionIsRead(permissions.contents) || + !permissionIsRead(permissions["pull-requests"]) || + Object.keys(permissions).some( + (key) => key !== "contents" && key !== "pull-requests", + ) + ) { + issues.push( + "permissions must be exactly contents: read and pull-requests: read", + ); + } + + const jobs = workflow.jobs; + const check = isRecord(jobs) ? jobs.check : undefined; + if (!isRecord(check)) { + issues.push("workflow must define the check job"); + return { ok: false, issues }; + } + if ("environment" in check || /\bsecrets\./.test(source)) { + issues.push("advisory check must not receive an environment or secrets"); + } + if (typeof check["timeout-minutes"] !== "number") { + issues.push("check job must have an explicit timeout"); + } + const steps = Array.isArray(check.steps) ? check.steps.filter(isRecord) : []; + const checkout = steps.find( + (step) => + typeof step.uses === "string" && + step.uses.startsWith("actions/checkout@"), + ); + if (!checkout || !isRecord(checkout.with)) { + issues.push("workflow must check out the exact candidate revision"); + } else { + if (checkout.with.ref !== "${{ github.event.pull_request.head.sha }}") { + issues.push("checkout ref must be the exact pull request head SHA"); + } + if (checkout.with["fetch-depth"] !== 0) { + issues.push("checkout must fetch base history"); + } + if (checkout.with["persist-credentials"] !== false) { + issues.push("checkout must not persist GitHub credentials"); + } + } + const runStep = steps.find( + (step) => step.run === "pnpm content-product-impact", + ); + if (!runStep || !isRecord(runStep.env)) { + issues.push("workflow must invoke the standalone impact checker"); + } else if (runStep["continue-on-error"] === true) { + issues.push( + "workflow must not hide checker input or infrastructure failures", + ); + } else if ( + runStep.env.CONTENT_IMPACT_BASE_SHA !== + "${{ github.event.pull_request.base.sha }}" || + runStep.env.CONTENT_IMPACT_HEAD_SHA !== + "${{ github.event.pull_request.head.sha }}" + ) { + issues.push("checker must receive exact base and head SHAs"); + } + + return { ok: issues.length === 0, issues }; +} diff --git a/scripts/validate-content-product-impact.test.ts b/scripts/validate-content-product-impact.test.ts new file mode 100644 index 0000000000..82e494c699 --- /dev/null +++ b/scripts/validate-content-product-impact.test.ts @@ -0,0 +1,448 @@ +import assert from "node:assert/strict"; +import { execFileSync, spawnSync } from "node:child_process"; +import { + cpSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { after, before, describe, it } from "node:test"; + +import { validateContentProductDocs } from "./validate-content-product-docs.ts"; +import { + analyzeContentProductImpact, + collectTransitions, + directContentEvidence, + parseContentImpactDeclaration, +} from "./validate-content-product-impact.ts"; + +const fixture = "scripts/fixtures/content-product-docs/valid"; +const repositoryRoot = process.cwd(); +const checkerPath = path.join( + repositoryRoot, + "scripts/validate-content-product-impact.ts", +); +const tsxPath = path.join(repositoryRoot, "node_modules/.bin/tsx"); +let temporaryRoot = ""; +let baseRoot = ""; +let headRoot = ""; + +function declaration(overrides: string[] = []): string { + return [ + "```yaml", + "content_product_impact:", + " lane: contract_fulfillment", + " features:", + " - content.feature.test", + " capabilities:", + " - content.test.alpha", + " record_change: included", + " proof:", + " - pnpm test:content-product-impact", + " rationale: The fixture changes the declared Content contract.", + ...overrides, + "```", + ].join("\n"); +} + +before(() => { + temporaryRoot = mkdtempSync(path.join(tmpdir(), "content-impact-test-")); + baseRoot = path.join(temporaryRoot, "base"); + headRoot = path.join(temporaryRoot, "head"); + cpSync(fixture, baseRoot, { recursive: true }); + cpSync(fixture, headRoot, { recursive: true }); + + const feature = path.join(headRoot, "features/content.feature.test.md"); + writeFileSync( + feature, + readFileSync(feature, "utf8").replace( + 'roadmap_status: "planned"', + 'roadmap_status: "in_validation"', + ), + ); + const capability = path.join(headRoot, "capabilities/content.test.alpha.md"); + writeFileSync( + capability, + readFileSync(capability, "utf8").replace( + 'state: "approved_shape"', + 'state: "in_progress"', + ), + ); +}); + +after(() => rmSync(temporaryRoot, { recursive: true, force: true })); + +describe("Content impact declaration", () => { + it("parses one complete declaration", () => { + const result = parseContentImpactDeclaration(declaration()); + assert.equal(result.status, "valid"); + if (result.status !== "valid") return; + assert.deepEqual(result.declaration.features, ["content.feature.test"]); + assert.deepEqual(result.declaration.capabilities, ["content.test.alpha"]); + }); + + it("distinguishes missing, duplicate, malformed, and unknown-key blocks", () => { + assert.equal( + parseContentImpactDeclaration("No impact block.").status, + "missing", + ); + assert.equal( + parseContentImpactDeclaration(`${declaration()}\n${declaration()}`) + .status, + "duplicate", + ); + assert.equal( + parseContentImpactDeclaration( + "```yaml\ncontent_product_impact: [unterminated\n```", + ).status, + "malformed", + ); + const unknown = parseContentImpactDeclaration( + declaration([" surprise: no"]), + ); + assert.equal(unknown.status, "malformed"); + if (unknown.status === "malformed") { + assert(unknown.errors.some((error) => error.includes("unknown field"))); + } + const sibling = parseContentImpactDeclaration( + declaration(["unrelated_top_level_key: true"]), + ); + assert.equal(sibling.status, "malformed"); + }); + + it("requires exact IDs except for a pending product decision", () => { + const emptyFulfillment = parseContentImpactDeclaration( + declaration() + .replace(" - content.feature.test\n", " []\n") + .replace(" - content.test.alpha\n", " []\n"), + ); + assert.equal(emptyFulfillment.status, "malformed"); + + const pendingDecision = parseContentImpactDeclaration( + declaration() + .replace("contract_fulfillment", "product_decision_candidate") + .replace(" - content.feature.test\n", " []\n") + .replace(" - content.test.alpha\n", " []\n") + .replace("record_change: included", "record_change: decision_pending"), + ); + assert.equal(pendingDecision.status, "valid"); + + const includedNewRecord = parseContentImpactDeclaration( + declaration() + .replace("contract_fulfillment", "product_decision_candidate") + .replace(" - content.feature.test\n", " []\n") + .replace(" - content.test.alpha\n", " []\n"), + ); + assert.equal(includedNewRecord.status, "valid"); + }); +}); + +describe("Content impact applicability", () => { + it("automatically includes direct Content surfaces and exact conformance policy", () => { + assert(directContentEvidence("templates/content/app/routes/home.tsx")); + assert( + directContentEvidence("templates/content/actions/create-document.ts"), + ); + assert(directContentEvidence("templates/content/docs/product/roadmap.md")); + assert(directContentEvidence("templates/content/parity/contract.test.ts")); + assert(directContentEvidence("templates/content/vite.config.ts")); + assert.equal( + directContentEvidence("scripts/validate-content-product-impact.ts"), + undefined, + ); + assert.equal( + directContentEvidence( + ".github/workflows/content-product-conformance.yml", + ), + undefined, + ); + }); + + it("keeps ordinary shared framework, CI, dependency, and infrastructure changes quiet", () => { + for (const file of [ + "packages/core/src/client/index.ts", + "packages/toolkit/src/index.ts", + ".github/workflows/ci.yml", + "pnpm-lock.yaml", + "infra/example.tf", + ]) { + assert.equal(directContentEvidence(file), undefined, file); + } + }); +}); + +describe("Content impact analysis", () => { + const changedFiles = [ + "templates/content/docs/product/features/content.feature.test.md", + "templates/content/docs/product/capabilities/content.test.alpha.md", + ]; + + function catalogs() { + return { + base: validateContentProductDocs(baseRoot, { + strictCatalog: false, + checkProjections: false, + }), + head: validateContentProductDocs(headRoot, { + strictCatalog: false, + checkProjections: false, + }), + }; + } + + it("reports deterministic Feature and Capability transitions", () => { + const { base, head } = catalogs(); + const result = collectTransitions(changedFiles, base.catalog, head.catalog); + assert.deepEqual(result.changedRecords, [ + "content.feature.test", + "content.test.alpha", + ]); + assert.deepEqual(result.transitions, [ + { + id: "content.feature.test", + kind: "feature", + from: "planned", + to: "in_validation", + }, + { + id: "content.test.alpha", + kind: "capability", + from: "approved_shape", + to: "in_progress", + }, + ]); + }); + + it("accepts a complete applicable declaration without deterministic findings", () => { + const { base, head } = catalogs(); + const result = analyzeContentProductImpact({ + body: declaration(), + changedFiles, + baseCatalog: base.catalog, + headCatalog: head.catalog, + baseCatalogErrors: base.errors, + headCatalogErrors: head.errors, + }); + assert.equal(result.applicable, true); + assert.deepEqual(result.findings, []); + }); + + it("warns for missing declarations, unknown IDs, and undeclared record changes", () => { + const { base, head } = catalogs(); + const missing = analyzeContentProductImpact({ + body: "", + changedFiles, + baseCatalog: base.catalog, + headCatalog: head.catalog, + }); + assert( + missing.findings.some((item) => item.code === "declaration-missing"), + ); + + const invalid = analyzeContentProductImpact({ + body: declaration().replace( + "content.test.alpha", + "content.missing.capability", + ), + changedFiles, + baseCatalog: base.catalog, + headCatalog: head.catalog, + }); + assert(invalid.findings.some((item) => item.code === "unknown-capability")); + assert( + invalid.findings.some( + (item) => item.code === "changed-record-undeclared", + ), + ); + }); + + it("lets a valid explicit declaration opt a shared-only PR into conformance", () => { + const { base, head } = catalogs(); + const result = analyzeContentProductImpact({ + body: declaration() + .replace(" - content.feature.test\n", " []\n") + .replace("record_change: included", "record_change: none"), + changedFiles: ["packages/core/src/client/index.ts"], + baseCatalog: base.catalog, + headCatalog: head.catalog, + }); + assert.equal(result.applicable, true); + assert.deepEqual(result.findings, []); + }); + + it("keeps a shared-only PR with no direct evidence quiet and non-notifying", () => { + const { base, head } = catalogs(); + const result = analyzeContentProductImpact({ + body: "", + changedFiles: ["packages/core/src/client/index.ts"], + baseCatalog: base.catalog, + headCatalog: head.catalog, + }); + assert.equal(result.applicable, false); + assert.deepEqual(result.findings, []); + }); + + it("keeps malformed and unknown-ID declarations quiet on shared-only PRs", () => { + const { base, head } = catalogs(); + for (const body of [ + "```yaml\ncontent_product_impact: [broken\n```", + declaration() + .replace(" - content.feature.test\n", " []\n") + .replace("content.test.alpha", "content.unknown.contract"), + ]) { + const result = analyzeContentProductImpact({ + body, + changedFiles: ["packages/core/src/client/index.ts"], + baseCatalog: base.catalog, + headCatalog: head.catalog, + }); + assert.equal(result.applicable, false); + assert.deepEqual(result.findings, []); + } + }); +}); + +function git(root: string, args: string[]): string { + return execFileSync("git", args, { cwd: root, encoding: "utf8" }).trim(); +} + +function createCliRepository( + name: string, + change: (root: string) => void, + body: string, + withCatalog = true, + prepareBase?: (root: string) => void, +) { + const root = path.join(temporaryRoot, name); + mkdirSync(root, { recursive: true }); + git(root, ["init", "-q"]); + git(root, ["config", "user.email", "content-conformance@example.invalid"]); + git(root, ["config", "user.name", "Content Conformance Test"]); + if (withCatalog) { + const productRoot = path.join(root, "templates/content/docs/product"); + mkdirSync(productRoot, { recursive: true }); + cpSync(path.join(repositoryRoot, fixture), productRoot, { + recursive: true, + }); + } else { + writeFileSync(path.join(root, "README.md"), "base\n"); + } + prepareBase?.(root); + git(root, ["add", "."]); + git(root, ["commit", "-qm", "base"]); + const baseSha = git(root, ["rev-parse", "HEAD"]); + change(root); + git(root, ["add", "."]); + git(root, ["commit", "-qm", "head"]); + const headSha = git(root, ["rev-parse", "HEAD"]); + const eventPath = path.join(temporaryRoot, `${name}-event.json`); + writeFileSync( + eventPath, + JSON.stringify({ + pull_request: { + body, + base: { sha: baseSha }, + head: { sha: headSha }, + }, + }), + ); + return { root, eventPath, baseSha, headSha }; +} + +function runCli(input: ReturnType) { + return spawnSync(tsxPath, [checkerPath], { + cwd: input.root, + env: { + ...process.env, + GITHUB_EVENT_PATH: input.eventPath, + CONTENT_IMPACT_BASE_SHA: input.baseSha, + CONTENT_IMPACT_HEAD_SHA: input.headSha, + }, + encoding: "utf8", + }); +} + +describe("Content impact CLI boundary", () => { + it("returns success for advisory findings and emits a copyable warning", () => { + const repository = createCliRepository( + "direct-cli", + (root) => { + const file = path.join(root, "templates/content/app/example.ts"); + mkdirSync(path.dirname(file), { recursive: true }); + writeFileSync(file, "export const example = true;\n"); + }, + "", + ); + const result = runCli(repository); + assert.equal(result.status, 0, result.stderr); + assert.match(result.stdout, /"advisory":true/); + assert.match(result.stdout, /::warning title=Content product impact/); + assert.match(result.stdout, /content_product_impact/); + }); + + it("does not notify for a malformed declaration on a shared-only change", () => { + const repository = createCliRepository( + "shared-cli", + (root) => { + const file = path.join(root, "packages/core/src/example.ts"); + mkdirSync(path.dirname(file), { recursive: true }); + writeFileSync(file, "export const example = true;\n"); + }, + "```yaml\ncontent_product_impact: [broken\n```", + ); + const result = runCli(repository); + assert.equal(result.status, 0, result.stderr); + assert.match(result.stdout, /Not applicable/); + assert.doesNotMatch(result.stdout, /::warning/); + }); + + it("fails loudly for mismatched event revisions and missing catalogs", () => { + const mismatch = createCliRepository( + "mismatch-cli", + (root) => writeFileSync(path.join(root, "README.md"), "head\n"), + "", + false, + ); + const mismatchEvent = JSON.parse(readFileSync(mismatch.eventPath, "utf8")); + mismatchEvent.pull_request.base.sha = mismatch.headSha; + writeFileSync(mismatch.eventPath, JSON.stringify(mismatchEvent)); + const mismatchResult = runCli(mismatch); + assert.notEqual(mismatchResult.status, 0); + assert.match(mismatchResult.stderr, /do not match/); + + const missingCatalog = createCliRepository( + "missing-catalog-cli", + (root) => writeFileSync(path.join(root, "README.md"), "head\n"), + "", + false, + ); + const missingResult = runCli(missingCatalog); + assert.notEqual(missingResult.status, 0); + assert.match(missingResult.stderr, /readable Content product records/); + + const invalidBase = createCliRepository( + "invalid-base-cli", + (root) => writeFileSync(path.join(root, "README.md"), "head\n"), + "", + true, + (root) => { + writeFileSync(path.join(root, "README.md"), "base\n"); + const feature = path.join( + root, + "templates/content/docs/product/features/content.feature.test.md", + ); + writeFileSync( + feature, + readFileSync(feature, "utf8").replace("---", ""), + ); + }, + ); + const invalidResult = runCli(invalidBase); + assert.notEqual(invalidResult.status, 0); + assert.match(invalidResult.stderr, /base Content product catalog/); + }); +}); diff --git a/scripts/validate-content-product-impact.ts b/scripts/validate-content-product-impact.ts new file mode 100644 index 0000000000..ff8dc5c031 --- /dev/null +++ b/scripts/validate-content-product-impact.ts @@ -0,0 +1,705 @@ +#!/usr/bin/env node +import { execFileSync } from "node:child_process"; +import { + appendFileSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { parse as parseYaml } from "yaml"; + +import { + type ProductCatalog, + type ProductRecord, + validateContentProductDocs, +} from "./validate-content-product-docs.ts"; + +const PRODUCT_ROOT = "templates/content/docs/product"; +const LANES = [ + "contract_repair", + "contract_fulfillment", + "local_refinement", + "product_decision_candidate", +] as const; +const RECORD_CHANGES = ["none", "included", "decision_pending"] as const; +const DECLARATION_KEYS = new Set([ + "lane", + "features", + "capabilities", + "record_change", + "proof", + "rationale", +]); +const DECLARATION_REPAIR = `\`\`\`yaml +content_product_impact: + lane: contract_repair + features: + - content.feature.see-your-information-your-way + capabilities: + - content.view.renderer-conformance + record_change: none + proof: + - focused test or acceptance artifact + rationale: Explain the exact Content impact and record obligation. +\`\`\``; + +export type ContentImpactLane = (typeof LANES)[number]; +export type RecordChange = (typeof RECORD_CHANGES)[number]; + +export type ContentImpactDeclaration = { + lane: ContentImpactLane; + features: string[]; + capabilities: string[]; + record_change: RecordChange; + proof: string[]; + rationale: string; +}; + +export type DeclarationResult = + | { status: "missing" } + | { status: "malformed"; errors: string[] } + | { status: "duplicate"; errors: string[] } + | { status: "valid"; declaration: ContentImpactDeclaration }; + +export type ProductTransition = { + id: string; + kind: "feature" | "capability"; + from: string | null; + to: string | null; +}; + +export type ImpactFinding = { + code: string; + message: string; +}; + +export type ImpactAnalysis = { + applicable: boolean; + applicabilityReasons: string[]; + declaration: DeclarationResult; + changedRecords: string[]; + transitions: ProductTransition[]; + findings: ImpactFinding[]; +}; + +export type ImpactAnalysisInput = { + body: string; + changedFiles: readonly string[]; + baseCatalog: ProductCatalog; + headCatalog: ProductCatalog; + baseCatalogErrors?: readonly string[]; + headCatalogErrors?: readonly string[]; +}; + +type PullRequestEvent = { + pull_request?: { + body?: string | null; + base?: { sha?: string }; + head?: { sha?: string }; + }; +}; + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function stringArray( + value: unknown, + field: string, + errors: string[], +): string[] { + if (!Array.isArray(value) || value.some((item) => typeof item !== "string")) { + errors.push(`${field} must be an array of strings`); + return []; + } + return value.map((item) => item.trim()); +} + +function parseDeclarationValue( + value: unknown, +): + | { declaration: ContentImpactDeclaration; errors: [] } + | { declaration?: undefined; errors: string[] } { + const errors: string[] = []; + if (!isRecord(value) || !isRecord(value.content_product_impact)) { + return { + errors: ["content_product_impact must be a YAML mapping"], + }; + } + const topLevelKeys = Object.keys(value); + if ( + topLevelKeys.length !== 1 || + topLevelKeys[0] !== "content_product_impact" + ) { + errors.push("declaration block must contain only content_product_impact"); + } + const impact = value.content_product_impact; + for (const key of Object.keys(impact)) { + if (!DECLARATION_KEYS.has(key)) errors.push(`unknown field: ${key}`); + } + + const lane = + typeof impact.lane === "string" ? impact.lane.trim() : impact.lane; + if (typeof lane !== "string" || !LANES.includes(lane as ContentImpactLane)) { + errors.push(`lane must be one of: ${LANES.join(", ")}`); + } + const recordChange = + typeof impact.record_change === "string" + ? impact.record_change.trim() + : impact.record_change; + if ( + typeof recordChange !== "string" || + !RECORD_CHANGES.includes(recordChange as RecordChange) + ) { + errors.push(`record_change must be one of: ${RECORD_CHANGES.join(", ")}`); + } + const features = stringArray(impact.features, "features", errors); + const capabilities = stringArray(impact.capabilities, "capabilities", errors); + const proof = stringArray(impact.proof, "proof", errors); + if (proof.length === 0) errors.push("proof must contain at least one entry"); + if (proof.some((item) => item.trim().length === 0)) { + errors.push("proof entries must not be empty"); + } + const rationale = + typeof impact.rationale === "string" + ? impact.rationale.trim() + : impact.rationale; + if (typeof rationale !== "string" || rationale.trim().length === 0) { + errors.push("rationale must be a non-empty string"); + } + if ( + typeof lane === "string" && + lane !== "product_decision_candidate" && + features.length + capabilities.length === 0 + ) { + errors.push(`${lane} requires at least one Feature or Capability ID`); + } + if ( + lane === "product_decision_candidate" && + features.length + capabilities.length === 0 && + recordChange !== "decision_pending" && + recordChange !== "included" + ) { + errors.push( + "a product_decision_candidate without IDs requires record_change decision_pending or included", + ); + } + for (const [field, ids] of [ + ["features", features], + ["capabilities", capabilities], + ] as const) { + if (new Set(ids).size !== ids.length) { + errors.push(`${field} must not contain duplicate IDs`); + } + } + if (errors.length > 0) return { errors }; + + return { + errors: [], + declaration: { + lane: lane as ContentImpactLane, + features, + capabilities, + record_change: recordChange as RecordChange, + proof, + rationale: rationale as string, + }, + }; +} + +export function parseContentImpactDeclaration(body: string): DeclarationResult { + const candidates = [ + ...body.matchAll(/```(?:yaml|yml)\s*\r?\n([\s\S]*?)```/gi), + ] + .map((match) => match[1]) + .filter((source) => /^\s*content_product_impact\s*:/m.test(source)); + + if (candidates.length === 0) return { status: "missing" }; + if (candidates.length > 1) { + return { + status: "duplicate", + errors: ["PR body must contain exactly one content_product_impact block"], + }; + } + + let parsed: unknown; + try { + parsed = parseYaml(candidates[0], { uniqueKeys: true }); + } catch (error) { + return { + status: "malformed", + errors: [ + `declaration YAML could not be parsed: ${error instanceof Error ? error.message : String(error)}`, + ], + }; + } + const result = parseDeclarationValue(parsed); + return result.declaration + ? { status: "valid", declaration: result.declaration } + : { status: "malformed", errors: result.errors }; +} + +const CONTENT_ROOT_CONFIG = new Set([ + "templates/content/agent-native.app-skill.json", + "templates/content/drizzle.config.ts", + "templates/content/netlify.toml", + "templates/content/package.json", + "templates/content/react-router.config.ts", + "templates/content/ssr-entry.ts", + "templates/content/vite.config.ts", +]); + +export function directContentEvidence(file: string): string | undefined { + const normalized = file.replaceAll("\\", "/"); + const prefixes = [ + "templates/content/actions/", + "templates/content/app/", + "templates/content/data/", + "templates/content/docs/product/", + "templates/content/e2e/", + "templates/content/parity/", + "templates/content/server/", + "templates/content/shared/", + "templates/content/.agents/skills/content-product-development/", + "scripts/fixtures/content-product-docs/", + "scripts/fixtures/content-product-impact/", + ]; + if (prefixes.some((prefix) => normalized.startsWith(prefix))) { + return normalized; + } + if (CONTENT_ROOT_CONFIG.has(normalized)) { + return normalized; + } + if ( + normalized.startsWith("templates/content/") && + /(?:content.*parity|parity.*content|content.*conformance|conformance.*content)/i.test( + normalized, + ) + ) { + return normalized; + } + return undefined; +} + +function stateOf(record: ProductRecord): string | null { + const value = + record.kind === "feature" ? record.data.roadmap_status : record.data.state; + return typeof value === "string" ? value : null; +} + +export function collectTransitions( + changedFiles: readonly string[], + baseCatalog: ProductCatalog, + headCatalog: ProductCatalog, +): { changedRecords: string[]; transitions: ProductTransition[] } { + const recordIds = new Set(); + const productRecordPath = new RegExp( + `^${PRODUCT_ROOT}/(?:features|capabilities)/(content\\.[a-z0-9.-]+)\\.md$`, + ); + for (const file of changedFiles) { + const match = file.match(productRecordPath); + if (match) recordIds.add(match[1]); + } + + const baseById = new Map( + baseCatalog.records.map((record) => [record.id, record]), + ); + const headById = new Map( + headCatalog.records.map((record) => [record.id, record]), + ); + const transitions: ProductTransition[] = []; + for (const id of [...recordIds].sort()) { + const base = baseById.get(id); + const head = headById.get(id); + const record = head ?? base; + if (!record || record.kind === "chapter") continue; + const from = base ? stateOf(base) : null; + const to = head ? stateOf(head) : null; + if (from !== to) transitions.push({ id, kind: record.kind, from, to }); + } + return { changedRecords: [...recordIds].sort(), transitions }; +} + +function finding(code: string, message: string): ImpactFinding { + return { code, message }; +} + +export function analyzeContentProductImpact( + input: ImpactAnalysisInput, +): ImpactAnalysis { + const declaration = parseContentImpactDeclaration(input.body); + const direct = input.changedFiles + .map(directContentEvidence) + .filter((file): file is string => file !== undefined); + const headIntroducedCatalogFailure = + (input.baseCatalogErrors?.length ?? 0) === 0 && + (input.headCatalogErrors?.length ?? 0) > 0; + const knownFeatureIds = new Set( + input.headCatalog.features.map((record) => record.id), + ); + const knownCapabilityIds = new Set( + input.headCatalog.capabilities.map((record) => record.id), + ); + const declarationNamesKnownId = + declaration.status === "valid" && + (declaration.declaration.features.some((id) => knownFeatureIds.has(id)) || + declaration.declaration.capabilities.some((id) => + knownCapabilityIds.has(id), + )); + const applicabilityReasons = [ + ...direct.map((file) => `direct:${file}`), + ...(declarationNamesKnownId ? ["declaration"] : []), + ...(headIntroducedCatalogFailure ? ["content-contract-failure"] : []), + ]; + const applicable = applicabilityReasons.length > 0; + const { changedRecords, transitions } = collectTransitions( + input.changedFiles, + input.baseCatalog, + input.headCatalog, + ); + const findings: ImpactFinding[] = []; + + if (!applicable) { + return { + applicable, + applicabilityReasons, + declaration, + changedRecords, + transitions, + findings, + }; + } + + if (declaration.status === "missing") { + findings.push( + finding( + "declaration-missing", + `Copy this declaration into the PR body and replace its values with the exact impact:\n${DECLARATION_REPAIR}`, + ), + ); + } else if (declaration.status !== "valid") { + for (const error of declaration.errors) { + findings.push(finding(`declaration-${declaration.status}`, error)); + } + } else { + const value = declaration.declaration; + const headFeatures = new Set( + input.headCatalog.features.map((record) => record.id), + ); + const headCapabilities = new Set( + input.headCatalog.capabilities.map((record) => record.id), + ); + const deletedFeatures = new Set( + input.baseCatalog.records + .filter( + (record) => + record.kind === "feature" && + changedRecords.includes(record.id) && + !input.headCatalog.records.some((head) => head.id === record.id), + ) + .map((record) => record.id), + ); + const deletedCapabilities = new Set( + input.baseCatalog.records + .filter( + (record) => + record.kind === "capability" && + changedRecords.includes(record.id) && + !input.headCatalog.records.some((head) => head.id === record.id), + ) + .map((record) => record.id), + ); + + for (const id of value.features) { + if (!headFeatures.has(id) && !deletedFeatures.has(id)) { + findings.push(finding("unknown-feature", `Unknown Feature ID: ${id}`)); + } + } + for (const id of value.capabilities) { + if (!headCapabilities.has(id) && !deletedCapabilities.has(id)) { + findings.push( + finding("unknown-capability", `Unknown Capability ID: ${id}`), + ); + } + } + + const declaredIds = new Set([...value.features, ...value.capabilities]); + const allowsUnnamedNewRecords = + value.lane === "product_decision_candidate" && + value.record_change === "included" && + declaredIds.size === 0; + for (const id of changedRecords) { + const recordExistedAtBase = input.baseCatalog.records.some( + (record) => record.id === id, + ); + if ( + !declaredIds.has(id) && + !(allowsUnnamedNewRecords && !recordExistedAtBase) + ) { + findings.push( + finding( + "changed-record-undeclared", + `Changed product record ${id} is not named in the declaration.`, + ), + ); + } + } + if (changedRecords.length > 0 && value.record_change === "none") { + findings.push( + finding( + "record-change-mismatch", + "record_change cannot be none when Feature or Capability records changed.", + ), + ); + } + if (changedRecords.length === 0 && value.record_change === "included") { + findings.push( + finding( + "record-change-mismatch", + "record_change is included, but no Feature or Capability record changed.", + ), + ); + } + if ( + value.record_change === "decision_pending" && + value.lane !== "product_decision_candidate" + ) { + findings.push( + finding( + "decision-lane-mismatch", + "record_change decision_pending requires lane product_decision_candidate.", + ), + ); + } + if ( + value.record_change === "decision_pending" && + transitions.some( + (transition) => + transition.to === "verified" || transition.to === "available", + ) + ) { + findings.push( + finding( + "decision-promotion-mismatch", + "record_change decision_pending cannot claim a verified Capability or available Feature promotion.", + ), + ); + } + } + + for (const error of input.headCatalogErrors ?? []) { + findings.push(finding("head-catalog-invalid", error)); + } + + return { + applicable, + applicabilityReasons, + declaration, + changedRecords, + transitions, + findings, + }; +} + +function git(args: string[]): string { + return execFileSync("git", args, { + cwd: process.cwd(), + encoding: "utf8", + maxBuffer: 32 * 1024 * 1024, + }); +} + +function assertSha(value: string, name: string): void { + if (!/^[0-9a-f]{40,64}$/i.test(value)) { + throw new Error(`${name} must be a full Git object SHA`); + } + git(["cat-file", "-e", `${value}^{commit}`]); +} + +function materializeProductRoot(sha: string, destination: string): string { + const root = path.join(destination, PRODUCT_ROOT); + for (const directory of ["chapters", "features", "capabilities"]) { + mkdirSync(path.join(root, directory), { recursive: true }); + } + const output = git([ + "ls-tree", + "-r", + "--name-only", + "-z", + sha, + "--", + PRODUCT_ROOT, + ]); + const files = output.split("\0").filter(Boolean); + const productRecordFiles = files.filter( + (file) => + file.endsWith(".md") && + /\/(?:chapters|features|capabilities)\//.test(file), + ); + if (productRecordFiles.length === 0) { + throw new Error(`${sha} does not contain readable Content product records`); + } + for (const file of files) { + if (!file.endsWith(".md")) continue; + const destinationFile = path.join(destination, file); + mkdirSync(path.dirname(destinationFile), { recursive: true }); + const source = git(["show", `${sha}:${file}`]); + writeFileSync(destinationFile, source); + } + return root; +} + +function changedFiles(baseSha: string, headSha: string): string[] { + return git([ + "diff", + "--name-only", + "-z", + "--no-renames", + "--diff-filter=ACDMRTUXB", + `${baseSha}...${headSha}`, + ]) + .split("\0") + .filter(Boolean); +} + +function annotationEscape(value: string): string { + return value + .replaceAll("%", "%25") + .replaceAll("\r", "%0D") + .replaceAll("\n", "%0A"); +} + +export function renderImpactSummary(analysis: ImpactAnalysis): string { + if (!analysis.applicable) { + return [ + "## Content product conformance", + "", + "Not applicable. No direct Content evidence or explicit Content impact declaration was found.", + ].join("\n"); + } + const lines = [ + "## Content product conformance — advisory pilot", + "", + `Applicability: ${analysis.applicabilityReasons.join(", ")}`, + `Declaration: ${analysis.declaration.status}`, + `Changed Feature/Capability records: ${analysis.changedRecords.join(", ") || "none"}`, + "", + ]; + if (analysis.transitions.length > 0) { + lines.push("| record | transition |", "| --- | --- |"); + for (const transition of analysis.transitions) { + lines.push( + `| ${transition.id} | ${transition.from ?? "absent"} -> ${transition.to ?? "absent"} |`, + ); + } + lines.push(""); + } + if (analysis.findings.length === 0) { + lines.push("No deterministic findings. This pilot remains advisory."); + } else { + lines.push("### Advisory findings", ""); + for (const item of analysis.findings) { + lines.push(`- **${item.code}:** ${item.message}`); + } + lines.push( + "", + "These findings do not block the pull request during calibration.", + ); + } + return lines.join("\n"); +} + +export function runContentProductImpactCheck(): void { + const eventPath = process.env.GITHUB_EVENT_PATH; + const baseSha = process.env.CONTENT_IMPACT_BASE_SHA; + const headSha = process.env.CONTENT_IMPACT_HEAD_SHA; + if (!eventPath || !baseSha || !headSha) { + throw new Error( + "GITHUB_EVENT_PATH, CONTENT_IMPACT_BASE_SHA, and CONTENT_IMPACT_HEAD_SHA are required", + ); + } + assertSha(baseSha, "CONTENT_IMPACT_BASE_SHA"); + assertSha(headSha, "CONTENT_IMPACT_HEAD_SHA"); + + const event = JSON.parse(readFileSync(eventPath, "utf8")) as PullRequestEvent; + const pullRequest = event.pull_request; + if (!pullRequest) throw new Error("event does not contain pull_request data"); + if (pullRequest.base?.sha !== baseSha || pullRequest.head?.sha !== headSha) { + throw new Error( + "event base/head SHAs do not match the requested revisions", + ); + } + + const temporaryRoot = mkdtempSync(path.join(tmpdir(), "content-impact-")); + try { + const baseRoot = materializeProductRoot( + baseSha, + path.join(temporaryRoot, "base"), + ); + const headRoot = materializeProductRoot( + headSha, + path.join(temporaryRoot, "head"), + ); + const base = validateContentProductDocs(baseRoot, { + strictCatalog: false, + checkProjections: false, + }); + const head = validateContentProductDocs(headRoot, { + strictCatalog: false, + checkProjections: false, + }); + if (base.errors.length > 0) { + throw new Error( + `base Content product catalog is unreadable or invalid:\n${base.errors.join("\n")}`, + ); + } + const analysis = analyzeContentProductImpact({ + body: pullRequest.body ?? "", + changedFiles: changedFiles(baseSha, headSha), + baseCatalog: base.catalog, + headCatalog: head.catalog, + baseCatalogErrors: base.errors, + headCatalogErrors: head.errors, + }); + const summary = renderImpactSummary(analysis); + process.stdout.write( + `${JSON.stringify({ + schema: "content-conformance.advisory-declaration.v1", + headSha, + advisory: true, + ...analysis, + })}\n`, + ); + if (process.env.GITHUB_STEP_SUMMARY) { + appendFileSync(process.env.GITHUB_STEP_SUMMARY, `${summary}\n`); + } + process.stdout.write(`${summary}\n`); + if (analysis.applicable) { + for (const item of analysis.findings) { + process.stdout.write( + `::warning title=Content product impact (${annotationEscape(item.code)})::${annotationEscape(item.message)}\n`, + ); + } + } + } finally { + rmSync(temporaryRoot, { recursive: true, force: true }); + } +} + +const invokedPath = process.argv[1] ? path.resolve(process.argv[1]) : ""; +if (invokedPath === fileURLToPath(import.meta.url)) { + try { + runContentProductImpactCheck(); + } catch (error) { + console.error( + `Content product impact check could not establish its inputs: ${error instanceof Error ? error.message : String(error)}`, + ); + process.exitCode = 1; + } +} diff --git a/templates/content/.agents/skills/content-product-development/SKILL.md b/templates/content/.agents/skills/content-product-development/SKILL.md index 2f52a2277e..e5687b4e60 100644 --- a/templates/content/.agents/skills/content-product-development/SKILL.md +++ b/templates/content/.agents/skills/content-product-development/SKILL.md @@ -72,6 +72,53 @@ require a permission slip for every useful bug fix. Load the domain skill for the work, especially `actions`, `security`, `sharing`, `storing-data`, `portability`, `real-time-collab`, or `real-time-sync`. +## Declare pull-request impact + +During the advisory conformance pilot, add exactly one fenced YAML declaration +to the pull-request body when a change directly affects Content or when a shared +framework change has specific Content impact: + +```yaml +content_product_impact: + lane: contract_repair + features: + - content.feature.example + capabilities: + - content.example.capability + record_change: none + proof: + - pnpm --filter content test + rationale: The change repairs the declared Capability without changing its contract. +``` + +Use stable IDs from `templates/content/docs/product/`. Valid lanes are +`contract_repair`, `contract_fulfillment`, `local_refinement`, and +`product_decision_candidate`. Set `record_change` to `included` when the PR +changes a Feature or Capability record, or `decision_pending` when the product +record must wait for an explicit product decision. + +The lane meanings are the same as the classification table above. A contract +repair does not require roadmap churn when the accepted record remains true. +Repairs, fulfillment, and local refinement must name at least one real Feature +or Capability. A product-decision candidate may omit IDs only with +`decision_pending`, or while the same PR includes the proposed new record; do +not invent an ID to make the declaration look complete. + +Run the focused declaration and workflow-policy tests with: + +```sh +pnpm test:content-product-impact +``` + +To repair a warning, copy the block above into the PR body, replace the example +IDs with exact catalog IDs, align `record_change` with any changed product +records, and name the focused proof you actually ran. + +The check is intentionally advisory while its deterministic rules calibrate. +It never edits the roadmap, assigns or tags a reviewer, and never turns an LLM +opinion into a merge gate. Shared framework changes with no direct Content +evidence should remain quiet. + ## Prove and record the result Use the affected Capability's proof requirements and the Feature's example From a2ace92de80e73c5e81e7c9d0364e9bd5c6058b2 Mon Sep 17 00:00:00 2001 From: Alice Alexandra Moore <86723305+3mdistal@users.noreply.github.com> Date: Wed, 29 Jul 2026 17:47:35 -0400 Subject: [PATCH 02/13] Fix snapshot validation root --- scripts/validate-content-product-impact.test.ts | 8 ++++++++ scripts/validate-content-product-impact.ts | 9 +++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/scripts/validate-content-product-impact.test.ts b/scripts/validate-content-product-impact.test.ts index 82e494c699..53b1c33186 100644 --- a/scripts/validate-content-product-impact.test.ts +++ b/scripts/validate-content-product-impact.test.ts @@ -328,6 +328,14 @@ function createCliRepository( cpSync(path.join(repositoryRoot, fixture), productRoot, { recursive: true, }); + writeFileSync( + path.join(productRoot, "architecture.md"), + "# Architecture\n", + ); + writeFileSync( + path.join(productRoot, "README.md"), + "Read the [architecture](architecture.md).\n", + ); } else { writeFileSync(path.join(root, "README.md"), "base\n"); } diff --git a/scripts/validate-content-product-impact.ts b/scripts/validate-content-product-impact.ts index ff8dc5c031..18ef3e2d75 100644 --- a/scripts/validate-content-product-impact.ts +++ b/scripts/validate-content-product-impact.ts @@ -8,7 +8,6 @@ import { rmSync, writeFileSync, } from "node:fs"; -import { tmpdir } from "node:os"; import path from "node:path"; import { fileURLToPath } from "node:url"; @@ -21,6 +20,10 @@ import { } from "./validate-content-product-docs.ts"; const PRODUCT_ROOT = "templates/content/docs/product"; +const REPOSITORY_ROOT = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + "..", +); const LANES = [ "contract_repair", "contract_fulfillment", @@ -636,7 +639,9 @@ export function runContentProductImpactCheck(): void { ); } - const temporaryRoot = mkdtempSync(path.join(tmpdir(), "content-impact-")); + const temporaryRoot = mkdtempSync( + path.join(REPOSITORY_ROOT, ".content-impact-"), + ); try { const baseRoot = materializeProductRoot( baseSha, From e2fb64fd2551a20a5c99cd7b69293f206a6126ab Mon Sep 17 00:00:00 2001 From: Alice Alexandra Moore <86723305+3mdistal@users.noreply.github.com> Date: Wed, 29 Jul 2026 18:03:33 -0400 Subject: [PATCH 03/13] Isolate Content validation snapshots --- scripts/validate-content-product-docs.ts | 4 +-- .../validate-content-product-impact.test.ts | 30 +++++++++++++++++++ scripts/validate-content-product-impact.ts | 12 ++++++-- 3 files changed, 41 insertions(+), 5 deletions(-) diff --git a/scripts/validate-content-product-docs.ts b/scripts/validate-content-product-docs.ts index d6c48df98c..39959f8b26 100644 --- a/scripts/validate-content-product-docs.ts +++ b/scripts/validate-content-product-docs.ts @@ -806,8 +806,8 @@ function validateCapabilityMiniSpec( function validateLinksAndPrivacy(catalog: ProductCatalog, errors: string[]) { const files = collectMarkdownFiles(catalog.root); const skillRoot = join( - repositoryRoot, - "templates/content/.agents/skills/content-product-development", + resolve(catalog.root, "../.."), + ".agents/skills/content-product-development", ); if (existsSync(skillRoot)) files.push(...collectMarkdownFiles(skillRoot)); diff --git a/scripts/validate-content-product-impact.test.ts b/scripts/validate-content-product-impact.test.ts index 53b1c33186..2bba6fca9c 100644 --- a/scripts/validate-content-product-impact.test.ts +++ b/scripts/validate-content-product-impact.test.ts @@ -324,7 +324,12 @@ function createCliRepository( git(root, ["config", "user.name", "Content Conformance Test"]); if (withCatalog) { const productRoot = path.join(root, "templates/content/docs/product"); + const skillRoot = path.join( + root, + "templates/content/.agents/skills/content-product-development", + ); mkdirSync(productRoot, { recursive: true }); + mkdirSync(skillRoot, { recursive: true }); cpSync(path.join(repositoryRoot, fixture), productRoot, { recursive: true, }); @@ -336,6 +341,10 @@ function createCliRepository( path.join(productRoot, "README.md"), "Read the [architecture](architecture.md).\n", ); + writeFileSync( + path.join(skillRoot, "SKILL.md"), + "# Content product skill\n", + ); } else { writeFileSync(path.join(root, "README.md"), "base\n"); } @@ -408,6 +417,27 @@ describe("Content impact CLI boundary", () => { assert.doesNotMatch(result.stdout, /::warning/); }); + it("validates the base and head developer skills from their own revisions", () => { + const repository = createCliRepository( + "head-skill-cli", + (root) => { + writeFileSync( + path.join( + root, + "templates/content/.agents/skills/content-product-development/SKILL.md", + ), + "# Content product skill\n\nDo not use [private files](file:///private/example).\n", + ); + }, + "", + ); + const result = runCli(repository); + assert.equal(result.status, 0, result.stderr); + assert.match(result.stdout, /head-catalog-invalid/); + assert.match(result.stdout, /remove file URL/); + assert.doesNotMatch(result.stderr, /base Content product catalog/); + }); + it("fails loudly for mismatched event revisions and missing catalogs", () => { const mismatch = createCliRepository( "mismatch-cli", diff --git a/scripts/validate-content-product-impact.ts b/scripts/validate-content-product-impact.ts index 18ef3e2d75..ea0b104b62 100644 --- a/scripts/validate-content-product-impact.ts +++ b/scripts/validate-content-product-impact.ts @@ -20,6 +20,8 @@ import { } from "./validate-content-product-docs.ts"; const PRODUCT_ROOT = "templates/content/docs/product"; +const CONTENT_PRODUCT_SKILL_ROOT = + "templates/content/.agents/skills/content-product-development"; const REPOSITORY_ROOT = path.resolve( path.dirname(fileURLToPath(import.meta.url)), "..", @@ -525,7 +527,10 @@ function assertSha(value: string, name: string): void { git(["cat-file", "-e", `${value}^{commit}`]); } -function materializeProductRoot(sha: string, destination: string): string { +function materializeContentProductSnapshot( + sha: string, + destination: string, +): string { const root = path.join(destination, PRODUCT_ROOT); for (const directory of ["chapters", "features", "capabilities"]) { mkdirSync(path.join(root, directory), { recursive: true }); @@ -538,6 +543,7 @@ function materializeProductRoot(sha: string, destination: string): string { sha, "--", PRODUCT_ROOT, + CONTENT_PRODUCT_SKILL_ROOT, ]); const files = output.split("\0").filter(Boolean); const productRecordFiles = files.filter( @@ -643,11 +649,11 @@ export function runContentProductImpactCheck(): void { path.join(REPOSITORY_ROOT, ".content-impact-"), ); try { - const baseRoot = materializeProductRoot( + const baseRoot = materializeContentProductSnapshot( baseSha, path.join(temporaryRoot, "base"), ); - const headRoot = materializeProductRoot( + const headRoot = materializeContentProductSnapshot( headSha, path.join(temporaryRoot, "head"), ); From d36f55626372d3454f6fe4828c897942fa723087 Mon Sep 17 00:00:00 2001 From: Alice Alexandra Moore <86723305+3mdistal@users.noreply.github.com> Date: Wed, 29 Jul 2026 18:13:32 -0400 Subject: [PATCH 04/13] Use revision-consistent Content baselines --- scripts/validate-content-product-docs.ts | 20 +++++++-- .../validate-content-product-impact.test.ts | 44 ++++++++++++++++++- scripts/validate-content-product-impact.ts | 20 ++++++--- 3 files changed, 73 insertions(+), 11 deletions(-) diff --git a/scripts/validate-content-product-docs.ts b/scripts/validate-content-product-docs.ts index 39959f8b26..14f690aaa1 100644 --- a/scripts/validate-content-product-docs.ts +++ b/scripts/validate-content-product-docs.ts @@ -77,7 +77,11 @@ export type ValidationResult = { export function validateContentProductDocs( root = defaultProductRoot, - options: { strictCatalog?: boolean; checkProjections?: boolean } = {}, + options: { + strictCatalog?: boolean; + checkProjections?: boolean; + validationRoot?: string; + } = {}, ): ValidationResult { const strictCatalog = options.strictCatalog ?? root === defaultProductRoot; const checkProjections = options.checkProjections ?? true; @@ -85,7 +89,11 @@ export function validateContentProductDocs( const catalog = loadCatalog(root, errors); validateCatalog(catalog, errors, strictCatalog); - validateLinksAndPrivacy(catalog, errors); + validateLinksAndPrivacy( + catalog, + errors, + options.validationRoot ?? repositoryRoot, + ); const roadmap = renderRoadmap(catalog); const encyclopedia = renderEncyclopedia(catalog); @@ -803,7 +811,11 @@ function validateCapabilityMiniSpec( } } -function validateLinksAndPrivacy(catalog: ProductCatalog, errors: string[]) { +function validateLinksAndPrivacy( + catalog: ProductCatalog, + errors: string[], + validationRoot: string, +) { const files = collectMarkdownFiles(catalog.root); const skillRoot = join( resolve(catalog.root, "../.."), @@ -859,7 +871,7 @@ function validateLinksAndPrivacy(catalog: ProductCatalog, errors: string[]) { } const withoutAnchor = target.split("#")[0]; const resolved = resolve(dirname(file), withoutAnchor); - if (!isInside(repositoryRoot, resolved)) { + if (!isInside(validationRoot, resolved)) { errors.push( `${displayPath(file)}:${lineNumber(source, match.index ?? 0)}: link escapes the repository: ${target}`, ); diff --git a/scripts/validate-content-product-impact.test.ts b/scripts/validate-content-product-impact.test.ts index 2bba6fca9c..8717ff9cf1 100644 --- a/scripts/validate-content-product-impact.test.ts +++ b/scripts/validate-content-product-impact.test.ts @@ -426,7 +426,7 @@ describe("Content impact CLI boundary", () => { root, "templates/content/.agents/skills/content-product-development/SKILL.md", ), - "# Content product skill\n\nDo not use [private files](file:///private/example).\n", + "# Content product skill\n\nDo not use [live checkout files](../../../../../../../package.json).\n", ); }, "", @@ -434,10 +434,50 @@ describe("Content impact CLI boundary", () => { const result = runCli(repository); assert.equal(result.status, 0, result.stderr); assert.match(result.stdout, /head-catalog-invalid/); - assert.match(result.stdout, /remove file URL/); + assert.match(result.stdout, /link escapes the repository/); assert.doesNotMatch(result.stderr, /base Content product catalog/); }); + it("uses the merge-base for a PR whose target branch has advanced", () => { + const repository = createCliRepository( + "diverged-base-cli", + (root) => writeFileSync(path.join(root, "README.md"), "head\n"), + declaration().replace("record_change: included", "record_change: none"), + ); + git(repository.root, ["checkout", "-q", "--detach", repository.baseSha]); + writeFileSync( + path.join( + repository.root, + "templates/content/.agents/skills/content-product-development/SKILL.md", + ), + "# Content product skill\n\nDo not use file:///private/example.\n", + ); + git(repository.root, ["add", "."]); + git(repository.root, ["commit", "-qm", "advanced base"]); + const advancedBaseSha = git(repository.root, ["rev-parse", "HEAD"]); + writeFileSync( + repository.eventPath, + JSON.stringify({ + pull_request: { + body: declaration().replace( + "record_change: included", + "record_change: none", + ), + base: { sha: advancedBaseSha }, + head: { sha: repository.headSha }, + }, + }), + ); + + const result = runCli({ ...repository, baseSha: advancedBaseSha }); + assert.equal(result.status, 0, result.stderr); + assert.doesNotMatch(result.stderr, /base Content product catalog/); + assert.match( + result.stdout, + new RegExp(`"comparisonBaseSha":"${repository.baseSha}"`), + ); + }); + it("fails loudly for mismatched event revisions and missing catalogs", () => { const mismatch = createCliRepository( "mismatch-cli", diff --git a/scripts/validate-content-product-impact.ts b/scripts/validate-content-product-impact.ts index ea0b104b62..d25dbbbd42 100644 --- a/scripts/validate-content-product-impact.ts +++ b/scripts/validate-content-product-impact.ts @@ -571,7 +571,8 @@ function changedFiles(baseSha: string, headSha: string): string[] { "-z", "--no-renames", "--diff-filter=ACDMRTUXB", - `${baseSha}...${headSha}`, + baseSha, + headSha, ]) .split("\0") .filter(Boolean); @@ -644,26 +645,34 @@ export function runContentProductImpactCheck(): void { "event base/head SHAs do not match the requested revisions", ); } + const comparisonBaseSha = git(["merge-base", baseSha, headSha]).trim(); + if (!/^[0-9a-f]{40,64}$/i.test(comparisonBaseSha)) { + throw new Error("could not resolve a full merge-base for the PR revisions"); + } const temporaryRoot = mkdtempSync( path.join(REPOSITORY_ROOT, ".content-impact-"), ); try { + const baseSnapshotRoot = path.join(temporaryRoot, "base"); + const headSnapshotRoot = path.join(temporaryRoot, "head"); const baseRoot = materializeContentProductSnapshot( - baseSha, - path.join(temporaryRoot, "base"), + comparisonBaseSha, + baseSnapshotRoot, ); const headRoot = materializeContentProductSnapshot( headSha, - path.join(temporaryRoot, "head"), + headSnapshotRoot, ); const base = validateContentProductDocs(baseRoot, { strictCatalog: false, checkProjections: false, + validationRoot: baseSnapshotRoot, }); const head = validateContentProductDocs(headRoot, { strictCatalog: false, checkProjections: false, + validationRoot: headSnapshotRoot, }); if (base.errors.length > 0) { throw new Error( @@ -672,7 +681,7 @@ export function runContentProductImpactCheck(): void { } const analysis = analyzeContentProductImpact({ body: pullRequest.body ?? "", - changedFiles: changedFiles(baseSha, headSha), + changedFiles: changedFiles(comparisonBaseSha, headSha), baseCatalog: base.catalog, headCatalog: head.catalog, baseCatalogErrors: base.errors, @@ -683,6 +692,7 @@ export function runContentProductImpactCheck(): void { `${JSON.stringify({ schema: "content-conformance.advisory-declaration.v1", headSha, + comparisonBaseSha, advisory: true, ...analysis, })}\n`, From bcbaa6b6784fc4d3639b45ea59b4b140c9e17c45 Mon Sep 17 00:00:00 2001 From: Alice Alexandra Moore <86723305+3mdistal@users.noreply.github.com> Date: Wed, 29 Jul 2026 18:27:16 -0400 Subject: [PATCH 05/13] Complete Content conformance perimeter --- .../validate-content-product-impact.test.ts | 32 +++++++++++++++++++ scripts/validate-content-product-impact.ts | 29 +++++++++++------ 2 files changed, 51 insertions(+), 10 deletions(-) diff --git a/scripts/validate-content-product-impact.test.ts b/scripts/validate-content-product-impact.test.ts index 8717ff9cf1..234bb41649 100644 --- a/scripts/validate-content-product-impact.test.ts +++ b/scripts/validate-content-product-impact.test.ts @@ -150,6 +150,18 @@ describe("Content impact applicability", () => { assert(directContentEvidence("templates/content/docs/product/roadmap.md")); assert(directContentEvidence("templates/content/parity/contract.test.ts")); assert(directContentEvidence("templates/content/vite.config.ts")); + assert(directContentEvidence("templates/content/tsconfig.json")); + assert(directContentEvidence("templates/content/vitest.config.ts")); + assert(directContentEvidence("templates/content/drizzle/0001_example.sql")); + assert(directContentEvidence("templates/content/public/example.svg")); + assert.equal( + directContentEvidence("templates/content/README.md"), + undefined, + ); + assert.equal( + directContentEvidence("templates/content/.oxfmtrc.json"), + undefined, + ); assert.equal( directContentEvidence("scripts/validate-content-product-impact.ts"), undefined, @@ -502,6 +514,26 @@ describe("Content impact CLI boundary", () => { assert.notEqual(missingResult.status, 0); assert.match(missingResult.stderr, /readable Content product records/); + const nestedCatalog = createCliRepository( + "nested-catalog-cli", + (root) => { + const productRoot = path.join(root, "templates/content/docs/product"); + for (const directory of ["chapters", "features", "capabilities"]) { + rmSync(path.join(productRoot, directory), { + recursive: true, + force: true, + }); + } + const archive = path.join(productRoot, "archive/capabilities"); + mkdirSync(archive, { recursive: true }); + writeFileSync(path.join(archive, "note.md"), "# Not a record\n"); + }, + "", + ); + const nestedResult = runCli(nestedCatalog); + assert.notEqual(nestedResult.status, 0); + assert.match(nestedResult.stderr, /readable Content product records/); + const invalidBase = createCliRepository( "invalid-base-cli", (root) => writeFileSync(path.join(root, "README.md"), "head\n"), diff --git a/scripts/validate-content-product-impact.ts b/scripts/validate-content-product-impact.ts index d25dbbbd42..22c3c1a15e 100644 --- a/scripts/validate-content-product-impact.ts +++ b/scripts/validate-content-product-impact.ts @@ -250,14 +250,15 @@ export function parseContentImpactDeclaration(body: string): DeclarationResult { : { status: "malformed", errors: result.errors }; } -const CONTENT_ROOT_CONFIG = new Set([ - "templates/content/agent-native.app-skill.json", - "templates/content/drizzle.config.ts", - "templates/content/netlify.toml", - "templates/content/package.json", - "templates/content/react-router.config.ts", - "templates/content/ssr-entry.ts", - "templates/content/vite.config.ts", +const CONTENT_ROOT_EXCLUSIONS = new Set([ + "templates/content/.dockerignore", + "templates/content/.gitignore", + "templates/content/.ignore", + "templates/content/.oxfmtrc.json", + "templates/content/CHANGELOG.md", + "templates/content/DEVELOPING.md", + "templates/content/README.md", + "templates/content/_gitignore", ]); export function directContentEvidence(file: string): string | undefined { @@ -267,8 +268,11 @@ export function directContentEvidence(file: string): string | undefined { "templates/content/app/", "templates/content/data/", "templates/content/docs/product/", + "templates/content/drizzle/", "templates/content/e2e/", "templates/content/parity/", + "templates/content/public/", + "templates/content/scripts/", "templates/content/server/", "templates/content/shared/", "templates/content/.agents/skills/content-product-development/", @@ -278,7 +282,10 @@ export function directContentEvidence(file: string): string | undefined { if (prefixes.some((prefix) => normalized.startsWith(prefix))) { return normalized; } - if (CONTENT_ROOT_CONFIG.has(normalized)) { + if ( + /^templates\/content\/[^/]+$/.test(normalized) && + !CONTENT_ROOT_EXCLUSIONS.has(normalized) + ) { return normalized; } if ( @@ -549,7 +556,9 @@ function materializeContentProductSnapshot( const productRecordFiles = files.filter( (file) => file.endsWith(".md") && - /\/(?:chapters|features|capabilities)\//.test(file), + new RegExp( + `^${PRODUCT_ROOT}/(?:chapters|features|capabilities)/[^/]+\\.md$`, + ).test(file), ); if (productRecordFiles.length === 0) { throw new Error(`${sha} does not contain readable Content product records`); From 6ff1f1a9f30a90749069df129f1c15c0a6adb5b8 Mon Sep 17 00:00:00 2001 From: Alice Alexandra Moore <86723305+3mdistal@users.noreply.github.com> Date: Wed, 29 Jul 2026 18:40:38 -0400 Subject: [PATCH 06/13] Enforce Content workflow security policy --- package.json | 1 + scripts/run-guards.ts | 1 + ...te-content-product-impact-workflow.test.ts | 32 ++++++++++--- ...alidate-content-product-impact-workflow.ts | 45 ++++++++++++++++--- 4 files changed, 67 insertions(+), 12 deletions(-) diff --git a/package.json b/package.json index 47de94fd40..becb1790ee 100644 --- a/package.json +++ b/package.json @@ -61,6 +61,7 @@ "guard:template-list": "node scripts/guard-template-list.mjs", "guard:netlify-private-env": "tsx scripts/guard-netlify-private-env.ts", "guard:trusted-acceptance": "tsx scripts/guard-trusted-acceptance-workflow.ts", + "guard:content-product-conformance": "tsx scripts/validate-content-product-impact-workflow.ts", "guard:content-product-docs": "tsx scripts/validate-content-product-docs.ts", "test:content-product-docs": "tsx --test scripts/validate-content-product-docs.test.ts", "content-product-impact": "tsx scripts/validate-content-product-impact.ts", diff --git a/scripts/run-guards.ts b/scripts/run-guards.ts index 4c64d4d45b..06b97a446b 100644 --- a/scripts/run-guards.ts +++ b/scripts/run-guards.ts @@ -13,6 +13,7 @@ const guards = [ "guard:template-list", "guard:netlify-private-env", "guard:trusted-acceptance", + "guard:content-product-conformance", "guard:content-product-docs", "guard:workspace-skills", "guard:template-standard", diff --git a/scripts/validate-content-product-impact-workflow.test.ts b/scripts/validate-content-product-impact-workflow.test.ts index dd6c36f4f4..af8aae9e4e 100644 --- a/scripts/validate-content-product-impact-workflow.test.ts +++ b/scripts/validate-content-product-impact-workflow.test.ts @@ -47,12 +47,30 @@ describe("Content product conformance workflow boundary", () => { }); it("does not hide checker infrastructure failures", () => { - const unsafe = workflow.replace( - " run: pnpm content-product-impact", - " continue-on-error: true\n run: pnpm content-product-impact", - ); - const result = validateContentProductImpactWorkflow(unsafe); - assert.equal(result.ok, false); - assert(result.issues.some((issue) => issue.includes("infrastructure"))); + for (const value of ["true", "${{ true }}"]) { + const unsafe = workflow.replace( + " run: pnpm content-product-impact", + ` continue-on-error: ${value}\n run: pnpm content-product-impact`, + ); + const result = validateContentProductImpactWorkflow(unsafe); + assert.equal(result.ok, false); + assert(result.issues.some((issue) => issue.includes("infrastructure"))); + } + }); + + it("rejects dot and bracket references to the secrets context", () => { + for (const expression of [ + "${{ secrets.DEPLOY_KEY }}", + "${{ secrets['DEPLOY_KEY'] }}", + "${{ toJSON(secrets) }}", + ]) { + const unsafe = workflow.replace( + " timeout-minutes:", + ` env:\n LEAK: "${expression}"\n timeout-minutes:`, + ); + const result = validateContentProductImpactWorkflow(unsafe); + assert.equal(result.ok, false); + assert(result.issues.some((issue) => issue.includes("secrets"))); + } }); }); diff --git a/scripts/validate-content-product-impact-workflow.ts b/scripts/validate-content-product-impact-workflow.ts index 9272c9d611..80fcf01857 100644 --- a/scripts/validate-content-product-impact-workflow.ts +++ b/scripts/validate-content-product-impact-workflow.ts @@ -1,3 +1,5 @@ +import { readFileSync } from "node:fs"; + import { parse } from "yaml"; export type ContentImpactWorkflowGuardResult = { @@ -13,6 +15,14 @@ function permissionIsRead(value: unknown): boolean { return value === "read"; } +function containsSecretContext(value: unknown): boolean { + if (typeof value === "string") { + return /\$\{\{[\s\S]*?\bsecrets\b/i.test(value); + } + if (Array.isArray(value)) return value.some(containsSecretContext); + return isRecord(value) && Object.values(value).some(containsSecretContext); +} + export function validateContentProductImpactWorkflow( source: string, ): ContentImpactWorkflowGuardResult { @@ -78,13 +88,24 @@ export function validateContentProductImpactWorkflow( issues.push("workflow must define the check job"); return { ok: false, issues }; } - if ("environment" in check || /\bsecrets\./.test(source)) { + if ("environment" in check || containsSecretContext(workflow)) { issues.push("advisory check must not receive an environment or secrets"); } if (typeof check["timeout-minutes"] !== "number") { issues.push("check job must have an explicit timeout"); } const steps = Array.isArray(check.steps) ? check.steps.filter(isRecord) : []; + if ( + ("continue-on-error" in check && check["continue-on-error"] !== false) || + steps.some( + (step) => + "continue-on-error" in step && step["continue-on-error"] !== false, + ) + ) { + issues.push( + "workflow must not hide checker input or infrastructure failures", + ); + } const checkout = steps.find( (step) => typeof step.uses === "string" && @@ -108,10 +129,6 @@ export function validateContentProductImpactWorkflow( ); if (!runStep || !isRecord(runStep.env)) { issues.push("workflow must invoke the standalone impact checker"); - } else if (runStep["continue-on-error"] === true) { - issues.push( - "workflow must not hide checker input or infrastructure failures", - ); } else if ( runStep.env.CONTENT_IMPACT_BASE_SHA !== "${{ github.event.pull_request.base.sha }}" || @@ -123,3 +140,21 @@ export function validateContentProductImpactWorkflow( return { ok: issues.length === 0, issues }; } + +function main(): void { + const result = validateContentProductImpactWorkflow( + readFileSync(".github/workflows/content-product-conformance.yml", "utf8"), + ); + if (!result.ok) { + for (const issue of result.issues) { + console.error(`[content-product-conformance] ${issue}`); + } + process.exitCode = 1; + return; + } + console.log("Content product conformance workflow boundary checks passed."); +} + +if (process.argv[1]?.endsWith("validate-content-product-impact-workflow.ts")) { + main(); +} From 7d478948b428ab81397c05ccb37c664ad9c2f074 Mon Sep 17 00:00:00 2001 From: Alice Alexandra Moore <86723305+3mdistal@users.noreply.github.com> Date: Wed, 29 Jul 2026 18:53:39 -0400 Subject: [PATCH 07/13] Close Content workflow override paths --- ...te-content-product-impact-workflow.test.ts | 34 +++++++++++++++++++ ...alidate-content-product-impact-workflow.ts | 28 +++++++++++---- 2 files changed, 56 insertions(+), 6 deletions(-) diff --git a/scripts/validate-content-product-impact-workflow.test.ts b/scripts/validate-content-product-impact-workflow.test.ts index af8aae9e4e..42d675a02d 100644 --- a/scripts/validate-content-product-impact-workflow.test.ts +++ b/scripts/validate-content-product-impact-workflow.test.ts @@ -73,4 +73,38 @@ describe("Content product conformance workflow boundary", () => { assert(result.issues.some((issue) => issue.includes("secrets"))); } }); + + it("rejects job-level permission overrides and explicit GitHub tokens", () => { + const unsafe = workflow + .replace( + " name: Advisory Content product impact", + " name: Advisory Content product impact\n permissions:\n contents: write", + ) + .replace( + " timeout-minutes:", + ` env:\n GH_TOKEN: "\${{ github['token'] }}"\n timeout-minutes:`, + ); + const result = validateContentProductImpactWorkflow(unsafe); + assert.equal(result.ok, false); + assert(result.issues.some((issue) => issue.includes("permissions"))); + assert(result.issues.some((issue) => issue.includes("credentials"))); + }); + + it("requires the check job and impact checker step to be unconditional", () => { + const unsafe = workflow + .replace( + " name: Advisory Content product impact", + " name: Advisory Content product impact\n if: false", + ) + .replace( + " - name: Check Content product impact", + " - name: Check Content product impact\n if: false", + ); + const result = validateContentProductImpactWorkflow(unsafe); + assert.equal(result.ok, false); + assert.equal( + result.issues.filter((issue) => issue.includes("unconditionally")).length, + 2, + ); + }); }); diff --git a/scripts/validate-content-product-impact-workflow.ts b/scripts/validate-content-product-impact-workflow.ts index 80fcf01857..a978d6f0dd 100644 --- a/scripts/validate-content-product-impact-workflow.ts +++ b/scripts/validate-content-product-impact-workflow.ts @@ -15,12 +15,16 @@ function permissionIsRead(value: unknown): boolean { return value === "read"; } -function containsSecretContext(value: unknown): boolean { +function containsCredentialContext(value: unknown): boolean { if (typeof value === "string") { - return /\$\{\{[\s\S]*?\bsecrets\b/i.test(value); + return /\$\{\{[\s\S]*?(?:\bsecrets\b|\bgithub\s*(?:\.\s*token|\[\s*["']token["']\s*\]))/i.test( + value, + ); } - if (Array.isArray(value)) return value.some(containsSecretContext); - return isRecord(value) && Object.values(value).some(containsSecretContext); + if (Array.isArray(value)) return value.some(containsCredentialContext); + return ( + isRecord(value) && Object.values(value).some(containsCredentialContext) + ); } export function validateContentProductImpactWorkflow( @@ -88,8 +92,18 @@ export function validateContentProductImpactWorkflow( issues.push("workflow must define the check job"); return { ok: false, issues }; } - if ("environment" in check || containsSecretContext(workflow)) { - issues.push("advisory check must not receive an environment or secrets"); + if ("permissions" in check) { + issues.push( + "check job must inherit the exact read-only workflow permissions", + ); + } + if ("environment" in check || containsCredentialContext(workflow)) { + issues.push( + "advisory check must not receive an environment, secrets, or explicit credentials", + ); + } + if ("if" in check || "needs" in check) { + issues.push("check job must run unconditionally"); } if (typeof check["timeout-minutes"] !== "number") { issues.push("check job must have an explicit timeout"); @@ -129,6 +143,8 @@ export function validateContentProductImpactWorkflow( ); if (!runStep || !isRecord(runStep.env)) { issues.push("workflow must invoke the standalone impact checker"); + } else if ("if" in runStep) { + issues.push("impact checker step must run unconditionally"); } else if ( runStep.env.CONTENT_IMPACT_BASE_SHA !== "${{ github.event.pull_request.base.sha }}" || From 820a24829a93797261e1e1fc378151cc6c750524 Mon Sep 17 00:00:00 2001 From: Alice Alexandra Moore <86723305+3mdistal@users.noreply.github.com> Date: Wed, 29 Jul 2026 19:07:46 -0400 Subject: [PATCH 08/13] Finish Content snapshot credential boundaries --- ...te-content-product-impact-workflow.test.ts | 15 ++++++++++++ ...alidate-content-product-impact-workflow.ts | 20 +++++++++++++--- .../validate-content-product-impact.test.ts | 24 +++++++++++++++++++ scripts/validate-content-product-impact.ts | 11 ++++++--- 4 files changed, 64 insertions(+), 6 deletions(-) diff --git a/scripts/validate-content-product-impact-workflow.test.ts b/scripts/validate-content-product-impact-workflow.test.ts index 42d675a02d..206b4f7858 100644 --- a/scripts/validate-content-product-impact-workflow.test.ts +++ b/scripts/validate-content-product-impact-workflow.test.ts @@ -90,6 +90,21 @@ describe("Content product conformance workflow boundary", () => { assert(result.issues.some((issue) => issue.includes("credentials"))); }); + it("rejects serialization of the full GitHub context", () => { + for (const expression of [ + "${{ toJSON(github) }}", + "${{ TOJSON( GITHUB ) }}", + ]) { + const unsafe = workflow.replace( + " timeout-minutes:", + ` env:\n GITHUB_CONTEXT: "${expression}"\n timeout-minutes:`, + ); + const result = validateContentProductImpactWorkflow(unsafe); + assert.equal(result.ok, false); + assert(result.issues.some((issue) => issue.includes("credentials"))); + } + }); + it("requires the check job and impact checker step to be unconditional", () => { const unsafe = workflow .replace( diff --git a/scripts/validate-content-product-impact-workflow.ts b/scripts/validate-content-product-impact-workflow.ts index a978d6f0dd..4c810cf7c5 100644 --- a/scripts/validate-content-product-impact-workflow.ts +++ b/scripts/validate-content-product-impact-workflow.ts @@ -15,11 +15,25 @@ function permissionIsRead(value: unknown): boolean { return value === "read"; } +const ALLOWED_GITHUB_EXPRESSIONS = new Set([ + "github.event.pull_request.number", + "github.event.pull_request.base.sha", + "github.event.pull_request.head.sha", +]); + function containsCredentialContext(value: unknown): boolean { if (typeof value === "string") { - return /\$\{\{[\s\S]*?(?:\bsecrets\b|\bgithub\s*(?:\.\s*token|\[\s*["']token["']\s*\]))/i.test( - value, - ); + for (const match of value.matchAll(/\$\{\{([\s\S]*?)\}\}/g)) { + const expression = match[1].trim().toLowerCase(); + if (/\bsecrets\b/i.test(expression)) return true; + if ( + /\bgithub\b/i.test(expression) && + !ALLOWED_GITHUB_EXPRESSIONS.has(expression) + ) { + return true; + } + } + return false; } if (Array.isArray(value)) return value.some(containsCredentialContext); return ( diff --git a/scripts/validate-content-product-impact.test.ts b/scripts/validate-content-product-impact.test.ts index 234bb41649..05127aa4a9 100644 --- a/scripts/validate-content-product-impact.test.ts +++ b/scripts/validate-content-product-impact.test.ts @@ -450,6 +450,30 @@ describe("Content impact CLI boundary", () => { assert.doesNotMatch(result.stderr, /base Content product catalog/); }); + it("materializes linked non-Markdown files inside each snapshot", () => { + const repository = createCliRepository( + "linked-asset-cli", + (root) => { + const file = path.join(root, "templates/content/app/example.ts"); + mkdirSync(path.dirname(file), { recursive: true }); + writeFileSync(file, "export const example = true;\n"); + }, + "", + true, + (root) => { + const productRoot = path.join(root, "templates/content/docs/product"); + writeFileSync(path.join(productRoot, "contract.json"), "{}\n"); + writeFileSync( + path.join(productRoot, "README.md"), + "Read the [architecture](architecture.md) and [contract](contract.json).\n", + ); + }, + ); + const result = runCli(repository); + assert.equal(result.status, 0, result.stderr); + assert.doesNotMatch(result.stdout, /broken relative link/); + }); + it("uses the merge-base for a PR whose target branch has advanced", () => { const repository = createCliRepository( "diverged-base-cli", diff --git a/scripts/validate-content-product-impact.ts b/scripts/validate-content-product-impact.ts index 22c3c1a15e..3658e077e6 100644 --- a/scripts/validate-content-product-impact.ts +++ b/scripts/validate-content-product-impact.ts @@ -527,6 +527,13 @@ function git(args: string[]): string { }); } +function gitFile(sha: string, file: string): Buffer { + return execFileSync("git", ["show", `${sha}:${file}`], { + cwd: process.cwd(), + maxBuffer: 32 * 1024 * 1024, + }); +} + function assertSha(value: string, name: string): void { if (!/^[0-9a-f]{40,64}$/i.test(value)) { throw new Error(`${name} must be a full Git object SHA`); @@ -564,11 +571,9 @@ function materializeContentProductSnapshot( throw new Error(`${sha} does not contain readable Content product records`); } for (const file of files) { - if (!file.endsWith(".md")) continue; const destinationFile = path.join(destination, file); mkdirSync(path.dirname(destinationFile), { recursive: true }); - const source = git(["show", `${sha}:${file}`]); - writeFileSync(destinationFile, source); + writeFileSync(destinationFile, gitFile(sha, file)); } return root; } From 77b8c28249f7f73207883e6a7097a174c5c516a4 Mon Sep 17 00:00:00 2001 From: Alice Alexandra Moore <86723305+3mdistal@users.noreply.github.com> Date: Wed, 29 Jul 2026 19:09:53 -0400 Subject: [PATCH 09/13] Constrain Content conformance execution --- ...idate-content-product-impact-workflow.test.ts | 16 ++++++++++++++++ .../validate-content-product-impact-workflow.ts | 9 ++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/scripts/validate-content-product-impact-workflow.test.ts b/scripts/validate-content-product-impact-workflow.test.ts index 206b4f7858..cc591e4658 100644 --- a/scripts/validate-content-product-impact-workflow.test.ts +++ b/scripts/validate-content-product-impact-workflow.test.ts @@ -122,4 +122,20 @@ describe("Content product conformance workflow boundary", () => { 2, ); }); + + it("rejects extra jobs and additional checkout steps", () => { + const unsafe = workflow + .replace( + "jobs:\n", + "jobs:\n leak:\n runs-on: ubuntu-latest\n environment: production\n steps: []\n", + ) + .replace( + " - uses: pnpm/action-setup@", + " - uses: actions/checkout@unsafe\n\n - uses: pnpm/action-setup@", + ); + const result = validateContentProductImpactWorkflow(unsafe); + assert.equal(result.ok, false); + assert(result.issues.some((issue) => issue.includes("only"))); + assert(result.issues.some((issue) => issue.includes("one checkout"))); + }); }); diff --git a/scripts/validate-content-product-impact-workflow.ts b/scripts/validate-content-product-impact-workflow.ts index 4c810cf7c5..48565aaa32 100644 --- a/scripts/validate-content-product-impact-workflow.ts +++ b/scripts/validate-content-product-impact-workflow.ts @@ -101,6 +101,9 @@ export function validateContentProductImpactWorkflow( } const jobs = workflow.jobs; + if (!isRecord(jobs) || Object.keys(jobs).some((key) => key !== "check")) { + issues.push("workflow must contain only the guarded check job"); + } const check = isRecord(jobs) ? jobs.check : undefined; if (!isRecord(check)) { issues.push("workflow must define the check job"); @@ -134,11 +137,15 @@ export function validateContentProductImpactWorkflow( "workflow must not hide checker input or infrastructure failures", ); } - const checkout = steps.find( + const checkoutSteps = steps.filter( (step) => typeof step.uses === "string" && step.uses.startsWith("actions/checkout@"), ); + const checkout = checkoutSteps[0]; + if (checkoutSteps.length !== 1) { + issues.push("check job must contain exactly one checkout step"); + } if (!checkout || !isRecord(checkout.with)) { issues.push("workflow must check out the exact candidate revision"); } else { From 87e6c607bde471edfcd82fa75601ea34d74bb744 Mon Sep 17 00:00:00 2001 From: Alice Alexandra Moore <86723305+3mdistal@users.noreply.github.com> Date: Wed, 29 Jul 2026 19:15:56 -0400 Subject: [PATCH 10/13] Pin Content conformance execution surface --- ...te-content-product-impact-workflow.test.ts | 13 ++++ ...alidate-content-product-impact-workflow.ts | 64 +++++++++++++++++++ 2 files changed, 77 insertions(+) diff --git a/scripts/validate-content-product-impact-workflow.test.ts b/scripts/validate-content-product-impact-workflow.test.ts index cc591e4658..13fdbd913e 100644 --- a/scripts/validate-content-product-impact-workflow.test.ts +++ b/scripts/validate-content-product-impact-workflow.test.ts @@ -138,4 +138,17 @@ describe("Content product conformance workflow boundary", () => { assert(result.issues.some((issue) => issue.includes("only"))); assert(result.issues.some((issue) => issue.includes("one checkout"))); }); + + it("rejects self-hosted runners and arbitrary additional steps", () => { + const unsafe = workflow + .replace("runs-on: ubuntu-latest", "runs-on: self-hosted") + .replace( + " - uses: pnpm/action-setup@", + " - uses: arbitrary/action@unsafe\n\n - uses: pnpm/action-setup@", + ); + const result = validateContentProductImpactWorkflow(unsafe); + assert.equal(result.ok, false); + assert(result.issues.some((issue) => issue.includes("GitHub-hosted"))); + assert(result.issues.some((issue) => issue.includes("exactly match"))); + }); }); diff --git a/scripts/validate-content-product-impact-workflow.ts b/scripts/validate-content-product-impact-workflow.ts index 48565aaa32..02bfd182c3 100644 --- a/scripts/validate-content-product-impact-workflow.ts +++ b/scripts/validate-content-product-impact-workflow.ts @@ -20,6 +20,62 @@ const ALLOWED_GITHUB_EXPRESSIONS = new Set([ "github.event.pull_request.base.sha", "github.event.pull_request.head.sha", ]); +const CHECKOUT_ACTION = + "actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5"; +const PNPM_ACTION = + "pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320"; +const NODE_ACTION = + "actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020"; + +function hasExactKeys( + value: Record, + keys: readonly string[], +): boolean { + return ( + Object.keys(value).length === keys.length && + keys.every((key) => key in value) + ); +} + +function hasExpectedSteps(steps: Array>): boolean { + if (steps.length !== 5) return false; + const [checkout, pnpm, node, install, checker] = steps; + return ( + hasExactKeys(checkout, ["uses", "with"]) && + checkout.uses === CHECKOUT_ACTION && + isRecord(checkout.with) && + hasExactKeys(checkout.with, [ + "ref", + "fetch-depth", + "persist-credentials", + ]) && + checkout.with.ref === "${{ github.event.pull_request.head.sha }}" && + checkout.with["fetch-depth"] === 0 && + checkout.with["persist-credentials"] === false && + hasExactKeys(pnpm, ["uses"]) && + pnpm.uses === PNPM_ACTION && + hasExactKeys(node, ["uses", "with"]) && + node.uses === NODE_ACTION && + isRecord(node.with) && + hasExactKeys(node.with, ["node-version", "cache"]) && + node.with["node-version"] === "22" && + node.with.cache === "pnpm" && + hasExactKeys(install, ["run"]) && + install.run === "pnpm install --frozen-lockfile --ignore-scripts" && + hasExactKeys(checker, ["name", "env", "run"]) && + checker.name === "Check Content product impact" && + isRecord(checker.env) && + hasExactKeys(checker.env, [ + "CONTENT_IMPACT_BASE_SHA", + "CONTENT_IMPACT_HEAD_SHA", + ]) && + checker.env.CONTENT_IMPACT_BASE_SHA === + "${{ github.event.pull_request.base.sha }}" && + checker.env.CONTENT_IMPACT_HEAD_SHA === + "${{ github.event.pull_request.head.sha }}" && + checker.run === "pnpm content-product-impact" + ); +} function containsCredentialContext(value: unknown): boolean { if (typeof value === "string") { @@ -114,6 +170,9 @@ export function validateContentProductImpactWorkflow( "check job must inherit the exact read-only workflow permissions", ); } + if (check["runs-on"] !== "ubuntu-latest") { + issues.push("check job must use the approved GitHub-hosted runner"); + } if ("environment" in check || containsCredentialContext(workflow)) { issues.push( "advisory check must not receive an environment, secrets, or explicit credentials", @@ -126,6 +185,11 @@ export function validateContentProductImpactWorkflow( issues.push("check job must have an explicit timeout"); } const steps = Array.isArray(check.steps) ? check.steps.filter(isRecord) : []; + if (!hasExpectedSteps(steps)) { + issues.push( + "check job steps must exactly match the pinned checkout, setup, install, and checker sequence", + ); + } if ( ("continue-on-error" in check && check["continue-on-error"] !== false) || steps.some( From 03caa13fd5bf6176ee01ab223452db9932b7ca8c Mon Sep 17 00:00:00 2001 From: Alice Alexandra Moore <86723305+3mdistal@users.noreply.github.com> Date: Wed, 29 Jul 2026 19:23:10 -0400 Subject: [PATCH 11/13] Separate Content checker source repository --- scripts/validate-content-product-impact.ts | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/scripts/validate-content-product-impact.ts b/scripts/validate-content-product-impact.ts index 3658e077e6..eaa610f79b 100644 --- a/scripts/validate-content-product-impact.ts +++ b/scripts/validate-content-product-impact.ts @@ -22,10 +22,14 @@ import { const PRODUCT_ROOT = "templates/content/docs/product"; const CONTENT_PRODUCT_SKILL_ROOT = "templates/content/.agents/skills/content-product-development"; -const REPOSITORY_ROOT = path.resolve( +const CHECKER_ROOT = path.resolve( path.dirname(fileURLToPath(import.meta.url)), "..", ); +const SOURCE_REPOSITORY_ROOT = path.resolve( + process.cwd(), + process.env.CONTENT_IMPACT_REPOSITORY ?? ".", +); const LANES = [ "contract_repair", "contract_fulfillment", @@ -521,7 +525,7 @@ export function analyzeContentProductImpact( function git(args: string[]): string { return execFileSync("git", args, { - cwd: process.cwd(), + cwd: SOURCE_REPOSITORY_ROOT, encoding: "utf8", maxBuffer: 32 * 1024 * 1024, }); @@ -529,7 +533,7 @@ function git(args: string[]): string { function gitFile(sha: string, file: string): Buffer { return execFileSync("git", ["show", `${sha}:${file}`], { - cwd: process.cwd(), + cwd: SOURCE_REPOSITORY_ROOT, maxBuffer: 32 * 1024 * 1024, }); } @@ -665,7 +669,7 @@ export function runContentProductImpactCheck(): void { } const temporaryRoot = mkdtempSync( - path.join(REPOSITORY_ROOT, ".content-impact-"), + path.join(CHECKER_ROOT, ".content-impact-"), ); try { const baseSnapshotRoot = path.join(temporaryRoot, "base"); From 8943be3810642752227ca84890b6298b04104c38 Mon Sep 17 00:00:00 2001 From: Alice Alexandra Moore <86723305+3mdistal@users.noreply.github.com> Date: Wed, 29 Jul 2026 19:27:16 -0400 Subject: [PATCH 12/13] Use immutable Content conformance controller --- .../workflows/content-product-conformance.yml | 21 +++- ...te-content-product-impact-workflow.test.ts | 22 +++- ...alidate-content-product-impact-workflow.ts | 102 +++++++++++++----- 3 files changed, 114 insertions(+), 31 deletions(-) diff --git a/.github/workflows/content-product-conformance.yml b/.github/workflows/content-product-conformance.yml index a4ba0cfd1d..0ec199ada5 100644 --- a/.github/workflows/content-product-conformance.yml +++ b/.github/workflows/content-product-conformance.yml @@ -28,23 +28,38 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + - name: Check out immutable conformance controller + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 with: + ref: 03caa13fd5bf6176ee01ab223452db9932b7ca8c + path: controller + fetch-depth: 1 + persist-credentials: false + + - name: Check out candidate as inert data + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + repository: ${{ github.event.pull_request.head.repo.full_name }} ref: ${{ github.event.pull_request.head.sha }} + path: candidate fetch-depth: 0 persist-credentials: false - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4.4.0 + with: + package_json_file: controller/package.json - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version: "22" cache: pnpm + cache-dependency-path: controller/pnpm-lock.yaml - - run: pnpm install --frozen-lockfile --ignore-scripts + - run: pnpm --dir controller install --frozen-lockfile --ignore-scripts - name: Check Content product impact env: + CONTENT_IMPACT_REPOSITORY: ../candidate CONTENT_IMPACT_BASE_SHA: ${{ github.event.pull_request.base.sha }} CONTENT_IMPACT_HEAD_SHA: ${{ github.event.pull_request.head.sha }} - run: pnpm content-product-impact + run: pnpm --dir controller exec tsx scripts/validate-content-product-impact.ts diff --git a/scripts/validate-content-product-impact-workflow.test.ts b/scripts/validate-content-product-impact-workflow.test.ts index 13fdbd913e..be70db9928 100644 --- a/scripts/validate-content-product-impact-workflow.test.ts +++ b/scripts/validate-content-product-impact-workflow.test.ts @@ -49,8 +49,8 @@ describe("Content product conformance workflow boundary", () => { it("does not hide checker infrastructure failures", () => { for (const value of ["true", "${{ true }}"]) { const unsafe = workflow.replace( - " run: pnpm content-product-impact", - ` continue-on-error: ${value}\n run: pnpm content-product-impact`, + " run: pnpm --dir controller exec tsx scripts/validate-content-product-impact.ts", + ` continue-on-error: ${value}\n run: pnpm --dir controller exec tsx scripts/validate-content-product-impact.ts`, ); const result = validateContentProductImpactWorkflow(unsafe); assert.equal(result.ok, false); @@ -136,7 +136,7 @@ describe("Content product conformance workflow boundary", () => { const result = validateContentProductImpactWorkflow(unsafe); assert.equal(result.ok, false); assert(result.issues.some((issue) => issue.includes("only"))); - assert(result.issues.some((issue) => issue.includes("one checkout"))); + assert(result.issues.some((issue) => issue.includes("two checkout"))); }); it("rejects self-hosted runners and arbitrary additional steps", () => { @@ -151,4 +151,20 @@ describe("Content product conformance workflow boundary", () => { assert(result.issues.some((issue) => issue.includes("GitHub-hosted"))); assert(result.issues.some((issue) => issue.includes("exactly match"))); }); + + it("rejects a candidate-controlled controller or package script", () => { + const unsafe = workflow + .replace( + "ref: 03caa13fd5bf6176ee01ab223452db9932b7ca8c", + "ref: ${{ github.event.pull_request.head.sha }}", + ) + .replace( + "pnpm --dir controller exec tsx scripts/validate-content-product-impact.ts", + "pnpm --dir candidate content-product-impact", + ); + const result = validateContentProductImpactWorkflow(unsafe); + assert.equal(result.ok, false); + assert(result.issues.some((issue) => issue.includes("trusted revision"))); + assert(result.issues.some((issue) => issue.includes("standalone"))); + }); }); diff --git a/scripts/validate-content-product-impact-workflow.ts b/scripts/validate-content-product-impact-workflow.ts index 02bfd182c3..f83f8970b4 100644 --- a/scripts/validate-content-product-impact-workflow.ts +++ b/scripts/validate-content-product-impact-workflow.ts @@ -18,6 +18,7 @@ function permissionIsRead(value: unknown): boolean { const ALLOWED_GITHUB_EXPRESSIONS = new Set([ "github.event.pull_request.number", "github.event.pull_request.base.sha", + "github.event.pull_request.head.repo.full_name", "github.event.pull_request.head.sha", ]); const CHECKOUT_ACTION = @@ -26,6 +27,9 @@ const PNPM_ACTION = "pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320"; const NODE_ACTION = "actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020"; +const CONTROLLER_REVISION = "03caa13fd5bf6176ee01ab223452db9932b7ca8c"; +const CHECKER_COMMAND = + "pnpm --dir controller exec tsx scripts/validate-content-product-impact.ts"; function hasExactKeys( value: Record, @@ -38,42 +42,73 @@ function hasExactKeys( } function hasExpectedSteps(steps: Array>): boolean { - if (steps.length !== 5) return false; - const [checkout, pnpm, node, install, checker] = steps; + if (steps.length !== 6) return false; + const [controller, candidate, pnpm, node, install, checker] = steps; return ( - hasExactKeys(checkout, ["uses", "with"]) && - checkout.uses === CHECKOUT_ACTION && - isRecord(checkout.with) && - hasExactKeys(checkout.with, [ + hasExactKeys(controller, ["name", "uses", "with"]) && + controller.name === "Check out immutable conformance controller" && + controller.uses === CHECKOUT_ACTION && + isRecord(controller.with) && + hasExactKeys(controller.with, [ "ref", + "path", "fetch-depth", "persist-credentials", ]) && - checkout.with.ref === "${{ github.event.pull_request.head.sha }}" && - checkout.with["fetch-depth"] === 0 && - checkout.with["persist-credentials"] === false && - hasExactKeys(pnpm, ["uses"]) && + controller.with.ref === CONTROLLER_REVISION && + controller.with.path === "controller" && + controller.with["fetch-depth"] === 1 && + controller.with["persist-credentials"] === false && + hasExactKeys(candidate, ["name", "uses", "with"]) && + candidate.name === "Check out candidate as inert data" && + candidate.uses === CHECKOUT_ACTION && + isRecord(candidate.with) && + hasExactKeys(candidate.with, [ + "repository", + "ref", + "path", + "fetch-depth", + "persist-credentials", + ]) && + candidate.with.repository === + "${{ github.event.pull_request.head.repo.full_name }}" && + candidate.with.ref === "${{ github.event.pull_request.head.sha }}" && + candidate.with.path === "candidate" && + candidate.with["fetch-depth"] === 0 && + candidate.with["persist-credentials"] === false && + hasExactKeys(pnpm, ["uses", "with"]) && pnpm.uses === PNPM_ACTION && + isRecord(pnpm.with) && + hasExactKeys(pnpm.with, ["package_json_file"]) && + pnpm.with.package_json_file === "controller/package.json" && hasExactKeys(node, ["uses", "with"]) && node.uses === NODE_ACTION && isRecord(node.with) && - hasExactKeys(node.with, ["node-version", "cache"]) && + hasExactKeys(node.with, [ + "node-version", + "cache", + "cache-dependency-path", + ]) && node.with["node-version"] === "22" && node.with.cache === "pnpm" && + node.with["cache-dependency-path"] === "controller/pnpm-lock.yaml" && hasExactKeys(install, ["run"]) && - install.run === "pnpm install --frozen-lockfile --ignore-scripts" && + install.run === + "pnpm --dir controller install --frozen-lockfile --ignore-scripts" && hasExactKeys(checker, ["name", "env", "run"]) && checker.name === "Check Content product impact" && isRecord(checker.env) && hasExactKeys(checker.env, [ + "CONTENT_IMPACT_REPOSITORY", "CONTENT_IMPACT_BASE_SHA", "CONTENT_IMPACT_HEAD_SHA", ]) && + checker.env.CONTENT_IMPACT_REPOSITORY === "../candidate" && checker.env.CONTENT_IMPACT_BASE_SHA === "${{ github.event.pull_request.base.sha }}" && checker.env.CONTENT_IMPACT_HEAD_SHA === "${{ github.event.pull_request.head.sha }}" && - checker.run === "pnpm content-product-impact" + checker.run === CHECKER_COMMAND ); } @@ -206,31 +241,48 @@ export function validateContentProductImpactWorkflow( typeof step.uses === "string" && step.uses.startsWith("actions/checkout@"), ); - const checkout = checkoutSteps[0]; - if (checkoutSteps.length !== 1) { - issues.push("check job must contain exactly one checkout step"); + const controllerCheckout = checkoutSteps[0]; + const candidateCheckout = checkoutSteps[1]; + if (checkoutSteps.length !== 2) { + issues.push("check job must contain exactly two checkout steps"); + } + if ( + !controllerCheckout || + !isRecord(controllerCheckout.with) || + controllerCheckout.with.ref !== CONTROLLER_REVISION + ) { + issues.push("controller checkout must use the immutable trusted revision"); } - if (!checkout || !isRecord(checkout.with)) { + if (!candidateCheckout || !isRecord(candidateCheckout.with)) { issues.push("workflow must check out the exact candidate revision"); } else { - if (checkout.with.ref !== "${{ github.event.pull_request.head.sha }}") { + if ( + candidateCheckout.with.repository !== + "${{ github.event.pull_request.head.repo.full_name }}" || + candidateCheckout.with.ref !== "${{ github.event.pull_request.head.sha }}" + ) { issues.push("checkout ref must be the exact pull request head SHA"); } - if (checkout.with["fetch-depth"] !== 0) { + if (candidateCheckout.with["fetch-depth"] !== 0) { issues.push("checkout must fetch base history"); } - if (checkout.with["persist-credentials"] !== false) { - issues.push("checkout must not persist GitHub credentials"); - } } - const runStep = steps.find( - (step) => step.run === "pnpm content-product-impact", - ); + if ( + checkoutSteps.some( + (checkout) => + !isRecord(checkout.with) || + checkout.with["persist-credentials"] !== false, + ) + ) { + issues.push("checkout must not persist GitHub credentials"); + } + const runStep = steps.find((step) => step.run === CHECKER_COMMAND); if (!runStep || !isRecord(runStep.env)) { issues.push("workflow must invoke the standalone impact checker"); } else if ("if" in runStep) { issues.push("impact checker step must run unconditionally"); } else if ( + runStep.env.CONTENT_IMPACT_REPOSITORY !== "../candidate" || runStep.env.CONTENT_IMPACT_BASE_SHA !== "${{ github.event.pull_request.base.sha }}" || runStep.env.CONTENT_IMPACT_HEAD_SHA !== From bd308501ea468d0f55b1b72aaf61a3bb87935c3a Mon Sep 17 00:00:00 2001 From: Alice Alexandra Moore <86723305+3mdistal@users.noreply.github.com> Date: Wed, 29 Jul 2026 19:33:46 -0400 Subject: [PATCH 13/13] Fetch Content conformance base revision --- .github/workflows/content-product-conformance.yml | 3 +++ ...validate-content-product-impact-workflow.test.ts | 10 ++++++++++ scripts/validate-content-product-impact-workflow.ts | 13 +++++++++++-- 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/.github/workflows/content-product-conformance.yml b/.github/workflows/content-product-conformance.yml index 0ec199ada5..d36089e111 100644 --- a/.github/workflows/content-product-conformance.yml +++ b/.github/workflows/content-product-conformance.yml @@ -45,6 +45,9 @@ jobs: fetch-depth: 0 persist-credentials: false + - name: Fetch exact target revision + run: git -C candidate fetch --no-tags https://github.com/BuilderIO/agent-native.git ${{ github.event.pull_request.base.sha }} + - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4.4.0 with: package_json_file: controller/package.json diff --git a/scripts/validate-content-product-impact-workflow.test.ts b/scripts/validate-content-product-impact-workflow.test.ts index be70db9928..4b36435740 100644 --- a/scripts/validate-content-product-impact-workflow.test.ts +++ b/scripts/validate-content-product-impact-workflow.test.ts @@ -167,4 +167,14 @@ describe("Content product conformance workflow boundary", () => { assert(result.issues.some((issue) => issue.includes("trusted revision"))); assert(result.issues.some((issue) => issue.includes("standalone"))); }); + + it("requires the exact upstream base revision for fork pull requests", () => { + const unsafe = workflow.replace( + "git -C candidate fetch --no-tags https://github.com/BuilderIO/agent-native.git ${{ github.event.pull_request.base.sha }}", + "git -C candidate fetch origin main", + ); + const result = validateContentProductImpactWorkflow(unsafe); + assert.equal(result.ok, false); + assert(result.issues.some((issue) => issue.includes("target revision"))); + }); }); diff --git a/scripts/validate-content-product-impact-workflow.ts b/scripts/validate-content-product-impact-workflow.ts index f83f8970b4..ba9ffd961c 100644 --- a/scripts/validate-content-product-impact-workflow.ts +++ b/scripts/validate-content-product-impact-workflow.ts @@ -30,6 +30,8 @@ const NODE_ACTION = const CONTROLLER_REVISION = "03caa13fd5bf6176ee01ab223452db9932b7ca8c"; const CHECKER_COMMAND = "pnpm --dir controller exec tsx scripts/validate-content-product-impact.ts"; +const BASE_FETCH_COMMAND = + "git -C candidate fetch --no-tags https://github.com/BuilderIO/agent-native.git ${{ github.event.pull_request.base.sha }}"; function hasExactKeys( value: Record, @@ -42,8 +44,9 @@ function hasExactKeys( } function hasExpectedSteps(steps: Array>): boolean { - if (steps.length !== 6) return false; - const [controller, candidate, pnpm, node, install, checker] = steps; + if (steps.length !== 7) return false; + const [controller, candidate, fetchBase, pnpm, node, install, checker] = + steps; return ( hasExactKeys(controller, ["name", "uses", "with"]) && controller.name === "Check out immutable conformance controller" && @@ -76,6 +79,9 @@ function hasExpectedSteps(steps: Array>): boolean { candidate.with.path === "candidate" && candidate.with["fetch-depth"] === 0 && candidate.with["persist-credentials"] === false && + hasExactKeys(fetchBase, ["name", "run"]) && + fetchBase.name === "Fetch exact target revision" && + fetchBase.run === BASE_FETCH_COMMAND && hasExactKeys(pnpm, ["uses", "with"]) && pnpm.uses === PNPM_ACTION && isRecord(pnpm.with) && @@ -290,6 +296,9 @@ export function validateContentProductImpactWorkflow( ) { issues.push("checker must receive exact base and head SHAs"); } + if (!steps.some((step) => step.run === BASE_FETCH_COMMAND)) { + issues.push("workflow must fetch the exact target revision from upstream"); + } return { ok: issues.length === 0, issues }; }