diff --git a/.github/workflows/content-product-conformance.yml b/.github/workflows/content-product-conformance.yml new file mode 100644 index 0000000000..d36089e111 --- /dev/null +++ b/.github/workflows/content-product-conformance.yml @@ -0,0 +1,68 @@ +name: Content product conformance + +on: + pull_request: + branches: [main] + types: + [ + opened, + synchronize, + reopened, + edited, + ready_for_review, + labeled, + unlabeled, + ] + +permissions: + contents: read + pull-requests: read + +concurrency: + group: content-product-conformance-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + check: + name: Advisory Content product impact + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Check out immutable conformance controller + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + ref: 03caa13fd5bf6176ee01ab223452db9932b7ca8c + path: controller + fetch-depth: 1 + persist-credentials: false + + - name: Check out candidate as inert data + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.sha }} + path: candidate + fetch-depth: 0 + persist-credentials: false + + - name: Fetch exact target revision + run: git -C candidate fetch --no-tags https://github.com/BuilderIO/agent-native.git ${{ github.event.pull_request.base.sha }} + + - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4.4.0 + with: + package_json_file: controller/package.json + + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: "22" + cache: pnpm + cache-dependency-path: controller/pnpm-lock.yaml + + - run: pnpm --dir controller install --frozen-lockfile --ignore-scripts + + - name: Check Content product impact + env: + CONTENT_IMPACT_REPOSITORY: ../candidate + CONTENT_IMPACT_BASE_SHA: ${{ github.event.pull_request.base.sha }} + CONTENT_IMPACT_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: pnpm --dir controller exec tsx scripts/validate-content-product-impact.ts diff --git a/package.json b/package.json index d66117ebb3..becb1790ee 100644 --- a/package.json +++ b/package.json @@ -61,8 +61,11 @@ "guard:template-list": "node scripts/guard-template-list.mjs", "guard:netlify-private-env": "tsx scripts/guard-netlify-private-env.ts", "guard:trusted-acceptance": "tsx scripts/guard-trusted-acceptance-workflow.ts", + "guard:content-product-conformance": "tsx scripts/validate-content-product-impact-workflow.ts", "guard:content-product-docs": "tsx scripts/validate-content-product-docs.ts", "test:content-product-docs": "tsx --test scripts/validate-content-product-docs.test.ts", + "content-product-impact": "tsx scripts/validate-content-product-impact.ts", + "test:content-product-impact": "tsx --test scripts/validate-content-product-impact.test.ts scripts/validate-content-product-impact-workflow.test.ts", "guard:workspace-skills": "tsx scripts/sync-workspace-core-skills.ts --check", "sync:template-standard": "tsx scripts/template-standard/index.ts", "guard:template-standard": "tsx scripts/template-standard/index.ts --check", diff --git a/scripts/run-guards.ts b/scripts/run-guards.ts index 4c64d4d45b..06b97a446b 100644 --- a/scripts/run-guards.ts +++ b/scripts/run-guards.ts @@ -13,6 +13,7 @@ const guards = [ "guard:template-list", "guard:netlify-private-env", "guard:trusted-acceptance", + "guard:content-product-conformance", "guard:content-product-docs", "guard:workspace-skills", "guard:template-standard", diff --git a/scripts/validate-content-product-docs.ts b/scripts/validate-content-product-docs.ts index d1b5a34217..14f690aaa1 100644 --- a/scripts/validate-content-product-docs.ts +++ b/scripts/validate-content-product-docs.ts @@ -50,9 +50,9 @@ const roadmapBoundaries = new Set([ "superseded", ]); -type RecordKind = "chapter" | "feature" | "capability"; +export type RecordKind = "chapter" | "feature" | "capability"; -type ProductRecord = { +export type ProductRecord = { file: string; body: string; data: Record; @@ -60,7 +60,7 @@ type ProductRecord = { id: string; }; -type ProductCatalog = { +export type ProductCatalog = { root: string; chapters: ProductRecord[]; features: ProductRecord[]; @@ -77,7 +77,11 @@ export type ValidationResult = { export function validateContentProductDocs( root = defaultProductRoot, - options: { strictCatalog?: boolean; checkProjections?: boolean } = {}, + options: { + strictCatalog?: boolean; + checkProjections?: boolean; + validationRoot?: string; + } = {}, ): ValidationResult { const strictCatalog = options.strictCatalog ?? root === defaultProductRoot; const checkProjections = options.checkProjections ?? true; @@ -85,7 +89,11 @@ export function validateContentProductDocs( const catalog = loadCatalog(root, errors); validateCatalog(catalog, errors, strictCatalog); - validateLinksAndPrivacy(catalog, errors); + validateLinksAndPrivacy( + catalog, + errors, + options.validationRoot ?? repositoryRoot, + ); const roadmap = renderRoadmap(catalog); const encyclopedia = renderEncyclopedia(catalog); @@ -803,11 +811,15 @@ function validateCapabilityMiniSpec( } } -function validateLinksAndPrivacy(catalog: ProductCatalog, errors: string[]) { +function validateLinksAndPrivacy( + catalog: ProductCatalog, + errors: string[], + validationRoot: string, +) { const files = collectMarkdownFiles(catalog.root); const skillRoot = join( - repositoryRoot, - "templates/content/.agents/skills/content-product-development", + resolve(catalog.root, "../.."), + ".agents/skills/content-product-development", ); if (existsSync(skillRoot)) files.push(...collectMarkdownFiles(skillRoot)); @@ -859,7 +871,7 @@ function validateLinksAndPrivacy(catalog: ProductCatalog, errors: string[]) { } const withoutAnchor = target.split("#")[0]; const resolved = resolve(dirname(file), withoutAnchor); - if (!isInside(repositoryRoot, resolved)) { + if (!isInside(validationRoot, resolved)) { errors.push( `${displayPath(file)}:${lineNumber(source, match.index ?? 0)}: link escapes the repository: ${target}`, ); diff --git a/scripts/validate-content-product-impact-workflow.test.ts b/scripts/validate-content-product-impact-workflow.test.ts new file mode 100644 index 0000000000..4b36435740 --- /dev/null +++ b/scripts/validate-content-product-impact-workflow.test.ts @@ -0,0 +1,180 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { describe, it } from "node:test"; + +import { validateContentProductImpactWorkflow } from "./validate-content-product-impact-workflow.ts"; + +const workflow = readFileSync( + ".github/workflows/content-product-conformance.yml", + "utf8", +); + +describe("Content product conformance workflow boundary", () => { + it("keeps the pilot read-only, exact-revision, and broadly observable", () => { + assert.deepEqual(validateContentProductImpactWorkflow(workflow), { + ok: true, + issues: [], + }); + }); + + it("rejects pull_request_target and write permissions", () => { + const unsafe = workflow + .replace(" pull_request:", " pull_request_target:") + .replace("contents: read", "contents: write"); + const result = validateContentProductImpactWorkflow(unsafe); + assert.equal(result.ok, false); + assert( + result.issues.some((issue) => issue.includes("pull_request_target")), + ); + assert(result.issues.some((issue) => issue.includes("permissions"))); + }); + + it("rejects path filtering or a candidate checkout with credentials", () => { + const unsafe = workflow + .replace(" types:", ' paths: ["templates/content/**"]\n types:') + .replace("persist-credentials: false", "persist-credentials: true"); + const result = validateContentProductImpactWorkflow(unsafe); + assert.equal(result.ok, false); + assert(result.issues.some((issue) => issue.includes("path filters"))); + assert(result.issues.some((issue) => issue.includes("credentials"))); + }); + + it("requires declaration and label edits to rerun the pilot", () => { + const unsafe = workflow.replace(" edited,\n", ""); + const result = validateContentProductImpactWorkflow(unsafe); + assert.equal(result.ok, false); + assert(result.issues.some((issue) => issue.includes("recalibration"))); + }); + + it("does not hide checker infrastructure failures", () => { + for (const value of ["true", "${{ true }}"]) { + const unsafe = workflow.replace( + " run: pnpm --dir controller exec tsx scripts/validate-content-product-impact.ts", + ` continue-on-error: ${value}\n run: pnpm --dir controller exec tsx scripts/validate-content-product-impact.ts`, + ); + const result = validateContentProductImpactWorkflow(unsafe); + assert.equal(result.ok, false); + assert(result.issues.some((issue) => issue.includes("infrastructure"))); + } + }); + + it("rejects dot and bracket references to the secrets context", () => { + for (const expression of [ + "${{ secrets.DEPLOY_KEY }}", + "${{ secrets['DEPLOY_KEY'] }}", + "${{ toJSON(secrets) }}", + ]) { + const unsafe = workflow.replace( + " timeout-minutes:", + ` env:\n LEAK: "${expression}"\n timeout-minutes:`, + ); + const result = validateContentProductImpactWorkflow(unsafe); + assert.equal(result.ok, false); + assert(result.issues.some((issue) => issue.includes("secrets"))); + } + }); + + it("rejects job-level permission overrides and explicit GitHub tokens", () => { + const unsafe = workflow + .replace( + " name: Advisory Content product impact", + " name: Advisory Content product impact\n permissions:\n contents: write", + ) + .replace( + " timeout-minutes:", + ` env:\n GH_TOKEN: "\${{ github['token'] }}"\n timeout-minutes:`, + ); + const result = validateContentProductImpactWorkflow(unsafe); + assert.equal(result.ok, false); + assert(result.issues.some((issue) => issue.includes("permissions"))); + assert(result.issues.some((issue) => issue.includes("credentials"))); + }); + + it("rejects serialization of the full GitHub context", () => { + for (const expression of [ + "${{ toJSON(github) }}", + "${{ TOJSON( GITHUB ) }}", + ]) { + const unsafe = workflow.replace( + " timeout-minutes:", + ` env:\n GITHUB_CONTEXT: "${expression}"\n timeout-minutes:`, + ); + const result = validateContentProductImpactWorkflow(unsafe); + assert.equal(result.ok, false); + assert(result.issues.some((issue) => issue.includes("credentials"))); + } + }); + + it("requires the check job and impact checker step to be unconditional", () => { + const unsafe = workflow + .replace( + " name: Advisory Content product impact", + " name: Advisory Content product impact\n if: false", + ) + .replace( + " - name: Check Content product impact", + " - name: Check Content product impact\n if: false", + ); + const result = validateContentProductImpactWorkflow(unsafe); + assert.equal(result.ok, false); + assert.equal( + result.issues.filter((issue) => issue.includes("unconditionally")).length, + 2, + ); + }); + + it("rejects extra jobs and additional checkout steps", () => { + const unsafe = workflow + .replace( + "jobs:\n", + "jobs:\n leak:\n runs-on: ubuntu-latest\n environment: production\n steps: []\n", + ) + .replace( + " - uses: pnpm/action-setup@", + " - uses: actions/checkout@unsafe\n\n - uses: pnpm/action-setup@", + ); + const result = validateContentProductImpactWorkflow(unsafe); + assert.equal(result.ok, false); + assert(result.issues.some((issue) => issue.includes("only"))); + assert(result.issues.some((issue) => issue.includes("two checkout"))); + }); + + it("rejects self-hosted runners and arbitrary additional steps", () => { + const unsafe = workflow + .replace("runs-on: ubuntu-latest", "runs-on: self-hosted") + .replace( + " - uses: pnpm/action-setup@", + " - uses: arbitrary/action@unsafe\n\n - uses: pnpm/action-setup@", + ); + const result = validateContentProductImpactWorkflow(unsafe); + assert.equal(result.ok, false); + assert(result.issues.some((issue) => issue.includes("GitHub-hosted"))); + assert(result.issues.some((issue) => issue.includes("exactly match"))); + }); + + it("rejects a candidate-controlled controller or package script", () => { + const unsafe = workflow + .replace( + "ref: 03caa13fd5bf6176ee01ab223452db9932b7ca8c", + "ref: ${{ github.event.pull_request.head.sha }}", + ) + .replace( + "pnpm --dir controller exec tsx scripts/validate-content-product-impact.ts", + "pnpm --dir candidate content-product-impact", + ); + const result = validateContentProductImpactWorkflow(unsafe); + assert.equal(result.ok, false); + assert(result.issues.some((issue) => issue.includes("trusted revision"))); + assert(result.issues.some((issue) => issue.includes("standalone"))); + }); + + it("requires the exact upstream base revision for fork pull requests", () => { + const unsafe = workflow.replace( + "git -C candidate fetch --no-tags https://github.com/BuilderIO/agent-native.git ${{ github.event.pull_request.base.sha }}", + "git -C candidate fetch origin main", + ); + const result = validateContentProductImpactWorkflow(unsafe); + assert.equal(result.ok, false); + assert(result.issues.some((issue) => issue.includes("target revision"))); + }); +}); diff --git a/scripts/validate-content-product-impact-workflow.ts b/scripts/validate-content-product-impact-workflow.ts new file mode 100644 index 0000000000..ba9ffd961c --- /dev/null +++ b/scripts/validate-content-product-impact-workflow.ts @@ -0,0 +1,322 @@ +import { readFileSync } from "node:fs"; + +import { parse } from "yaml"; + +export type ContentImpactWorkflowGuardResult = { + ok: boolean; + issues: string[]; +}; + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function permissionIsRead(value: unknown): boolean { + return value === "read"; +} + +const ALLOWED_GITHUB_EXPRESSIONS = new Set([ + "github.event.pull_request.number", + "github.event.pull_request.base.sha", + "github.event.pull_request.head.repo.full_name", + "github.event.pull_request.head.sha", +]); +const CHECKOUT_ACTION = + "actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5"; +const PNPM_ACTION = + "pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320"; +const NODE_ACTION = + "actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020"; +const CONTROLLER_REVISION = "03caa13fd5bf6176ee01ab223452db9932b7ca8c"; +const CHECKER_COMMAND = + "pnpm --dir controller exec tsx scripts/validate-content-product-impact.ts"; +const BASE_FETCH_COMMAND = + "git -C candidate fetch --no-tags https://github.com/BuilderIO/agent-native.git ${{ github.event.pull_request.base.sha }}"; + +function hasExactKeys( + value: Record, + keys: readonly string[], +): boolean { + return ( + Object.keys(value).length === keys.length && + keys.every((key) => key in value) + ); +} + +function hasExpectedSteps(steps: Array>): boolean { + if (steps.length !== 7) return false; + const [controller, candidate, fetchBase, pnpm, node, install, checker] = + steps; + return ( + hasExactKeys(controller, ["name", "uses", "with"]) && + controller.name === "Check out immutable conformance controller" && + controller.uses === CHECKOUT_ACTION && + isRecord(controller.with) && + hasExactKeys(controller.with, [ + "ref", + "path", + "fetch-depth", + "persist-credentials", + ]) && + controller.with.ref === CONTROLLER_REVISION && + controller.with.path === "controller" && + controller.with["fetch-depth"] === 1 && + controller.with["persist-credentials"] === false && + hasExactKeys(candidate, ["name", "uses", "with"]) && + candidate.name === "Check out candidate as inert data" && + candidate.uses === CHECKOUT_ACTION && + isRecord(candidate.with) && + hasExactKeys(candidate.with, [ + "repository", + "ref", + "path", + "fetch-depth", + "persist-credentials", + ]) && + candidate.with.repository === + "${{ github.event.pull_request.head.repo.full_name }}" && + candidate.with.ref === "${{ github.event.pull_request.head.sha }}" && + candidate.with.path === "candidate" && + candidate.with["fetch-depth"] === 0 && + candidate.with["persist-credentials"] === false && + hasExactKeys(fetchBase, ["name", "run"]) && + fetchBase.name === "Fetch exact target revision" && + fetchBase.run === BASE_FETCH_COMMAND && + hasExactKeys(pnpm, ["uses", "with"]) && + pnpm.uses === PNPM_ACTION && + isRecord(pnpm.with) && + hasExactKeys(pnpm.with, ["package_json_file"]) && + pnpm.with.package_json_file === "controller/package.json" && + hasExactKeys(node, ["uses", "with"]) && + node.uses === NODE_ACTION && + isRecord(node.with) && + hasExactKeys(node.with, [ + "node-version", + "cache", + "cache-dependency-path", + ]) && + node.with["node-version"] === "22" && + node.with.cache === "pnpm" && + node.with["cache-dependency-path"] === "controller/pnpm-lock.yaml" && + hasExactKeys(install, ["run"]) && + install.run === + "pnpm --dir controller install --frozen-lockfile --ignore-scripts" && + hasExactKeys(checker, ["name", "env", "run"]) && + checker.name === "Check Content product impact" && + isRecord(checker.env) && + hasExactKeys(checker.env, [ + "CONTENT_IMPACT_REPOSITORY", + "CONTENT_IMPACT_BASE_SHA", + "CONTENT_IMPACT_HEAD_SHA", + ]) && + checker.env.CONTENT_IMPACT_REPOSITORY === "../candidate" && + checker.env.CONTENT_IMPACT_BASE_SHA === + "${{ github.event.pull_request.base.sha }}" && + checker.env.CONTENT_IMPACT_HEAD_SHA === + "${{ github.event.pull_request.head.sha }}" && + checker.run === CHECKER_COMMAND + ); +} + +function containsCredentialContext(value: unknown): boolean { + if (typeof value === "string") { + for (const match of value.matchAll(/\$\{\{([\s\S]*?)\}\}/g)) { + const expression = match[1].trim().toLowerCase(); + if (/\bsecrets\b/i.test(expression)) return true; + if ( + /\bgithub\b/i.test(expression) && + !ALLOWED_GITHUB_EXPRESSIONS.has(expression) + ) { + return true; + } + } + return false; + } + if (Array.isArray(value)) return value.some(containsCredentialContext); + return ( + isRecord(value) && Object.values(value).some(containsCredentialContext) + ); +} + +export function validateContentProductImpactWorkflow( + source: string, +): ContentImpactWorkflowGuardResult { + const issues: string[] = []; + let workflow: unknown; + try { + workflow = parse(source); + } catch { + return { ok: false, issues: ["workflow must be valid YAML"] }; + } + if (!isRecord(workflow)) { + return { ok: false, issues: ["workflow must be a YAML mapping"] }; + } + + const trigger = workflow.on; + if (!isRecord(trigger) || !isRecord(trigger.pull_request)) { + issues.push("workflow must use pull_request"); + } else { + const pullRequest = trigger.pull_request; + const requiredTypes = [ + "opened", + "synchronize", + "reopened", + "edited", + "ready_for_review", + "labeled", + "unlabeled", + ]; + const eventTypes = pullRequest.types; + if ( + !Array.isArray(eventTypes) || + requiredTypes.some((type) => !eventTypes.includes(type)) + ) { + issues.push( + "pull_request must include every advisory recalibration event", + ); + } + if ("paths" in pullRequest || "paths-ignore" in pullRequest) { + issues.push("workflow must not use path filters"); + } + } + if ("pull_request_target" in (isRecord(trigger) ? trigger : {})) { + issues.push("workflow must not use pull_request_target"); + } + + const permissions = workflow.permissions; + if ( + !isRecord(permissions) || + !permissionIsRead(permissions.contents) || + !permissionIsRead(permissions["pull-requests"]) || + Object.keys(permissions).some( + (key) => key !== "contents" && key !== "pull-requests", + ) + ) { + issues.push( + "permissions must be exactly contents: read and pull-requests: read", + ); + } + + const jobs = workflow.jobs; + if (!isRecord(jobs) || Object.keys(jobs).some((key) => key !== "check")) { + issues.push("workflow must contain only the guarded check job"); + } + const check = isRecord(jobs) ? jobs.check : undefined; + if (!isRecord(check)) { + issues.push("workflow must define the check job"); + return { ok: false, issues }; + } + if ("permissions" in check) { + issues.push( + "check job must inherit the exact read-only workflow permissions", + ); + } + if (check["runs-on"] !== "ubuntu-latest") { + issues.push("check job must use the approved GitHub-hosted runner"); + } + if ("environment" in check || containsCredentialContext(workflow)) { + issues.push( + "advisory check must not receive an environment, secrets, or explicit credentials", + ); + } + if ("if" in check || "needs" in check) { + issues.push("check job must run unconditionally"); + } + if (typeof check["timeout-minutes"] !== "number") { + issues.push("check job must have an explicit timeout"); + } + const steps = Array.isArray(check.steps) ? check.steps.filter(isRecord) : []; + if (!hasExpectedSteps(steps)) { + issues.push( + "check job steps must exactly match the pinned checkout, setup, install, and checker sequence", + ); + } + if ( + ("continue-on-error" in check && check["continue-on-error"] !== false) || + steps.some( + (step) => + "continue-on-error" in step && step["continue-on-error"] !== false, + ) + ) { + issues.push( + "workflow must not hide checker input or infrastructure failures", + ); + } + const checkoutSteps = steps.filter( + (step) => + typeof step.uses === "string" && + step.uses.startsWith("actions/checkout@"), + ); + const controllerCheckout = checkoutSteps[0]; + const candidateCheckout = checkoutSteps[1]; + if (checkoutSteps.length !== 2) { + issues.push("check job must contain exactly two checkout steps"); + } + if ( + !controllerCheckout || + !isRecord(controllerCheckout.with) || + controllerCheckout.with.ref !== CONTROLLER_REVISION + ) { + issues.push("controller checkout must use the immutable trusted revision"); + } + if (!candidateCheckout || !isRecord(candidateCheckout.with)) { + issues.push("workflow must check out the exact candidate revision"); + } else { + if ( + candidateCheckout.with.repository !== + "${{ github.event.pull_request.head.repo.full_name }}" || + candidateCheckout.with.ref !== "${{ github.event.pull_request.head.sha }}" + ) { + issues.push("checkout ref must be the exact pull request head SHA"); + } + if (candidateCheckout.with["fetch-depth"] !== 0) { + issues.push("checkout must fetch base history"); + } + } + if ( + checkoutSteps.some( + (checkout) => + !isRecord(checkout.with) || + checkout.with["persist-credentials"] !== false, + ) + ) { + issues.push("checkout must not persist GitHub credentials"); + } + const runStep = steps.find((step) => step.run === CHECKER_COMMAND); + if (!runStep || !isRecord(runStep.env)) { + issues.push("workflow must invoke the standalone impact checker"); + } else if ("if" in runStep) { + issues.push("impact checker step must run unconditionally"); + } else if ( + runStep.env.CONTENT_IMPACT_REPOSITORY !== "../candidate" || + runStep.env.CONTENT_IMPACT_BASE_SHA !== + "${{ github.event.pull_request.base.sha }}" || + runStep.env.CONTENT_IMPACT_HEAD_SHA !== + "${{ github.event.pull_request.head.sha }}" + ) { + issues.push("checker must receive exact base and head SHAs"); + } + if (!steps.some((step) => step.run === BASE_FETCH_COMMAND)) { + issues.push("workflow must fetch the exact target revision from upstream"); + } + + return { ok: issues.length === 0, issues }; +} + +function main(): void { + const result = validateContentProductImpactWorkflow( + readFileSync(".github/workflows/content-product-conformance.yml", "utf8"), + ); + if (!result.ok) { + for (const issue of result.issues) { + console.error(`[content-product-conformance] ${issue}`); + } + process.exitCode = 1; + return; + } + console.log("Content product conformance workflow boundary checks passed."); +} + +if (process.argv[1]?.endsWith("validate-content-product-impact-workflow.ts")) { + main(); +} diff --git a/scripts/validate-content-product-impact.test.ts b/scripts/validate-content-product-impact.test.ts new file mode 100644 index 0000000000..05127aa4a9 --- /dev/null +++ b/scripts/validate-content-product-impact.test.ts @@ -0,0 +1,582 @@ +import assert from "node:assert/strict"; +import { execFileSync, spawnSync } from "node:child_process"; +import { + cpSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { after, before, describe, it } from "node:test"; + +import { validateContentProductDocs } from "./validate-content-product-docs.ts"; +import { + analyzeContentProductImpact, + collectTransitions, + directContentEvidence, + parseContentImpactDeclaration, +} from "./validate-content-product-impact.ts"; + +const fixture = "scripts/fixtures/content-product-docs/valid"; +const repositoryRoot = process.cwd(); +const checkerPath = path.join( + repositoryRoot, + "scripts/validate-content-product-impact.ts", +); +const tsxPath = path.join(repositoryRoot, "node_modules/.bin/tsx"); +let temporaryRoot = ""; +let baseRoot = ""; +let headRoot = ""; + +function declaration(overrides: string[] = []): string { + return [ + "```yaml", + "content_product_impact:", + " lane: contract_fulfillment", + " features:", + " - content.feature.test", + " capabilities:", + " - content.test.alpha", + " record_change: included", + " proof:", + " - pnpm test:content-product-impact", + " rationale: The fixture changes the declared Content contract.", + ...overrides, + "```", + ].join("\n"); +} + +before(() => { + temporaryRoot = mkdtempSync(path.join(tmpdir(), "content-impact-test-")); + baseRoot = path.join(temporaryRoot, "base"); + headRoot = path.join(temporaryRoot, "head"); + cpSync(fixture, baseRoot, { recursive: true }); + cpSync(fixture, headRoot, { recursive: true }); + + const feature = path.join(headRoot, "features/content.feature.test.md"); + writeFileSync( + feature, + readFileSync(feature, "utf8").replace( + 'roadmap_status: "planned"', + 'roadmap_status: "in_validation"', + ), + ); + const capability = path.join(headRoot, "capabilities/content.test.alpha.md"); + writeFileSync( + capability, + readFileSync(capability, "utf8").replace( + 'state: "approved_shape"', + 'state: "in_progress"', + ), + ); +}); + +after(() => rmSync(temporaryRoot, { recursive: true, force: true })); + +describe("Content impact declaration", () => { + it("parses one complete declaration", () => { + const result = parseContentImpactDeclaration(declaration()); + assert.equal(result.status, "valid"); + if (result.status !== "valid") return; + assert.deepEqual(result.declaration.features, ["content.feature.test"]); + assert.deepEqual(result.declaration.capabilities, ["content.test.alpha"]); + }); + + it("distinguishes missing, duplicate, malformed, and unknown-key blocks", () => { + assert.equal( + parseContentImpactDeclaration("No impact block.").status, + "missing", + ); + assert.equal( + parseContentImpactDeclaration(`${declaration()}\n${declaration()}`) + .status, + "duplicate", + ); + assert.equal( + parseContentImpactDeclaration( + "```yaml\ncontent_product_impact: [unterminated\n```", + ).status, + "malformed", + ); + const unknown = parseContentImpactDeclaration( + declaration([" surprise: no"]), + ); + assert.equal(unknown.status, "malformed"); + if (unknown.status === "malformed") { + assert(unknown.errors.some((error) => error.includes("unknown field"))); + } + const sibling = parseContentImpactDeclaration( + declaration(["unrelated_top_level_key: true"]), + ); + assert.equal(sibling.status, "malformed"); + }); + + it("requires exact IDs except for a pending product decision", () => { + const emptyFulfillment = parseContentImpactDeclaration( + declaration() + .replace(" - content.feature.test\n", " []\n") + .replace(" - content.test.alpha\n", " []\n"), + ); + assert.equal(emptyFulfillment.status, "malformed"); + + const pendingDecision = parseContentImpactDeclaration( + declaration() + .replace("contract_fulfillment", "product_decision_candidate") + .replace(" - content.feature.test\n", " []\n") + .replace(" - content.test.alpha\n", " []\n") + .replace("record_change: included", "record_change: decision_pending"), + ); + assert.equal(pendingDecision.status, "valid"); + + const includedNewRecord = parseContentImpactDeclaration( + declaration() + .replace("contract_fulfillment", "product_decision_candidate") + .replace(" - content.feature.test\n", " []\n") + .replace(" - content.test.alpha\n", " []\n"), + ); + assert.equal(includedNewRecord.status, "valid"); + }); +}); + +describe("Content impact applicability", () => { + it("automatically includes direct Content surfaces and exact conformance policy", () => { + assert(directContentEvidence("templates/content/app/routes/home.tsx")); + assert( + directContentEvidence("templates/content/actions/create-document.ts"), + ); + assert(directContentEvidence("templates/content/docs/product/roadmap.md")); + assert(directContentEvidence("templates/content/parity/contract.test.ts")); + assert(directContentEvidence("templates/content/vite.config.ts")); + assert(directContentEvidence("templates/content/tsconfig.json")); + assert(directContentEvidence("templates/content/vitest.config.ts")); + assert(directContentEvidence("templates/content/drizzle/0001_example.sql")); + assert(directContentEvidence("templates/content/public/example.svg")); + assert.equal( + directContentEvidence("templates/content/README.md"), + undefined, + ); + assert.equal( + directContentEvidence("templates/content/.oxfmtrc.json"), + undefined, + ); + assert.equal( + directContentEvidence("scripts/validate-content-product-impact.ts"), + undefined, + ); + assert.equal( + directContentEvidence( + ".github/workflows/content-product-conformance.yml", + ), + undefined, + ); + }); + + it("keeps ordinary shared framework, CI, dependency, and infrastructure changes quiet", () => { + for (const file of [ + "packages/core/src/client/index.ts", + "packages/toolkit/src/index.ts", + ".github/workflows/ci.yml", + "pnpm-lock.yaml", + "infra/example.tf", + ]) { + assert.equal(directContentEvidence(file), undefined, file); + } + }); +}); + +describe("Content impact analysis", () => { + const changedFiles = [ + "templates/content/docs/product/features/content.feature.test.md", + "templates/content/docs/product/capabilities/content.test.alpha.md", + ]; + + function catalogs() { + return { + base: validateContentProductDocs(baseRoot, { + strictCatalog: false, + checkProjections: false, + }), + head: validateContentProductDocs(headRoot, { + strictCatalog: false, + checkProjections: false, + }), + }; + } + + it("reports deterministic Feature and Capability transitions", () => { + const { base, head } = catalogs(); + const result = collectTransitions(changedFiles, base.catalog, head.catalog); + assert.deepEqual(result.changedRecords, [ + "content.feature.test", + "content.test.alpha", + ]); + assert.deepEqual(result.transitions, [ + { + id: "content.feature.test", + kind: "feature", + from: "planned", + to: "in_validation", + }, + { + id: "content.test.alpha", + kind: "capability", + from: "approved_shape", + to: "in_progress", + }, + ]); + }); + + it("accepts a complete applicable declaration without deterministic findings", () => { + const { base, head } = catalogs(); + const result = analyzeContentProductImpact({ + body: declaration(), + changedFiles, + baseCatalog: base.catalog, + headCatalog: head.catalog, + baseCatalogErrors: base.errors, + headCatalogErrors: head.errors, + }); + assert.equal(result.applicable, true); + assert.deepEqual(result.findings, []); + }); + + it("warns for missing declarations, unknown IDs, and undeclared record changes", () => { + const { base, head } = catalogs(); + const missing = analyzeContentProductImpact({ + body: "", + changedFiles, + baseCatalog: base.catalog, + headCatalog: head.catalog, + }); + assert( + missing.findings.some((item) => item.code === "declaration-missing"), + ); + + const invalid = analyzeContentProductImpact({ + body: declaration().replace( + "content.test.alpha", + "content.missing.capability", + ), + changedFiles, + baseCatalog: base.catalog, + headCatalog: head.catalog, + }); + assert(invalid.findings.some((item) => item.code === "unknown-capability")); + assert( + invalid.findings.some( + (item) => item.code === "changed-record-undeclared", + ), + ); + }); + + it("lets a valid explicit declaration opt a shared-only PR into conformance", () => { + const { base, head } = catalogs(); + const result = analyzeContentProductImpact({ + body: declaration() + .replace(" - content.feature.test\n", " []\n") + .replace("record_change: included", "record_change: none"), + changedFiles: ["packages/core/src/client/index.ts"], + baseCatalog: base.catalog, + headCatalog: head.catalog, + }); + assert.equal(result.applicable, true); + assert.deepEqual(result.findings, []); + }); + + it("keeps a shared-only PR with no direct evidence quiet and non-notifying", () => { + const { base, head } = catalogs(); + const result = analyzeContentProductImpact({ + body: "", + changedFiles: ["packages/core/src/client/index.ts"], + baseCatalog: base.catalog, + headCatalog: head.catalog, + }); + assert.equal(result.applicable, false); + assert.deepEqual(result.findings, []); + }); + + it("keeps malformed and unknown-ID declarations quiet on shared-only PRs", () => { + const { base, head } = catalogs(); + for (const body of [ + "```yaml\ncontent_product_impact: [broken\n```", + declaration() + .replace(" - content.feature.test\n", " []\n") + .replace("content.test.alpha", "content.unknown.contract"), + ]) { + const result = analyzeContentProductImpact({ + body, + changedFiles: ["packages/core/src/client/index.ts"], + baseCatalog: base.catalog, + headCatalog: head.catalog, + }); + assert.equal(result.applicable, false); + assert.deepEqual(result.findings, []); + } + }); +}); + +function git(root: string, args: string[]): string { + return execFileSync("git", args, { cwd: root, encoding: "utf8" }).trim(); +} + +function createCliRepository( + name: string, + change: (root: string) => void, + body: string, + withCatalog = true, + prepareBase?: (root: string) => void, +) { + const root = path.join(temporaryRoot, name); + mkdirSync(root, { recursive: true }); + git(root, ["init", "-q"]); + git(root, ["config", "user.email", "content-conformance@example.invalid"]); + git(root, ["config", "user.name", "Content Conformance Test"]); + if (withCatalog) { + const productRoot = path.join(root, "templates/content/docs/product"); + const skillRoot = path.join( + root, + "templates/content/.agents/skills/content-product-development", + ); + mkdirSync(productRoot, { recursive: true }); + mkdirSync(skillRoot, { recursive: true }); + cpSync(path.join(repositoryRoot, fixture), productRoot, { + recursive: true, + }); + writeFileSync( + path.join(productRoot, "architecture.md"), + "# Architecture\n", + ); + writeFileSync( + path.join(productRoot, "README.md"), + "Read the [architecture](architecture.md).\n", + ); + writeFileSync( + path.join(skillRoot, "SKILL.md"), + "# Content product skill\n", + ); + } else { + writeFileSync(path.join(root, "README.md"), "base\n"); + } + prepareBase?.(root); + git(root, ["add", "."]); + git(root, ["commit", "-qm", "base"]); + const baseSha = git(root, ["rev-parse", "HEAD"]); + change(root); + git(root, ["add", "."]); + git(root, ["commit", "-qm", "head"]); + const headSha = git(root, ["rev-parse", "HEAD"]); + const eventPath = path.join(temporaryRoot, `${name}-event.json`); + writeFileSync( + eventPath, + JSON.stringify({ + pull_request: { + body, + base: { sha: baseSha }, + head: { sha: headSha }, + }, + }), + ); + return { root, eventPath, baseSha, headSha }; +} + +function runCli(input: ReturnType) { + return spawnSync(tsxPath, [checkerPath], { + cwd: input.root, + env: { + ...process.env, + GITHUB_EVENT_PATH: input.eventPath, + CONTENT_IMPACT_BASE_SHA: input.baseSha, + CONTENT_IMPACT_HEAD_SHA: input.headSha, + }, + encoding: "utf8", + }); +} + +describe("Content impact CLI boundary", () => { + it("returns success for advisory findings and emits a copyable warning", () => { + const repository = createCliRepository( + "direct-cli", + (root) => { + const file = path.join(root, "templates/content/app/example.ts"); + mkdirSync(path.dirname(file), { recursive: true }); + writeFileSync(file, "export const example = true;\n"); + }, + "", + ); + const result = runCli(repository); + assert.equal(result.status, 0, result.stderr); + assert.match(result.stdout, /"advisory":true/); + assert.match(result.stdout, /::warning title=Content product impact/); + assert.match(result.stdout, /content_product_impact/); + }); + + it("does not notify for a malformed declaration on a shared-only change", () => { + const repository = createCliRepository( + "shared-cli", + (root) => { + const file = path.join(root, "packages/core/src/example.ts"); + mkdirSync(path.dirname(file), { recursive: true }); + writeFileSync(file, "export const example = true;\n"); + }, + "```yaml\ncontent_product_impact: [broken\n```", + ); + const result = runCli(repository); + assert.equal(result.status, 0, result.stderr); + assert.match(result.stdout, /Not applicable/); + assert.doesNotMatch(result.stdout, /::warning/); + }); + + it("validates the base and head developer skills from their own revisions", () => { + const repository = createCliRepository( + "head-skill-cli", + (root) => { + writeFileSync( + path.join( + root, + "templates/content/.agents/skills/content-product-development/SKILL.md", + ), + "# Content product skill\n\nDo not use [live checkout files](../../../../../../../package.json).\n", + ); + }, + "", + ); + const result = runCli(repository); + assert.equal(result.status, 0, result.stderr); + assert.match(result.stdout, /head-catalog-invalid/); + assert.match(result.stdout, /link escapes the repository/); + assert.doesNotMatch(result.stderr, /base Content product catalog/); + }); + + it("materializes linked non-Markdown files inside each snapshot", () => { + const repository = createCliRepository( + "linked-asset-cli", + (root) => { + const file = path.join(root, "templates/content/app/example.ts"); + mkdirSync(path.dirname(file), { recursive: true }); + writeFileSync(file, "export const example = true;\n"); + }, + "", + true, + (root) => { + const productRoot = path.join(root, "templates/content/docs/product"); + writeFileSync(path.join(productRoot, "contract.json"), "{}\n"); + writeFileSync( + path.join(productRoot, "README.md"), + "Read the [architecture](architecture.md) and [contract](contract.json).\n", + ); + }, + ); + const result = runCli(repository); + assert.equal(result.status, 0, result.stderr); + assert.doesNotMatch(result.stdout, /broken relative link/); + }); + + it("uses the merge-base for a PR whose target branch has advanced", () => { + const repository = createCliRepository( + "diverged-base-cli", + (root) => writeFileSync(path.join(root, "README.md"), "head\n"), + declaration().replace("record_change: included", "record_change: none"), + ); + git(repository.root, ["checkout", "-q", "--detach", repository.baseSha]); + writeFileSync( + path.join( + repository.root, + "templates/content/.agents/skills/content-product-development/SKILL.md", + ), + "# Content product skill\n\nDo not use file:///private/example.\n", + ); + git(repository.root, ["add", "."]); + git(repository.root, ["commit", "-qm", "advanced base"]); + const advancedBaseSha = git(repository.root, ["rev-parse", "HEAD"]); + writeFileSync( + repository.eventPath, + JSON.stringify({ + pull_request: { + body: declaration().replace( + "record_change: included", + "record_change: none", + ), + base: { sha: advancedBaseSha }, + head: { sha: repository.headSha }, + }, + }), + ); + + const result = runCli({ ...repository, baseSha: advancedBaseSha }); + assert.equal(result.status, 0, result.stderr); + assert.doesNotMatch(result.stderr, /base Content product catalog/); + assert.match( + result.stdout, + new RegExp(`"comparisonBaseSha":"${repository.baseSha}"`), + ); + }); + + it("fails loudly for mismatched event revisions and missing catalogs", () => { + const mismatch = createCliRepository( + "mismatch-cli", + (root) => writeFileSync(path.join(root, "README.md"), "head\n"), + "", + false, + ); + const mismatchEvent = JSON.parse(readFileSync(mismatch.eventPath, "utf8")); + mismatchEvent.pull_request.base.sha = mismatch.headSha; + writeFileSync(mismatch.eventPath, JSON.stringify(mismatchEvent)); + const mismatchResult = runCli(mismatch); + assert.notEqual(mismatchResult.status, 0); + assert.match(mismatchResult.stderr, /do not match/); + + const missingCatalog = createCliRepository( + "missing-catalog-cli", + (root) => writeFileSync(path.join(root, "README.md"), "head\n"), + "", + false, + ); + const missingResult = runCli(missingCatalog); + assert.notEqual(missingResult.status, 0); + assert.match(missingResult.stderr, /readable Content product records/); + + const nestedCatalog = createCliRepository( + "nested-catalog-cli", + (root) => { + const productRoot = path.join(root, "templates/content/docs/product"); + for (const directory of ["chapters", "features", "capabilities"]) { + rmSync(path.join(productRoot, directory), { + recursive: true, + force: true, + }); + } + const archive = path.join(productRoot, "archive/capabilities"); + mkdirSync(archive, { recursive: true }); + writeFileSync(path.join(archive, "note.md"), "# Not a record\n"); + }, + "", + ); + const nestedResult = runCli(nestedCatalog); + assert.notEqual(nestedResult.status, 0); + assert.match(nestedResult.stderr, /readable Content product records/); + + const invalidBase = createCliRepository( + "invalid-base-cli", + (root) => writeFileSync(path.join(root, "README.md"), "head\n"), + "", + true, + (root) => { + writeFileSync(path.join(root, "README.md"), "base\n"); + const feature = path.join( + root, + "templates/content/docs/product/features/content.feature.test.md", + ); + writeFileSync( + feature, + readFileSync(feature, "utf8").replace("---", ""), + ); + }, + ); + const invalidResult = runCli(invalidBase); + assert.notEqual(invalidResult.status, 0); + assert.match(invalidResult.stderr, /base Content product catalog/); + }); +}); diff --git a/scripts/validate-content-product-impact.ts b/scripts/validate-content-product-impact.ts new file mode 100644 index 0000000000..eaa610f79b --- /dev/null +++ b/scripts/validate-content-product-impact.ts @@ -0,0 +1,744 @@ +#!/usr/bin/env node +import { execFileSync } from "node:child_process"; +import { + appendFileSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { parse as parseYaml } from "yaml"; + +import { + type ProductCatalog, + type ProductRecord, + validateContentProductDocs, +} from "./validate-content-product-docs.ts"; + +const PRODUCT_ROOT = "templates/content/docs/product"; +const CONTENT_PRODUCT_SKILL_ROOT = + "templates/content/.agents/skills/content-product-development"; +const CHECKER_ROOT = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + "..", +); +const SOURCE_REPOSITORY_ROOT = path.resolve( + process.cwd(), + process.env.CONTENT_IMPACT_REPOSITORY ?? ".", +); +const LANES = [ + "contract_repair", + "contract_fulfillment", + "local_refinement", + "product_decision_candidate", +] as const; +const RECORD_CHANGES = ["none", "included", "decision_pending"] as const; +const DECLARATION_KEYS = new Set([ + "lane", + "features", + "capabilities", + "record_change", + "proof", + "rationale", +]); +const DECLARATION_REPAIR = `\`\`\`yaml +content_product_impact: + lane: contract_repair + features: + - content.feature.see-your-information-your-way + capabilities: + - content.view.renderer-conformance + record_change: none + proof: + - focused test or acceptance artifact + rationale: Explain the exact Content impact and record obligation. +\`\`\``; + +export type ContentImpactLane = (typeof LANES)[number]; +export type RecordChange = (typeof RECORD_CHANGES)[number]; + +export type ContentImpactDeclaration = { + lane: ContentImpactLane; + features: string[]; + capabilities: string[]; + record_change: RecordChange; + proof: string[]; + rationale: string; +}; + +export type DeclarationResult = + | { status: "missing" } + | { status: "malformed"; errors: string[] } + | { status: "duplicate"; errors: string[] } + | { status: "valid"; declaration: ContentImpactDeclaration }; + +export type ProductTransition = { + id: string; + kind: "feature" | "capability"; + from: string | null; + to: string | null; +}; + +export type ImpactFinding = { + code: string; + message: string; +}; + +export type ImpactAnalysis = { + applicable: boolean; + applicabilityReasons: string[]; + declaration: DeclarationResult; + changedRecords: string[]; + transitions: ProductTransition[]; + findings: ImpactFinding[]; +}; + +export type ImpactAnalysisInput = { + body: string; + changedFiles: readonly string[]; + baseCatalog: ProductCatalog; + headCatalog: ProductCatalog; + baseCatalogErrors?: readonly string[]; + headCatalogErrors?: readonly string[]; +}; + +type PullRequestEvent = { + pull_request?: { + body?: string | null; + base?: { sha?: string }; + head?: { sha?: string }; + }; +}; + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function stringArray( + value: unknown, + field: string, + errors: string[], +): string[] { + if (!Array.isArray(value) || value.some((item) => typeof item !== "string")) { + errors.push(`${field} must be an array of strings`); + return []; + } + return value.map((item) => item.trim()); +} + +function parseDeclarationValue( + value: unknown, +): + | { declaration: ContentImpactDeclaration; errors: [] } + | { declaration?: undefined; errors: string[] } { + const errors: string[] = []; + if (!isRecord(value) || !isRecord(value.content_product_impact)) { + return { + errors: ["content_product_impact must be a YAML mapping"], + }; + } + const topLevelKeys = Object.keys(value); + if ( + topLevelKeys.length !== 1 || + topLevelKeys[0] !== "content_product_impact" + ) { + errors.push("declaration block must contain only content_product_impact"); + } + const impact = value.content_product_impact; + for (const key of Object.keys(impact)) { + if (!DECLARATION_KEYS.has(key)) errors.push(`unknown field: ${key}`); + } + + const lane = + typeof impact.lane === "string" ? impact.lane.trim() : impact.lane; + if (typeof lane !== "string" || !LANES.includes(lane as ContentImpactLane)) { + errors.push(`lane must be one of: ${LANES.join(", ")}`); + } + const recordChange = + typeof impact.record_change === "string" + ? impact.record_change.trim() + : impact.record_change; + if ( + typeof recordChange !== "string" || + !RECORD_CHANGES.includes(recordChange as RecordChange) + ) { + errors.push(`record_change must be one of: ${RECORD_CHANGES.join(", ")}`); + } + const features = stringArray(impact.features, "features", errors); + const capabilities = stringArray(impact.capabilities, "capabilities", errors); + const proof = stringArray(impact.proof, "proof", errors); + if (proof.length === 0) errors.push("proof must contain at least one entry"); + if (proof.some((item) => item.trim().length === 0)) { + errors.push("proof entries must not be empty"); + } + const rationale = + typeof impact.rationale === "string" + ? impact.rationale.trim() + : impact.rationale; + if (typeof rationale !== "string" || rationale.trim().length === 0) { + errors.push("rationale must be a non-empty string"); + } + if ( + typeof lane === "string" && + lane !== "product_decision_candidate" && + features.length + capabilities.length === 0 + ) { + errors.push(`${lane} requires at least one Feature or Capability ID`); + } + if ( + lane === "product_decision_candidate" && + features.length + capabilities.length === 0 && + recordChange !== "decision_pending" && + recordChange !== "included" + ) { + errors.push( + "a product_decision_candidate without IDs requires record_change decision_pending or included", + ); + } + for (const [field, ids] of [ + ["features", features], + ["capabilities", capabilities], + ] as const) { + if (new Set(ids).size !== ids.length) { + errors.push(`${field} must not contain duplicate IDs`); + } + } + if (errors.length > 0) return { errors }; + + return { + errors: [], + declaration: { + lane: lane as ContentImpactLane, + features, + capabilities, + record_change: recordChange as RecordChange, + proof, + rationale: rationale as string, + }, + }; +} + +export function parseContentImpactDeclaration(body: string): DeclarationResult { + const candidates = [ + ...body.matchAll(/```(?:yaml|yml)\s*\r?\n([\s\S]*?)```/gi), + ] + .map((match) => match[1]) + .filter((source) => /^\s*content_product_impact\s*:/m.test(source)); + + if (candidates.length === 0) return { status: "missing" }; + if (candidates.length > 1) { + return { + status: "duplicate", + errors: ["PR body must contain exactly one content_product_impact block"], + }; + } + + let parsed: unknown; + try { + parsed = parseYaml(candidates[0], { uniqueKeys: true }); + } catch (error) { + return { + status: "malformed", + errors: [ + `declaration YAML could not be parsed: ${error instanceof Error ? error.message : String(error)}`, + ], + }; + } + const result = parseDeclarationValue(parsed); + return result.declaration + ? { status: "valid", declaration: result.declaration } + : { status: "malformed", errors: result.errors }; +} + +const CONTENT_ROOT_EXCLUSIONS = new Set([ + "templates/content/.dockerignore", + "templates/content/.gitignore", + "templates/content/.ignore", + "templates/content/.oxfmtrc.json", + "templates/content/CHANGELOG.md", + "templates/content/DEVELOPING.md", + "templates/content/README.md", + "templates/content/_gitignore", +]); + +export function directContentEvidence(file: string): string | undefined { + const normalized = file.replaceAll("\\", "/"); + const prefixes = [ + "templates/content/actions/", + "templates/content/app/", + "templates/content/data/", + "templates/content/docs/product/", + "templates/content/drizzle/", + "templates/content/e2e/", + "templates/content/parity/", + "templates/content/public/", + "templates/content/scripts/", + "templates/content/server/", + "templates/content/shared/", + "templates/content/.agents/skills/content-product-development/", + "scripts/fixtures/content-product-docs/", + "scripts/fixtures/content-product-impact/", + ]; + if (prefixes.some((prefix) => normalized.startsWith(prefix))) { + return normalized; + } + if ( + /^templates\/content\/[^/]+$/.test(normalized) && + !CONTENT_ROOT_EXCLUSIONS.has(normalized) + ) { + return normalized; + } + if ( + normalized.startsWith("templates/content/") && + /(?:content.*parity|parity.*content|content.*conformance|conformance.*content)/i.test( + normalized, + ) + ) { + return normalized; + } + return undefined; +} + +function stateOf(record: ProductRecord): string | null { + const value = + record.kind === "feature" ? record.data.roadmap_status : record.data.state; + return typeof value === "string" ? value : null; +} + +export function collectTransitions( + changedFiles: readonly string[], + baseCatalog: ProductCatalog, + headCatalog: ProductCatalog, +): { changedRecords: string[]; transitions: ProductTransition[] } { + const recordIds = new Set(); + const productRecordPath = new RegExp( + `^${PRODUCT_ROOT}/(?:features|capabilities)/(content\\.[a-z0-9.-]+)\\.md$`, + ); + for (const file of changedFiles) { + const match = file.match(productRecordPath); + if (match) recordIds.add(match[1]); + } + + const baseById = new Map( + baseCatalog.records.map((record) => [record.id, record]), + ); + const headById = new Map( + headCatalog.records.map((record) => [record.id, record]), + ); + const transitions: ProductTransition[] = []; + for (const id of [...recordIds].sort()) { + const base = baseById.get(id); + const head = headById.get(id); + const record = head ?? base; + if (!record || record.kind === "chapter") continue; + const from = base ? stateOf(base) : null; + const to = head ? stateOf(head) : null; + if (from !== to) transitions.push({ id, kind: record.kind, from, to }); + } + return { changedRecords: [...recordIds].sort(), transitions }; +} + +function finding(code: string, message: string): ImpactFinding { + return { code, message }; +} + +export function analyzeContentProductImpact( + input: ImpactAnalysisInput, +): ImpactAnalysis { + const declaration = parseContentImpactDeclaration(input.body); + const direct = input.changedFiles + .map(directContentEvidence) + .filter((file): file is string => file !== undefined); + const headIntroducedCatalogFailure = + (input.baseCatalogErrors?.length ?? 0) === 0 && + (input.headCatalogErrors?.length ?? 0) > 0; + const knownFeatureIds = new Set( + input.headCatalog.features.map((record) => record.id), + ); + const knownCapabilityIds = new Set( + input.headCatalog.capabilities.map((record) => record.id), + ); + const declarationNamesKnownId = + declaration.status === "valid" && + (declaration.declaration.features.some((id) => knownFeatureIds.has(id)) || + declaration.declaration.capabilities.some((id) => + knownCapabilityIds.has(id), + )); + const applicabilityReasons = [ + ...direct.map((file) => `direct:${file}`), + ...(declarationNamesKnownId ? ["declaration"] : []), + ...(headIntroducedCatalogFailure ? ["content-contract-failure"] : []), + ]; + const applicable = applicabilityReasons.length > 0; + const { changedRecords, transitions } = collectTransitions( + input.changedFiles, + input.baseCatalog, + input.headCatalog, + ); + const findings: ImpactFinding[] = []; + + if (!applicable) { + return { + applicable, + applicabilityReasons, + declaration, + changedRecords, + transitions, + findings, + }; + } + + if (declaration.status === "missing") { + findings.push( + finding( + "declaration-missing", + `Copy this declaration into the PR body and replace its values with the exact impact:\n${DECLARATION_REPAIR}`, + ), + ); + } else if (declaration.status !== "valid") { + for (const error of declaration.errors) { + findings.push(finding(`declaration-${declaration.status}`, error)); + } + } else { + const value = declaration.declaration; + const headFeatures = new Set( + input.headCatalog.features.map((record) => record.id), + ); + const headCapabilities = new Set( + input.headCatalog.capabilities.map((record) => record.id), + ); + const deletedFeatures = new Set( + input.baseCatalog.records + .filter( + (record) => + record.kind === "feature" && + changedRecords.includes(record.id) && + !input.headCatalog.records.some((head) => head.id === record.id), + ) + .map((record) => record.id), + ); + const deletedCapabilities = new Set( + input.baseCatalog.records + .filter( + (record) => + record.kind === "capability" && + changedRecords.includes(record.id) && + !input.headCatalog.records.some((head) => head.id === record.id), + ) + .map((record) => record.id), + ); + + for (const id of value.features) { + if (!headFeatures.has(id) && !deletedFeatures.has(id)) { + findings.push(finding("unknown-feature", `Unknown Feature ID: ${id}`)); + } + } + for (const id of value.capabilities) { + if (!headCapabilities.has(id) && !deletedCapabilities.has(id)) { + findings.push( + finding("unknown-capability", `Unknown Capability ID: ${id}`), + ); + } + } + + const declaredIds = new Set([...value.features, ...value.capabilities]); + const allowsUnnamedNewRecords = + value.lane === "product_decision_candidate" && + value.record_change === "included" && + declaredIds.size === 0; + for (const id of changedRecords) { + const recordExistedAtBase = input.baseCatalog.records.some( + (record) => record.id === id, + ); + if ( + !declaredIds.has(id) && + !(allowsUnnamedNewRecords && !recordExistedAtBase) + ) { + findings.push( + finding( + "changed-record-undeclared", + `Changed product record ${id} is not named in the declaration.`, + ), + ); + } + } + if (changedRecords.length > 0 && value.record_change === "none") { + findings.push( + finding( + "record-change-mismatch", + "record_change cannot be none when Feature or Capability records changed.", + ), + ); + } + if (changedRecords.length === 0 && value.record_change === "included") { + findings.push( + finding( + "record-change-mismatch", + "record_change is included, but no Feature or Capability record changed.", + ), + ); + } + if ( + value.record_change === "decision_pending" && + value.lane !== "product_decision_candidate" + ) { + findings.push( + finding( + "decision-lane-mismatch", + "record_change decision_pending requires lane product_decision_candidate.", + ), + ); + } + if ( + value.record_change === "decision_pending" && + transitions.some( + (transition) => + transition.to === "verified" || transition.to === "available", + ) + ) { + findings.push( + finding( + "decision-promotion-mismatch", + "record_change decision_pending cannot claim a verified Capability or available Feature promotion.", + ), + ); + } + } + + for (const error of input.headCatalogErrors ?? []) { + findings.push(finding("head-catalog-invalid", error)); + } + + return { + applicable, + applicabilityReasons, + declaration, + changedRecords, + transitions, + findings, + }; +} + +function git(args: string[]): string { + return execFileSync("git", args, { + cwd: SOURCE_REPOSITORY_ROOT, + encoding: "utf8", + maxBuffer: 32 * 1024 * 1024, + }); +} + +function gitFile(sha: string, file: string): Buffer { + return execFileSync("git", ["show", `${sha}:${file}`], { + cwd: SOURCE_REPOSITORY_ROOT, + maxBuffer: 32 * 1024 * 1024, + }); +} + +function assertSha(value: string, name: string): void { + if (!/^[0-9a-f]{40,64}$/i.test(value)) { + throw new Error(`${name} must be a full Git object SHA`); + } + git(["cat-file", "-e", `${value}^{commit}`]); +} + +function materializeContentProductSnapshot( + sha: string, + destination: string, +): string { + const root = path.join(destination, PRODUCT_ROOT); + for (const directory of ["chapters", "features", "capabilities"]) { + mkdirSync(path.join(root, directory), { recursive: true }); + } + const output = git([ + "ls-tree", + "-r", + "--name-only", + "-z", + sha, + "--", + PRODUCT_ROOT, + CONTENT_PRODUCT_SKILL_ROOT, + ]); + const files = output.split("\0").filter(Boolean); + const productRecordFiles = files.filter( + (file) => + file.endsWith(".md") && + new RegExp( + `^${PRODUCT_ROOT}/(?:chapters|features|capabilities)/[^/]+\\.md$`, + ).test(file), + ); + if (productRecordFiles.length === 0) { + throw new Error(`${sha} does not contain readable Content product records`); + } + for (const file of files) { + const destinationFile = path.join(destination, file); + mkdirSync(path.dirname(destinationFile), { recursive: true }); + writeFileSync(destinationFile, gitFile(sha, file)); + } + return root; +} + +function changedFiles(baseSha: string, headSha: string): string[] { + return git([ + "diff", + "--name-only", + "-z", + "--no-renames", + "--diff-filter=ACDMRTUXB", + baseSha, + headSha, + ]) + .split("\0") + .filter(Boolean); +} + +function annotationEscape(value: string): string { + return value + .replaceAll("%", "%25") + .replaceAll("\r", "%0D") + .replaceAll("\n", "%0A"); +} + +export function renderImpactSummary(analysis: ImpactAnalysis): string { + if (!analysis.applicable) { + return [ + "## Content product conformance", + "", + "Not applicable. No direct Content evidence or explicit Content impact declaration was found.", + ].join("\n"); + } + const lines = [ + "## Content product conformance — advisory pilot", + "", + `Applicability: ${analysis.applicabilityReasons.join(", ")}`, + `Declaration: ${analysis.declaration.status}`, + `Changed Feature/Capability records: ${analysis.changedRecords.join(", ") || "none"}`, + "", + ]; + if (analysis.transitions.length > 0) { + lines.push("| record | transition |", "| --- | --- |"); + for (const transition of analysis.transitions) { + lines.push( + `| ${transition.id} | ${transition.from ?? "absent"} -> ${transition.to ?? "absent"} |`, + ); + } + lines.push(""); + } + if (analysis.findings.length === 0) { + lines.push("No deterministic findings. This pilot remains advisory."); + } else { + lines.push("### Advisory findings", ""); + for (const item of analysis.findings) { + lines.push(`- **${item.code}:** ${item.message}`); + } + lines.push( + "", + "These findings do not block the pull request during calibration.", + ); + } + return lines.join("\n"); +} + +export function runContentProductImpactCheck(): void { + const eventPath = process.env.GITHUB_EVENT_PATH; + const baseSha = process.env.CONTENT_IMPACT_BASE_SHA; + const headSha = process.env.CONTENT_IMPACT_HEAD_SHA; + if (!eventPath || !baseSha || !headSha) { + throw new Error( + "GITHUB_EVENT_PATH, CONTENT_IMPACT_BASE_SHA, and CONTENT_IMPACT_HEAD_SHA are required", + ); + } + assertSha(baseSha, "CONTENT_IMPACT_BASE_SHA"); + assertSha(headSha, "CONTENT_IMPACT_HEAD_SHA"); + + const event = JSON.parse(readFileSync(eventPath, "utf8")) as PullRequestEvent; + const pullRequest = event.pull_request; + if (!pullRequest) throw new Error("event does not contain pull_request data"); + if (pullRequest.base?.sha !== baseSha || pullRequest.head?.sha !== headSha) { + throw new Error( + "event base/head SHAs do not match the requested revisions", + ); + } + const comparisonBaseSha = git(["merge-base", baseSha, headSha]).trim(); + if (!/^[0-9a-f]{40,64}$/i.test(comparisonBaseSha)) { + throw new Error("could not resolve a full merge-base for the PR revisions"); + } + + const temporaryRoot = mkdtempSync( + path.join(CHECKER_ROOT, ".content-impact-"), + ); + try { + const baseSnapshotRoot = path.join(temporaryRoot, "base"); + const headSnapshotRoot = path.join(temporaryRoot, "head"); + const baseRoot = materializeContentProductSnapshot( + comparisonBaseSha, + baseSnapshotRoot, + ); + const headRoot = materializeContentProductSnapshot( + headSha, + headSnapshotRoot, + ); + const base = validateContentProductDocs(baseRoot, { + strictCatalog: false, + checkProjections: false, + validationRoot: baseSnapshotRoot, + }); + const head = validateContentProductDocs(headRoot, { + strictCatalog: false, + checkProjections: false, + validationRoot: headSnapshotRoot, + }); + if (base.errors.length > 0) { + throw new Error( + `base Content product catalog is unreadable or invalid:\n${base.errors.join("\n")}`, + ); + } + const analysis = analyzeContentProductImpact({ + body: pullRequest.body ?? "", + changedFiles: changedFiles(comparisonBaseSha, headSha), + baseCatalog: base.catalog, + headCatalog: head.catalog, + baseCatalogErrors: base.errors, + headCatalogErrors: head.errors, + }); + const summary = renderImpactSummary(analysis); + process.stdout.write( + `${JSON.stringify({ + schema: "content-conformance.advisory-declaration.v1", + headSha, + comparisonBaseSha, + advisory: true, + ...analysis, + })}\n`, + ); + if (process.env.GITHUB_STEP_SUMMARY) { + appendFileSync(process.env.GITHUB_STEP_SUMMARY, `${summary}\n`); + } + process.stdout.write(`${summary}\n`); + if (analysis.applicable) { + for (const item of analysis.findings) { + process.stdout.write( + `::warning title=Content product impact (${annotationEscape(item.code)})::${annotationEscape(item.message)}\n`, + ); + } + } + } finally { + rmSync(temporaryRoot, { recursive: true, force: true }); + } +} + +const invokedPath = process.argv[1] ? path.resolve(process.argv[1]) : ""; +if (invokedPath === fileURLToPath(import.meta.url)) { + try { + runContentProductImpactCheck(); + } catch (error) { + console.error( + `Content product impact check could not establish its inputs: ${error instanceof Error ? error.message : String(error)}`, + ); + process.exitCode = 1; + } +} diff --git a/templates/content/.agents/skills/content-product-development/SKILL.md b/templates/content/.agents/skills/content-product-development/SKILL.md index 2f52a2277e..e5687b4e60 100644 --- a/templates/content/.agents/skills/content-product-development/SKILL.md +++ b/templates/content/.agents/skills/content-product-development/SKILL.md @@ -72,6 +72,53 @@ require a permission slip for every useful bug fix. Load the domain skill for the work, especially `actions`, `security`, `sharing`, `storing-data`, `portability`, `real-time-collab`, or `real-time-sync`. +## Declare pull-request impact + +During the advisory conformance pilot, add exactly one fenced YAML declaration +to the pull-request body when a change directly affects Content or when a shared +framework change has specific Content impact: + +```yaml +content_product_impact: + lane: contract_repair + features: + - content.feature.example + capabilities: + - content.example.capability + record_change: none + proof: + - pnpm --filter content test + rationale: The change repairs the declared Capability without changing its contract. +``` + +Use stable IDs from `templates/content/docs/product/`. Valid lanes are +`contract_repair`, `contract_fulfillment`, `local_refinement`, and +`product_decision_candidate`. Set `record_change` to `included` when the PR +changes a Feature or Capability record, or `decision_pending` when the product +record must wait for an explicit product decision. + +The lane meanings are the same as the classification table above. A contract +repair does not require roadmap churn when the accepted record remains true. +Repairs, fulfillment, and local refinement must name at least one real Feature +or Capability. A product-decision candidate may omit IDs only with +`decision_pending`, or while the same PR includes the proposed new record; do +not invent an ID to make the declaration look complete. + +Run the focused declaration and workflow-policy tests with: + +```sh +pnpm test:content-product-impact +``` + +To repair a warning, copy the block above into the PR body, replace the example +IDs with exact catalog IDs, align `record_change` with any changed product +records, and name the focused proof you actually ran. + +The check is intentionally advisory while its deterministic rules calibrate. +It never edits the roadmap, assigns or tags a reviewer, and never turns an LLM +opinion into a merge gate. Shared framework changes with no direct Content +evidence should remain quiet. + ## Prove and record the result Use the affected Capability's proof requirements and the Feature's example