Validate Content product impact in pull requests

.github/workflows/content-product-conformance.yml

@@ -0,0 +1,68 @@
+name: Content product conformance
+
+on:
+  pull_request:
+    branches: [main]
+    types:
+      [
+        opened,
+        synchronize,
+        reopened,
+        edited,
+        ready_for_review,
+        labeled,
+        unlabeled,
+      ]
+
+permissions:
+  contents: read
+  pull-requests: read
+
+concurrency:
+  group: content-product-conformance-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+jobs:
+  check:
+    name: Advisory Content product impact
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - name: Check out immutable conformance controller
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+        with:
+          ref: 03caa13fd5bf6176ee01ab223452db9932b7ca8c
+          path: controller
+          fetch-depth: 1
+          persist-credentials: false
+
+      - name: Check out candidate as inert data
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+        with:
+          repository: ${{ github.event.pull_request.head.repo.full_name }}
+          ref: ${{ github.event.pull_request.head.sha }}
+          path: candidate
+          fetch-depth: 0
+          persist-credentials: false
+
+      - name: Fetch exact target revision
+        run: git -C candidate fetch --no-tags https://github.com/BuilderIO/agent-native.git ${{ github.event.pull_request.base.sha }}
+
+      - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4.4.0
+        with:
+          package_json_file: controller/package.json
+
+      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
+        with:
+          node-version: "22"
+          cache: pnpm
+          cache-dependency-path: controller/pnpm-lock.yaml
+
+      - run: pnpm --dir controller install --frozen-lockfile --ignore-scripts
+
+      - name: Check Content product impact
+        env:
+          CONTENT_IMPACT_REPOSITORY: ../candidate
+          CONTENT_IMPACT_BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          CONTENT_IMPACT_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+        run: pnpm --dir controller exec tsx scripts/validate-content-product-impact.ts

package.json

@@ -61,8 +61,11 @@
     "guard:template-list": "node scripts/guard-template-list.mjs",
     "guard:netlify-private-env": "tsx scripts/guard-netlify-private-env.ts",
     "guard:trusted-acceptance": "tsx scripts/guard-trusted-acceptance-workflow.ts",
+    "guard:content-product-conformance": "tsx scripts/validate-content-product-impact-workflow.ts",
     "guard:content-product-docs": "tsx scripts/validate-content-product-docs.ts",
     "test:content-product-docs": "tsx --test scripts/validate-content-product-docs.test.ts",
+    "content-product-impact": "tsx scripts/validate-content-product-impact.ts",
+    "test:content-product-impact": "tsx --test scripts/validate-content-product-impact.test.ts scripts/validate-content-product-impact-workflow.test.ts",
     "guard:workspace-skills": "tsx scripts/sync-workspace-core-skills.ts --check",
     "sync:template-standard": "tsx scripts/template-standard/index.ts",
     "guard:template-standard": "tsx scripts/template-standard/index.ts --check",

scripts/run-guards.ts

@@ -13,6 +13,7 @@ const guards = [
   "guard:template-list",
   "guard:netlify-private-env",
   "guard:trusted-acceptance",
+  "guard:content-product-conformance",
   "guard:content-product-docs",
   "guard:workspace-skills",
   "guard:template-standard",

scripts/validate-content-product-docs.ts

@@ -50,17 +50,17 @@ const roadmapBoundaries = new Set([
   "superseded",
 ]);
 
-type RecordKind = "chapter" | "feature" | "capability";
+export type RecordKind = "chapter" | "feature" | "capability";
 
-type ProductRecord = {
+export type ProductRecord = {
   file: string;
   body: string;
   data: Record<string, unknown>;
   kind: RecordKind;
   id: string;
 };
 
-type ProductCatalog = {
+export type ProductCatalog = {
   root: string;
   chapters: ProductRecord[];
   features: ProductRecord[];
@@ -77,15 +77,23 @@ export type ValidationResult = {
 
 export function validateContentProductDocs(
   root = defaultProductRoot,
-  options: { strictCatalog?: boolean; checkProjections?: boolean } = {},
+  options: {
+    strictCatalog?: boolean;
+    checkProjections?: boolean;
+    validationRoot?: string;
+  } = {},
 ): ValidationResult {
   const strictCatalog = options.strictCatalog ?? root === defaultProductRoot;
   const checkProjections = options.checkProjections ?? true;
   const errors: string[] = [];
   const catalog = loadCatalog(root, errors);
 
   validateCatalog(catalog, errors, strictCatalog);
-  validateLinksAndPrivacy(catalog, errors);
+  validateLinksAndPrivacy(
+    catalog,
+    errors,
+    options.validationRoot ?? repositoryRoot,
+  );
 
   const roadmap = renderRoadmap(catalog);
   const encyclopedia = renderEncyclopedia(catalog);
@@ -803,11 +811,15 @@ function validateCapabilityMiniSpec(
   }
 }
 
-function validateLinksAndPrivacy(catalog: ProductCatalog, errors: string[]) {
+function validateLinksAndPrivacy(
+  catalog: ProductCatalog,
+  errors: string[],
+  validationRoot: string,
+) {
   const files = collectMarkdownFiles(catalog.root);
   const skillRoot = join(
-    repositoryRoot,
-    "templates/content/.agents/skills/content-product-development",
+    resolve(catalog.root, "../.."),
+    ".agents/skills/content-product-development",
   );
   if (existsSync(skillRoot)) files.push(...collectMarkdownFiles(skillRoot));
 
@@ -859,7 +871,7 @@ function validateLinksAndPrivacy(catalog: ProductCatalog, errors: string[]) {
       }
       const withoutAnchor = target.split("#")[0];
       const resolved = resolve(dirname(file), withoutAnchor);
-      if (!isInside(repositoryRoot, resolved)) {
+      if (!isInside(validationRoot, resolved)) {
         errors.push(
           `${displayPath(file)}:${lineNumber(source, match.index ?? 0)}: link escapes the repository: ${target}`,
         );

scripts/validate-content-product-impact-workflow.test.ts

@@ -0,0 +1,180 @@
+import assert from "node:assert/strict";
+import { readFileSync } from "node:fs";
+import { describe, it } from "node:test";
+
+import { validateContentProductImpactWorkflow } from "./validate-content-product-impact-workflow.ts";
+
+const workflow = readFileSync(
+  ".github/workflows/content-product-conformance.yml",
+  "utf8",
+);
+
+describe("Content product conformance workflow boundary", () => {
+  it("keeps the pilot read-only, exact-revision, and broadly observable", () => {
+    assert.deepEqual(validateContentProductImpactWorkflow(workflow), {
+      ok: true,
+      issues: [],
+    });
+  });
+
+  it("rejects pull_request_target and write permissions", () => {
+    const unsafe = workflow
+      .replace("  pull_request:", "  pull_request_target:")
+      .replace("contents: read", "contents: write");
+    const result = validateContentProductImpactWorkflow(unsafe);
+    assert.equal(result.ok, false);
+    assert(
+      result.issues.some((issue) => issue.includes("pull_request_target")),
+    );
+    assert(result.issues.some((issue) => issue.includes("permissions")));
+  });
+
+  it("rejects path filtering or a candidate checkout with credentials", () => {
+    const unsafe = workflow
+      .replace("    types:", '    paths: ["templates/content/**"]\n    types:')
+      .replace("persist-credentials: false", "persist-credentials: true");
+    const result = validateContentProductImpactWorkflow(unsafe);
+    assert.equal(result.ok, false);
+    assert(result.issues.some((issue) => issue.includes("path filters")));
+    assert(result.issues.some((issue) => issue.includes("credentials")));
+  });
+
+  it("requires declaration and label edits to rerun the pilot", () => {
+    const unsafe = workflow.replace("        edited,\n", "");
+    const result = validateContentProductImpactWorkflow(unsafe);
+    assert.equal(result.ok, false);
+    assert(result.issues.some((issue) => issue.includes("recalibration")));
+  });
+
+  it("does not hide checker infrastructure failures", () => {
+    for (const value of ["true", "${{ true }}"]) {
+      const unsafe = workflow.replace(
+        "        run: pnpm --dir controller exec tsx scripts/validate-content-product-impact.ts",
+        `        continue-on-error: ${value}\n        run: pnpm --dir controller exec tsx scripts/validate-content-product-impact.ts`,
+      );
+      const result = validateContentProductImpactWorkflow(unsafe);
+      assert.equal(result.ok, false);
+      assert(result.issues.some((issue) => issue.includes("infrastructure")));
+    }
+  });
+
+  it("rejects dot and bracket references to the secrets context", () => {
+    for (const expression of [
+      "${{ secrets.DEPLOY_KEY }}",
+      "${{ secrets['DEPLOY_KEY'] }}",
+      "${{ toJSON(secrets) }}",
+    ]) {
+      const unsafe = workflow.replace(
+        "    timeout-minutes:",
+        `    env:\n      LEAK: "${expression}"\n    timeout-minutes:`,
+      );
+      const result = validateContentProductImpactWorkflow(unsafe);
+      assert.equal(result.ok, false);
+      assert(result.issues.some((issue) => issue.includes("secrets")));
+    }
+  });
+
+  it("rejects job-level permission overrides and explicit GitHub tokens", () => {
+    const unsafe = workflow
+      .replace(
+        "    name: Advisory Content product impact",
+        "    name: Advisory Content product impact\n    permissions:\n      contents: write",
+      )
+      .replace(
+        "    timeout-minutes:",
+        `    env:\n      GH_TOKEN: "\${{ github['token'] }}"\n    timeout-minutes:`,
+      );
+    const result = validateContentProductImpactWorkflow(unsafe);
+    assert.equal(result.ok, false);
+    assert(result.issues.some((issue) => issue.includes("permissions")));
+    assert(result.issues.some((issue) => issue.includes("credentials")));
+  });
+
+  it("rejects serialization of the full GitHub context", () => {
+    for (const expression of [
+      "${{ toJSON(github) }}",
+      "${{ TOJSON( GITHUB ) }}",
+    ]) {
+      const unsafe = workflow.replace(
+        "    timeout-minutes:",
+        `    env:\n      GITHUB_CONTEXT: "${expression}"\n    timeout-minutes:`,
+      );
+      const result = validateContentProductImpactWorkflow(unsafe);
+      assert.equal(result.ok, false);
+      assert(result.issues.some((issue) => issue.includes("credentials")));
+    }
+  });
+
+  it("requires the check job and impact checker step to be unconditional", () => {
+    const unsafe = workflow
+      .replace(
+        "    name: Advisory Content product impact",
+        "    name: Advisory Content product impact\n    if: false",
+      )
+      .replace(
+        "      - name: Check Content product impact",
+        "      - name: Check Content product impact\n        if: false",
+      );
+    const result = validateContentProductImpactWorkflow(unsafe);
+    assert.equal(result.ok, false);
+    assert.equal(
+      result.issues.filter((issue) => issue.includes("unconditionally")).length,
+      2,
+    );
+  });
+
+  it("rejects extra jobs and additional checkout steps", () => {
+    const unsafe = workflow
+      .replace(
+        "jobs:\n",
+        "jobs:\n  leak:\n    runs-on: ubuntu-latest\n    environment: production\n    steps: []\n",
+      )
+      .replace(
+        "      - uses: pnpm/action-setup@",
+        "      - uses: actions/checkout@unsafe\n\n      - uses: pnpm/action-setup@",
+      );
+    const result = validateContentProductImpactWorkflow(unsafe);
+    assert.equal(result.ok, false);
+    assert(result.issues.some((issue) => issue.includes("only")));
+    assert(result.issues.some((issue) => issue.includes("two checkout")));
+  });
+
+  it("rejects self-hosted runners and arbitrary additional steps", () => {
+    const unsafe = workflow
+      .replace("runs-on: ubuntu-latest", "runs-on: self-hosted")
+      .replace(
+        "      - uses: pnpm/action-setup@",
+        "      - uses: arbitrary/action@unsafe\n\n      - uses: pnpm/action-setup@",
+      );
+    const result = validateContentProductImpactWorkflow(unsafe);
+    assert.equal(result.ok, false);
+    assert(result.issues.some((issue) => issue.includes("GitHub-hosted")));
+    assert(result.issues.some((issue) => issue.includes("exactly match")));
+  });
+
+  it("rejects a candidate-controlled controller or package script", () => {
+    const unsafe = workflow
+      .replace(
+        "ref: 03caa13fd5bf6176ee01ab223452db9932b7ca8c",
+        "ref: ${{ github.event.pull_request.head.sha }}",
+      )
+      .replace(
+        "pnpm --dir controller exec tsx scripts/validate-content-product-impact.ts",
+        "pnpm --dir candidate content-product-impact",
+      );
+    const result = validateContentProductImpactWorkflow(unsafe);
+    assert.equal(result.ok, false);
+    assert(result.issues.some((issue) => issue.includes("trusted revision")));
+    assert(result.issues.some((issue) => issue.includes("standalone")));
+  });
+
+  it("requires the exact upstream base revision for fork pull requests", () => {
+    const unsafe = workflow.replace(
+      "git -C candidate fetch --no-tags https://github.com/BuilderIO/agent-native.git ${{ github.event.pull_request.base.sha }}",
+      "git -C candidate fetch origin main",
+    );
+    const result = validateContentProductImpactWorkflow(unsafe);
+    assert.equal(result.ok, false);
+    assert(result.issues.some((issue) => issue.includes("target revision")));
+  });
+});