From a58b0abd787c11354f1e86c0366433075b431d24 Mon Sep 17 00:00:00 2001 From: milosjovanovic Date: Thu, 18 Jun 2026 11:24:59 +0200 Subject: [PATCH 1/9] Add support for detecting and managing unmanaged repositories --- .github/actions/compare/action.yaml | 20 ++++- .github/actions/drift-pr/action.yaml | 87 +++++++++++++++++++ .github/workflows/detect-unmanaged-repos.yaml | 82 +++++++++++++++++ 3 files changed, 186 insertions(+), 3 deletions(-) create mode 100644 .github/actions/drift-pr/action.yaml create mode 100644 .github/workflows/detect-unmanaged-repos.yaml diff --git a/.github/actions/compare/action.yaml b/.github/actions/compare/action.yaml index 96adfdd..d1019bc 100644 --- a/.github/actions/compare/action.yaml +++ b/.github/actions/compare/action.yaml @@ -10,6 +10,10 @@ inputs: target-directory: description: "Target directory to compare" required: true + keep: + description: "Which configs to keep in the source dir: 'changed' (new + modified, default) or 'new-only' (only repos that have no existing config)" + required: false + default: "changed" runs: using: "composite" steps: @@ -19,16 +23,26 @@ runs: env: SOURCE_DIRECTORY: ${{ inputs.source-directory }} TARGET_DIRECTORY: ${{ inputs.target-directory }} + KEEP: ${{ inputs.keep }} run: | set -euo pipefail dirA="${SOURCE_DIRECTORY}" dirB="${TARGET_DIRECTORY}" result=$(just compare "$dirA" "$dirB") - files_to_delete=$(echo "$result" | jq -r '.identical[]?') + + # Always drop unchanged (identical) configs. In 'new-only' mode also drop + # modified (different) configs, leaving only repos that have no config yet. + case "${KEEP}" in + new-only) jq_filter='(.identical[]?, .different[]?)' ;; + changed) jq_filter='.identical[]?' ;; + *) echo "Invalid 'keep' value: ${KEEP} (expected 'changed' or 'new-only')" >&2; exit 1 ;; + esac + files_to_delete=$(echo "$result" | jq -r "$jq_filter") + if [[ -z "$files_to_delete" ]]; then - echo "No identical (unchanged) files to remove." + echo "No files to remove (keep mode: ${KEEP})." else - echo "Removing unchanged configs from $dirA so the PR only shows real changes:" + echo "Removing configs from $dirA (keep mode: ${KEEP}) so the PR only shows the intended changes:" while IFS= read -r relpath; do [[ -z "$relpath" ]] && continue rm "$dirA/$relpath" diff --git a/.github/actions/drift-pr/action.yaml b/.github/actions/drift-pr/action.yaml new file mode 100644 index 0000000..dbdabd3 --- /dev/null +++ b/.github/actions/drift-pr/action.yaml @@ -0,0 +1,87 @@ +name: "Drift PR for unmanaged repositories" +description: "Opens, updates, or closes a single PR listing repos that exist in the org but are not yet managed by GCSS" +inputs: + branch-name: + description: "Stable branch used for the drift PR" + required: false + default: "drift/unmanaged-repos" + reviewers: + description: "Comma-separated reviewers (users or org/team slugs) to request on the PR" + required: false + default: "" + github-token: + description: "Token used to push and manage the PR" + required: true +runs: + using: "composite" + steps: + - name: Configure git + shell: bash + run: | + git config --global user.name 'github-actions[bot]' + git config --global user.email 'github-actions[bot]@users.noreply.github.com' + + - name: Open, update or close the drift PR + shell: bash + working-directory: feature/github-repo-provisioning/gcss_config + env: + GH_TOKEN: ${{ inputs.github-token }} + BRANCH: ${{ inputs.branch-name }} + REVIEWERS: ${{ inputs.reviewers }} + BASE: ${{ github.ref_name }} + run: | + set -euo pipefail + SRC="importer_tmp_dir" + + # How many unmanaged repo configs did the compare step leave behind? + detected=$(find "$SRC" -maxdepth 1 -type f \( -name '*.yaml' -o -name '.*.yaml' \) 2>/dev/null | wc -l | tr -d ' ') + existing_pr=$(gh pr list --head "$BRANCH" --state open --json number --jq '.[0].number // empty') + + # No drift: close any open drift PR (and its branch) so it doesn't go stale. + if [[ "$detected" -eq 0 ]]; then + if [[ -n "$existing_pr" ]]; then + gh pr close "$existing_pr" --delete-branch \ + --comment "No unmanaged repositories detected — drift resolved. Closing automatically." + echo "Closed drift PR #$existing_pr (no unmanaged repositories remain)." + else + echo "No unmanaged repositories detected; nothing to do." + fi + exit 0 + fi + + echo "Detected $detected unmanaged repository config(s)." + + # Rebuild the drift branch from the base each run so the PR only ever + # reflects the current set of unmanaged repos (untracked importer_tmp_dir + # files are preserved across the checkout). + git fetch origin "$BASE" + git checkout -B "$BRANCH" "origin/$BASE" + git add "$SRC" + git commit -m "Detected unmanaged repositories (GCSS-1132)" + + # If an identical drift branch already exists remotely, don't re-push + # (avoids re-notifying reviewers every scheduled run). + if git ls-remote --exit-code --heads origin "$BRANCH" >/dev/null 2>&1; then + git fetch origin "$BRANCH" + if git diff --quiet "origin/$BRANCH" HEAD; then + echo "Drift PR already up to date; nothing to push." + exit 0 + fi + git push --force-with-lease origin "$BRANCH" + else + git push origin "$BRANCH" + fi + + if [[ -z "$existing_pr" ]]; then + pr_url=$(gh pr create \ + --title "Unmanaged repositories detected" \ + --body "These repositories exist in the organisation but are not yet managed by GCSS. Merge this PR to bring them under management, or add them to \`ignored_repos\` in \`config/import-config.yaml\` to acknowledge them." \ + --base "$BASE" \ + --head "$BRANCH") + echo "Opened new drift PR: $pr_url" + if [[ -n "${REVIEWERS}" ]]; then + gh pr edit "$pr_url" --add-reviewer "${REVIEWERS}" || echo "Warning: could not request reviewers '${REVIEWERS}'" + fi + else + echo "Updated existing drift PR #$existing_pr." + fi diff --git a/.github/workflows/detect-unmanaged-repos.yaml b/.github/workflows/detect-unmanaged-repos.yaml new file mode 100644 index 0000000..8eb7c32 --- /dev/null +++ b/.github/workflows/detect-unmanaged-repos.yaml @@ -0,0 +1,82 @@ +name: Detect unmanaged repositories + +# Detects repositories that exist in the organisation but are not yet managed by +# GCSS (e.g. created manually in the GitHub UI) and surfaces them in a single, +# continuously-updated PR. Intended to be called on a schedule. See GCSS-1132. + +on: + workflow_call: + inputs: + gcss_ref: + type: string + description: "GCSS ref to checkout" + required: false + default: "main" + reviewers: + type: string + description: "Comma-separated reviewers (users or org/team slugs) to request on the drift PR" + required: false + default: "" + secrets: + app_private_key: + required: true + +jobs: + detect-unmanaged-repos: + runs-on: ubuntu-latest + name: Detect unmanaged repositories + # Uses the 'import' environment so it inherits APP_ID for the management app, + # which must have org-wide ("All repositories") access or new repos are + # invisible to the importer (see GCSS-1132). + environment: import + permissions: + contents: write + pull-requests: write + steps: + - name: Generate a token + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + id: generate-token + with: + app-id: ${{ vars.APP_ID }} + private-key: ${{ secrets.app_private_key }} + owner: ${{ github.repository_owner }} + + - name: Checkout GCSS + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + repository: G-Research/github-terraformer + ref: ${{ inputs.gcss_ref }} + persist-credentials: false + + - name: GCSS config setup + uses: ./.github/actions/gcss-config-setup + with: + checkout-sha: ${{ github.sha }} + checkout-token: ${{ steps.generate-token.outputs.token }} + + - name: Setup Just + uses: extractions/setup-just@f8a3cce218d9f83db3a2ecd90e41ac3de6cdfd9b # v3.1.0 + with: + just-version: '1.4.0' + + - name: Import repos + working-directory: feature/github-repo-importer + run: just import-repos + env: + GITHUB_TOKEN: ${{ steps.generate-token.outputs.token }} + OWNER: ${{ github.repository_owner }} + + - name: Filter to unmanaged repos only + uses: ./.github/actions/compare + with: + working-directory: feature/github-repo-importer + source-directory: "../github-repo-provisioning/gcss_config/importer_tmp_dir/" + target-directory: "../github-repo-provisioning/gcss_config/repos/" + keep: "new-only" + + - name: Open or update the drift PR + uses: ./.github/actions/drift-pr + with: + branch-name: "drift/unmanaged-repos" + reviewers: ${{ inputs.reviewers }} + github-token: ${{ steps.generate-token.outputs.token }} From e410e0a39ccf5d171eb73bb25c16c330a8483965 Mon Sep 17 00:00:00 2001 From: milosjovanovic Date: Thu, 18 Jun 2026 15:51:52 +0200 Subject: [PATCH 2/9] Improve drift detection logic and add environment configurability --- .github/actions/drift-pr/action.yaml | 22 +++++++++++++---- .github/workflows/detect-unmanaged-repos.yaml | 24 +++++++++++++++---- 2 files changed, 37 insertions(+), 9 deletions(-) diff --git a/.github/actions/drift-pr/action.yaml b/.github/actions/drift-pr/action.yaml index dbdabd3..19a7b90 100644 --- a/.github/actions/drift-pr/action.yaml +++ b/.github/actions/drift-pr/action.yaml @@ -57,21 +57,33 @@ runs: git fetch origin "$BASE" git checkout -B "$BRANCH" "origin/$BASE" git add "$SRC" + + # Nothing staged means the detected configs are already present on the + # base branch (e.g. a drift PR was just merged and is awaiting + # promotion); there is nothing to surface, so exit cleanly instead of + # failing on an empty commit under `set -e`. + if git diff --cached --quiet; then + echo "Detected configs already present on $BASE (pending promotion); nothing to surface." + exit 0 + fi git commit -m "Detected unmanaged repositories (GCSS-1132)" - # If an identical drift branch already exists remotely, don't re-push - # (avoids re-notifying reviewers every scheduled run). + # Push only when the remote branch isn't already identical (avoids + # re-notifying reviewers every run), but always continue so that a PR is + # (re)opened below if one isn't already present — e.g. when a previous + # drift PR was closed without deleting the branch. if git ls-remote --exit-code --heads origin "$BRANCH" >/dev/null 2>&1; then git fetch origin "$BRANCH" if git diff --quiet "origin/$BRANCH" HEAD; then - echo "Drift PR already up to date; nothing to push." - exit 0 + echo "Remote drift branch already up to date; skipping push." + else + git push --force-with-lease origin "$BRANCH" fi - git push --force-with-lease origin "$BRANCH" else git push origin "$BRANCH" fi + # Ensure exactly one open drift PR exists. if [[ -z "$existing_pr" ]]; then pr_url=$(gh pr create \ --title "Unmanaged repositories detected" \ diff --git a/.github/workflows/detect-unmanaged-repos.yaml b/.github/workflows/detect-unmanaged-repos.yaml index 8eb7c32..fd4d0e4 100644 --- a/.github/workflows/detect-unmanaged-repos.yaml +++ b/.github/workflows/detect-unmanaged-repos.yaml @@ -17,18 +17,34 @@ on: description: "Comma-separated reviewers (users or org/team slugs) to request on the drift PR" required: false default: "" + environment: + type: string + description: >- + Deployment environment that provides APP_ID for the management app. + Because this runs unattended on a schedule, the chosen environment + MUST NOT have required-reviewer or wait-timer protection rules, or + every run will stall pending manual approval. Defaults to 'schedule'. + required: false + default: "schedule" secrets: app_private_key: required: true +# A full org import can run longer than the schedule interval; prevent +# overlapping runs from racing on the shared drift branch / PR. +concurrency: + group: detect-unmanaged-repos + cancel-in-progress: false + jobs: detect-unmanaged-repos: runs-on: ubuntu-latest name: Detect unmanaged repositories - # Uses the 'import' environment so it inherits APP_ID for the management app, - # which must have org-wide ("All repositories") access or new repos are - # invisible to the importer (see GCSS-1132). - environment: import + # Environment supplies APP_ID for the management app, which must have + # org-wide ("All repositories") access or new repos are invisible to the + # importer (see GCSS-1132). Must be free of approval/protection rules — see + # the 'environment' input description. + environment: ${{ inputs.environment }} permissions: contents: write pull-requests: write From df2d0633a55d09873d4863f09d74ce2c7e90fb51 Mon Sep 17 00:00:00 2001 From: milosjovanovic Date: Fri, 19 Jun 2026 09:17:59 +0200 Subject: [PATCH 3/9] Refactor workflows to consolidate unmanaged repo detection logic within drift-check workflow --- .github/workflows/detect-unmanaged-repos.yaml | 98 ------------------- .github/workflows/drift-check.yaml | 90 ++++++++++++++++- 2 files changed, 89 insertions(+), 99 deletions(-) delete mode 100644 .github/workflows/detect-unmanaged-repos.yaml diff --git a/.github/workflows/detect-unmanaged-repos.yaml b/.github/workflows/detect-unmanaged-repos.yaml deleted file mode 100644 index fd4d0e4..0000000 --- a/.github/workflows/detect-unmanaged-repos.yaml +++ /dev/null @@ -1,98 +0,0 @@ -name: Detect unmanaged repositories - -# Detects repositories that exist in the organisation but are not yet managed by -# GCSS (e.g. created manually in the GitHub UI) and surfaces them in a single, -# continuously-updated PR. Intended to be called on a schedule. See GCSS-1132. - -on: - workflow_call: - inputs: - gcss_ref: - type: string - description: "GCSS ref to checkout" - required: false - default: "main" - reviewers: - type: string - description: "Comma-separated reviewers (users or org/team slugs) to request on the drift PR" - required: false - default: "" - environment: - type: string - description: >- - Deployment environment that provides APP_ID for the management app. - Because this runs unattended on a schedule, the chosen environment - MUST NOT have required-reviewer or wait-timer protection rules, or - every run will stall pending manual approval. Defaults to 'schedule'. - required: false - default: "schedule" - secrets: - app_private_key: - required: true - -# A full org import can run longer than the schedule interval; prevent -# overlapping runs from racing on the shared drift branch / PR. -concurrency: - group: detect-unmanaged-repos - cancel-in-progress: false - -jobs: - detect-unmanaged-repos: - runs-on: ubuntu-latest - name: Detect unmanaged repositories - # Environment supplies APP_ID for the management app, which must have - # org-wide ("All repositories") access or new repos are invisible to the - # importer (see GCSS-1132). Must be free of approval/protection rules — see - # the 'environment' input description. - environment: ${{ inputs.environment }} - permissions: - contents: write - pull-requests: write - steps: - - name: Generate a token - uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 - id: generate-token - with: - app-id: ${{ vars.APP_ID }} - private-key: ${{ secrets.app_private_key }} - owner: ${{ github.repository_owner }} - - - name: Checkout GCSS - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 - with: - repository: G-Research/github-terraformer - ref: ${{ inputs.gcss_ref }} - persist-credentials: false - - - name: GCSS config setup - uses: ./.github/actions/gcss-config-setup - with: - checkout-sha: ${{ github.sha }} - checkout-token: ${{ steps.generate-token.outputs.token }} - - - name: Setup Just - uses: extractions/setup-just@f8a3cce218d9f83db3a2ecd90e41ac3de6cdfd9b # v3.1.0 - with: - just-version: '1.4.0' - - - name: Import repos - working-directory: feature/github-repo-importer - run: just import-repos - env: - GITHUB_TOKEN: ${{ steps.generate-token.outputs.token }} - OWNER: ${{ github.repository_owner }} - - - name: Filter to unmanaged repos only - uses: ./.github/actions/compare - with: - working-directory: feature/github-repo-importer - source-directory: "../github-repo-provisioning/gcss_config/importer_tmp_dir/" - target-directory: "../github-repo-provisioning/gcss_config/repos/" - keep: "new-only" - - - name: Open or update the drift PR - uses: ./.github/actions/drift-pr - with: - branch-name: "drift/unmanaged-repos" - reviewers: ${{ inputs.reviewers }} - github-token: ${{ steps.generate-token.outputs.token }} diff --git a/.github/workflows/drift-check.yaml b/.github/workflows/drift-check.yaml index 51b9af3..2465f3c 100644 --- a/.github/workflows/drift-check.yaml +++ b/.github/workflows/drift-check.yaml @@ -12,12 +12,31 @@ on: type: string description: "Terraform Cloud organization" required: true + reviewers: + type: string + description: "Comma-separated reviewers (users or org/team slugs) to request on the unmanaged-repos drift PR" + required: false + default: "" + detect_environment: + type: string + description: >- + Deployment environment that provides APP_ID for the management app, + used by the unmanaged-repos detection job. Because this runs unattended + on a schedule, the chosen environment MUST NOT have required-reviewer + or wait-timer protection rules, or every run will stall pending manual + approval. Defaults to 'schedule'. + required: false + default: "schedule" secrets: tfc_token: description: "Terraform Cloud API token" required: true + app_private_key: + description: "GitHub App private key for the management app (used by the unmanaged-repos detection job)" + required: true jobs: + # Detects config drift on repositories already managed by Terraform. drift-check: name: "Drift Check" runs-on: ubuntu-latest @@ -53,4 +72,73 @@ jobs: exit 1 else echo "No drift detected." - fi \ No newline at end of file + fi + + # Detects repositories that exist in the organisation but are not yet managed + # by GCSS (e.g. created manually in the GitHub UI) — invisible to `terraform + # plan` above — and surfaces them in a single, continuously-updated PR. + # See GCSS-1132. Runs independently of the terraform-plan drift job. + detect-unmanaged-repos: + name: "Detect unmanaged repositories" + runs-on: ubuntu-latest + # Environment supplies APP_ID for the management app, which must have + # org-wide ("All repositories") access or new repos are invisible to the + # importer (see GCSS-1132). Must be free of approval/protection rules — see + # the 'detect_environment' input description. + environment: ${{ inputs.detect_environment }} + permissions: + contents: write + pull-requests: write + # A full org import can run longer than the schedule interval; prevent + # overlapping runs from racing on the shared drift branch / PR. + concurrency: + group: detect-unmanaged-repos + cancel-in-progress: false + steps: + - name: Generate a token + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + id: generate-token + with: + app-id: ${{ vars.APP_ID }} + private-key: ${{ secrets.app_private_key }} + owner: ${{ github.repository_owner }} + + - name: Checkout GCSS + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + repository: G-Research/github-terraformer + ref: ${{ inputs.gcss_ref }} + persist-credentials: false + + - name: GCSS config setup + uses: ./.github/actions/gcss-config-setup + with: + checkout-sha: ${{ github.sha }} + checkout-token: ${{ steps.generate-token.outputs.token }} + + - name: Setup Just + uses: extractions/setup-just@f8a3cce218d9f83db3a2ecd90e41ac3de6cdfd9b # v3.1.0 + with: + just-version: '1.4.0' + + - name: Import repos + working-directory: feature/github-repo-importer + run: just import-repos + env: + GITHUB_TOKEN: ${{ steps.generate-token.outputs.token }} + OWNER: ${{ github.repository_owner }} + + - name: Filter to unmanaged repos only + uses: ./.github/actions/compare + with: + working-directory: feature/github-repo-importer + source-directory: "../github-repo-provisioning/gcss_config/importer_tmp_dir/" + target-directory: "../github-repo-provisioning/gcss_config/repos/" + keep: "new-only" + + - name: Open or update the drift PR + uses: ./.github/actions/drift-pr + with: + branch-name: "drift/unmanaged-repos" + reviewers: ${{ inputs.reviewers }} + github-token: ${{ steps.generate-token.outputs.token }} From cea7b68f89035ee8a66d0eb1a5888a74298b0333 Mon Sep 17 00:00:00 2001 From: milosjovanovic Date: Fri, 19 Jun 2026 10:54:34 +0200 Subject: [PATCH 4/9] Remove `keep` input and associated logic from compare action; refactor unmanaged repo handling in drift-check workflow --- .github/actions/compare/action.yaml | 20 +++---------------- .github/workflows/drift-check.yaml | 30 ++++++++++++++++++++--------- 2 files changed, 24 insertions(+), 26 deletions(-) diff --git a/.github/actions/compare/action.yaml b/.github/actions/compare/action.yaml index d1019bc..96adfdd 100644 --- a/.github/actions/compare/action.yaml +++ b/.github/actions/compare/action.yaml @@ -10,10 +10,6 @@ inputs: target-directory: description: "Target directory to compare" required: true - keep: - description: "Which configs to keep in the source dir: 'changed' (new + modified, default) or 'new-only' (only repos that have no existing config)" - required: false - default: "changed" runs: using: "composite" steps: @@ -23,26 +19,16 @@ runs: env: SOURCE_DIRECTORY: ${{ inputs.source-directory }} TARGET_DIRECTORY: ${{ inputs.target-directory }} - KEEP: ${{ inputs.keep }} run: | set -euo pipefail dirA="${SOURCE_DIRECTORY}" dirB="${TARGET_DIRECTORY}" result=$(just compare "$dirA" "$dirB") - - # Always drop unchanged (identical) configs. In 'new-only' mode also drop - # modified (different) configs, leaving only repos that have no config yet. - case "${KEEP}" in - new-only) jq_filter='(.identical[]?, .different[]?)' ;; - changed) jq_filter='.identical[]?' ;; - *) echo "Invalid 'keep' value: ${KEEP} (expected 'changed' or 'new-only')" >&2; exit 1 ;; - esac - files_to_delete=$(echo "$result" | jq -r "$jq_filter") - + files_to_delete=$(echo "$result" | jq -r '.identical[]?') if [[ -z "$files_to_delete" ]]; then - echo "No files to remove (keep mode: ${KEEP})." + echo "No identical (unchanged) files to remove." else - echo "Removing configs from $dirA (keep mode: ${KEEP}) so the PR only shows the intended changes:" + echo "Removing unchanged configs from $dirA so the PR only shows real changes:" while IFS= read -r relpath; do [[ -z "$relpath" ]] && continue rm "$dirA/$relpath" diff --git a/.github/workflows/drift-check.yaml b/.github/workflows/drift-check.yaml index 2465f3c..293ec2e 100644 --- a/.github/workflows/drift-check.yaml +++ b/.github/workflows/drift-check.yaml @@ -121,21 +121,33 @@ jobs: with: just-version: '1.4.0' - - name: Import repos + - name: Exclude already-managed repositories from the import + shell: bash + env: + OWNER: ${{ github.repository_owner }} + run: | + set -euo pipefail + config="feature/github-repo-importer/import-config.yaml" + repos_dir="feature/github-repo-provisioning/gcss_config/repos" + # The importer skips repos listed in ignored_repos, so treating every + # already-configured repo as ignored makes it import ONLY unmanaged + # repositories — avoiding a full-org import on every run (GCSS-1132). + if [[ -d "$repos_dir" ]]; then + while IFS= read -r f; do + name="$(basename "$f" .yaml)" + yq -i ".ignored_repos = ((.ignored_repos // []) + [\"${OWNER}/${name}\"])" "$config" + done < <(find "$repos_dir" -maxdepth 1 -type f -name '*.yaml') + fi + echo "Effective ignored_repos after excluding managed repositories:" + yq '.ignored_repos' "$config" + + - name: Import unmanaged repos working-directory: feature/github-repo-importer run: just import-repos env: GITHUB_TOKEN: ${{ steps.generate-token.outputs.token }} OWNER: ${{ github.repository_owner }} - - name: Filter to unmanaged repos only - uses: ./.github/actions/compare - with: - working-directory: feature/github-repo-importer - source-directory: "../github-repo-provisioning/gcss_config/importer_tmp_dir/" - target-directory: "../github-repo-provisioning/gcss_config/repos/" - keep: "new-only" - - name: Open or update the drift PR uses: ./.github/actions/drift-pr with: From 5d53aada15c34d02786f4d1ee335edc13ca55503 Mon Sep 17 00:00:00 2001 From: milosjovanovic Date: Fri, 19 Jun 2026 11:17:52 +0200 Subject: [PATCH 5/9] Update `import-repos` command to support dynamic config paths based on owner --- feature/github-repo-importer/Justfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/feature/github-repo-importer/Justfile b/feature/github-repo-importer/Justfile index fd2313b..4462fd2 100644 --- a/feature/github-repo-importer/Justfile +++ b/feature/github-repo-importer/Justfile @@ -8,7 +8,7 @@ import-repo repoName: import-repos: go run main.go bulk-import -c import-config.yaml - mkdir -p "../../feature/github-repo-provisioning/gcss_config/importer_tmp_dir/" + mkdir -p "configs/$OWNER" "../../feature/github-repo-provisioning/gcss_config/importer_tmp_dir/" find configs/"$OWNER" -maxdepth 1 -type f \( -name "*.yaml" -o -name ".*.yaml" \) -print -exec cp {} "../../feature/github-repo-provisioning/gcss_config/importer_tmp_dir/" \; test: From 206233e6829ffd22c26d83d830b8497652299dba Mon Sep 17 00:00:00 2001 From: milosjovanovic Date: Fri, 19 Jun 2026 12:52:55 +0200 Subject: [PATCH 6/9] Refactor drift-check workflow to consolidate unmanaged repo detection and improve drift handling logic --- .github/actions/drift-pr/action.yaml | 26 ++--- .github/workflows/drift-check.yaml | 139 ++++++++++----------------- 2 files changed, 66 insertions(+), 99 deletions(-) diff --git a/.github/actions/drift-pr/action.yaml b/.github/actions/drift-pr/action.yaml index 19a7b90..c8de59d 100644 --- a/.github/actions/drift-pr/action.yaml +++ b/.github/actions/drift-pr/action.yaml @@ -1,10 +1,10 @@ -name: "Drift PR for unmanaged repositories" -description: "Opens, updates, or closes a single PR listing repos that exist in the org but are not yet managed by GCSS" +name: "Drift PR" +description: "Opens, updates, or closes a single PR with repos whose GitHub state has drifted from the managed config (new repos or manual edits)" inputs: branch-name: description: "Stable branch used for the drift PR" required: false - default: "drift/unmanaged-repos" + default: "drift/detected-changes" reviewers: description: "Comma-separated reviewers (users or org/team slugs) to request on the PR" required: false @@ -33,7 +33,7 @@ runs: set -euo pipefail SRC="importer_tmp_dir" - # How many unmanaged repo configs did the compare step leave behind? + # How many changed repo configs did the compare step leave behind? detected=$(find "$SRC" -maxdepth 1 -type f \( -name '*.yaml' -o -name '.*.yaml' \) 2>/dev/null | wc -l | tr -d ' ') existing_pr=$(gh pr list --head "$BRANCH" --state open --json number --jq '.[0].number // empty') @@ -41,19 +41,19 @@ runs: if [[ "$detected" -eq 0 ]]; then if [[ -n "$existing_pr" ]]; then gh pr close "$existing_pr" --delete-branch \ - --comment "No unmanaged repositories detected — drift resolved. Closing automatically." - echo "Closed drift PR #$existing_pr (no unmanaged repositories remain)." + --comment "No drift detected — resolved. Closing automatically." + echo "Closed drift PR #$existing_pr (no drift remains)." else - echo "No unmanaged repositories detected; nothing to do." + echo "No drift detected; nothing to do." fi exit 0 fi - echo "Detected $detected unmanaged repository config(s)." + echo "Detected $detected changed repository config(s)." # Rebuild the drift branch from the base each run so the PR only ever - # reflects the current set of unmanaged repos (untracked importer_tmp_dir - # files are preserved across the checkout). + # reflects the current set of changes (untracked importer_tmp_dir files + # are preserved across the checkout). git fetch origin "$BASE" git checkout -B "$BRANCH" "origin/$BASE" git add "$SRC" @@ -66,7 +66,7 @@ runs: echo "Detected configs already present on $BASE (pending promotion); nothing to surface." exit 0 fi - git commit -m "Detected unmanaged repositories (GCSS-1132)" + git commit -m "Detected configuration drift (GCSS-1132)" # Push only when the remote branch isn't already identical (avoids # re-notifying reviewers every run), but always continue so that a PR is @@ -86,8 +86,8 @@ runs: # Ensure exactly one open drift PR exists. if [[ -z "$existing_pr" ]]; then pr_url=$(gh pr create \ - --title "Unmanaged repositories detected" \ - --body "These repositories exist in the organisation but are not yet managed by GCSS. Merge this PR to bring them under management, or add them to \`ignored_repos\` in \`config/import-config.yaml\` to acknowledge them." \ + --title "Configuration drift detected" \ + --body "These repositories have drifted from their managed configuration — either created outside GCSS (not yet managed) or manually edited outside Terraform. The YAML reflects the current GitHub state. Merge this PR to adopt the change into config, or revert the change manually if it was unintended." \ --base "$BASE" \ --head "$BRANCH") echo "Opened new drift PR: $pr_url" diff --git a/.github/workflows/drift-check.yaml b/.github/workflows/drift-check.yaml index 293ec2e..12264fc 100644 --- a/.github/workflows/drift-check.yaml +++ b/.github/workflows/drift-check.yaml @@ -14,86 +14,39 @@ on: required: true reviewers: type: string - description: "Comma-separated reviewers (users or org/team slugs) to request on the unmanaged-repos drift PR" + description: "Comma-separated reviewers (users or org/team slugs) to request on the drift PR" required: false default: "" - detect_environment: - type: string - description: >- - Deployment environment that provides APP_ID for the management app, - used by the unmanaged-repos detection job. Because this runs unattended - on a schedule, the chosen environment MUST NOT have required-reviewer - or wait-timer protection rules, or every run will stall pending manual - approval. Defaults to 'schedule'. - required: false - default: "schedule" secrets: tfc_token: description: "Terraform Cloud API token" required: true app_private_key: - description: "GitHub App private key for the management app (used by the unmanaged-repos detection job)" + description: "GitHub App private key for the management app (used to import repos and raise the drift PR)" required: true +# A full org import + plan can run longer than the schedule interval; prevent +# overlapping runs from racing on the shared drift branch / PR. +concurrency: + group: drift-check + cancel-in-progress: false + jobs: - # Detects config drift on repositories already managed by Terraform. + # Detects drift between the desired config and the actual GitHub state — both + # manual edits to already-managed repos AND repos created outside of GCSS — + # and surfaces everything in a single, continuously-updated PR. See GCSS-1132. drift-check: name: "Drift Check" runs-on: ubuntu-latest + # The 'schedule' environment must provide APP_ID + app_private_key (management + # app, which needs org-wide "All repositories" access so new repos are visible + # to the importer), plus WORKSPACE for the plan. Because this runs unattended + # on a schedule, the environment MUST NOT have required-reviewer/wait-timer + # protection rules, or every run will stall pending manual approval. environment: schedule - steps: - - name: Checkout GCSS - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 - with: - repository: G-Research/github-terraformer - ref: ${{ inputs.gcss_ref }} - persist-credentials: false - - - name: GCSS config setup - uses: ./.github/actions/gcss-config-setup - with: - checkout-sha: ${{ github.sha }} - checkout-token: ${{ github.token }} - - - name: Setup terraform and run plan - id: graformer - uses: ./.github/actions/graformer - with: - tfc-token: ${{ secrets.tfc_token }} - tfc-organization: ${{ inputs.tfc_org }} - tfc-workspace: ${{ vars.WORKSPACE }} - refresh: 'true' - - # 0 for green plan no changes, 2 for green plan with changes - - name: Inspect drift - run: | - if [[ "${{ steps.graformer.outputs.plan-exitcode }}" == 2 ]]; then - echo "Drift detected!" - exit 1 - else - echo "No drift detected." - fi - - # Detects repositories that exist in the organisation but are not yet managed - # by GCSS (e.g. created manually in the GitHub UI) — invisible to `terraform - # plan` above — and surfaces them in a single, continuously-updated PR. - # See GCSS-1132. Runs independently of the terraform-plan drift job. - detect-unmanaged-repos: - name: "Detect unmanaged repositories" - runs-on: ubuntu-latest - # Environment supplies APP_ID for the management app, which must have - # org-wide ("All repositories") access or new repos are invisible to the - # importer (see GCSS-1132). Must be free of approval/protection rules — see - # the 'detect_environment' input description. - environment: ${{ inputs.detect_environment }} permissions: contents: write pull-requests: write - # A full org import can run longer than the schedule interval; prevent - # overlapping runs from racing on the shared drift branch / PR. - concurrency: - group: detect-unmanaged-repos - cancel-in-progress: false steps: - name: Generate a token uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 @@ -121,36 +74,50 @@ jobs: with: just-version: '1.4.0' - - name: Exclude already-managed repositories from the import - shell: bash - env: - OWNER: ${{ github.repository_owner }} - run: | - set -euo pipefail - config="feature/github-repo-importer/import-config.yaml" - repos_dir="feature/github-repo-provisioning/gcss_config/repos" - # The importer skips repos listed in ignored_repos, so treating every - # already-configured repo as ignored makes it import ONLY unmanaged - # repositories — avoiding a full-org import on every run (GCSS-1132). - if [[ -d "$repos_dir" ]]; then - while IFS= read -r f; do - name="$(basename "$f" .yaml)" - yq -i ".ignored_repos = ((.ignored_repos // []) + [\"${OWNER}/${name}\"])" "$config" - done < <(find "$repos_dir" -maxdepth 1 -type f -name '*.yaml') - fi - echo "Effective ignored_repos after excluding managed repositories:" - yq '.ignored_repos' "$config" - - - name: Import unmanaged repos + # Import the current GitHub state of every repo. New repos (absent from + # Terraform state) get YAML generated for them; managed repos get their + # current state captured for comparison. + - name: Import repos working-directory: feature/github-repo-importer run: just import-repos env: GITHUB_TOKEN: ${{ steps.generate-token.outputs.token }} OWNER: ${{ github.repository_owner }} - - name: Open or update the drift PR + # Drop everything that matches the committed config, leaving only repos + # that actually changed — new repos and manual edits to managed ones. + - name: Filter to changed configs + uses: ./.github/actions/compare + with: + working-directory: feature/github-repo-importer + source-directory: "../github-repo-provisioning/gcss_config/importer_tmp_dir/" + target-directory: "../github-repo-provisioning/gcss_config/repos/" + + # Run the plan over the changed objects (new repos are imported via the + # module's import blocks, managed-repo drift is caught by -refresh). + - name: Setup terraform and run plan + id: graformer + uses: ./.github/actions/graformer + with: + tfc-token: ${{ secrets.tfc_token }} + tfc-organization: ${{ inputs.tfc_org }} + tfc-workspace: ${{ vars.WORKSPACE }} + refresh: 'true' + + # 0 = no changes, 2 = drift present. + - name: Inspect drift + run: | + if [[ "${{ steps.graformer.outputs.plan-exitcode }}" == 2 ]]; then + echo "Drift detected — see the drift PR." + else + echo "No drift detected." + fi + + # Raise / update / close a single PR with the detected changes. Reviewers + # decide whether to merge (adopt the change into config) or revert manually. + - name: Open, update or close the drift PR uses: ./.github/actions/drift-pr with: - branch-name: "drift/unmanaged-repos" + branch-name: "drift/detected-changes" reviewers: ${{ inputs.reviewers }} github-token: ${{ steps.generate-token.outputs.token }} From d6f923cc039dd956aea45fc4972db42609a4f3bf Mon Sep 17 00:00:00 2001 From: milosjovanovic Date: Fri, 19 Jun 2026 13:40:35 +0200 Subject: [PATCH 7/9] Add documentation for `Drift Check` workflow and improve drift detection messaging --- .github/workflows/drift-check.yaml | 7 ++++++- docs/workflows.md | 31 ++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/.github/workflows/drift-check.yaml b/.github/workflows/drift-check.yaml index 12264fc..d01ac96 100644 --- a/.github/workflows/drift-check.yaml +++ b/.github/workflows/drift-check.yaml @@ -108,7 +108,12 @@ jobs: - name: Inspect drift run: | if [[ "${{ steps.graformer.outputs.plan-exitcode }}" == 2 ]]; then - echo "Drift detected — see the drift PR." + # Most drift is surfaced as a PR by the next step. Some terraform-level + # drift can't be represented as a config change and so won't appear in + # the PR — e.g. a managed repo deleted or archived on GitHub. Emit a + # warning so that case stays visible in the run summary. + echo "::warning::Terraform detected drift. Config-representable changes are raised as a drift PR; if no PR appears, investigate (e.g. a deleted or archived managed repo)." + echo "Drift detected — see the drift PR (or the warning above if none was raised)." else echo "No drift detected." fi diff --git a/docs/workflows.md b/docs/workflows.md index 4f3beae..aaa8657 100644 --- a/docs/workflows.md +++ b/docs/workflows.md @@ -26,6 +26,37 @@ 5. Upon PR merge, Terraform Cloud plans and applies the configuration 6. Configuration file is then sanitized (ids removed) and moved to the appropriate directory `feature/github-repo-provisioning/repo_configs/{branch}/{organization}` +### 🔍 `Drift Check` Workflow + +- **Trigger**: Scheduled (cron) from the config repo. +- **Behavior**: + 1. Imports the current GitHub state of every repo in the org. + 2. `compare` drops everything that matches the committed config, leaving only **changes** — repos created outside GCSS *and* manual edits to already-managed repos. + 3. Runs `terraform plan` (with `-refresh`) over the result. + 4. Opens / updates / closes a single PR (`drift/detected-changes`) with the detected changes, assigned to the configured reviewers. Reviewers either **merge** (adopt the change into config) or **revert** the change manually. + +#### Consumer setup requirements + +The reusable `drift-check.yaml` runs in the **`schedule`** environment. That environment **must** provide all of: + +| Name | Type | Purpose | +|---|---|---| +| `APP_ID` | variable | Management GitHub App (must have **org-wide "All repositories" access**, or new repos are invisible to the importer) | +| `app_private_key` | secret | Private key for that App | +| `WORKSPACE` | variable | Terraform Cloud workspace | +| `tfc_token` | secret | Terraform Cloud API token | + +> [!IMPORTANT] +> The `schedule` environment **must not** have required-reviewer or wait-timer protection rules. The workflow runs unattended on a schedule, so any approval gate makes every run stall forever. + +Caller also passes `reviewers` (comma-separated users or `org/team` slugs) to request on the drift PR. + +#### Notes / limitations + +- **Scale**: each run does a **full org import** plus a plan. On large orgs this is the dominant cost (several API calls per repo). Pick a cron interval that comfortably exceeds a run's duration — overlapping runs are queued (`cancel-in-progress: false`), so too-frequent scheduling lags detection. +- **Deleted / archived managed repos**: these can't be represented as a config change, so they won't appear in the drift PR. `terraform plan` still flags them and the `Inspect drift` step emits a warning, but resolving them (remove config or recreate the repo) is manual. +- GitHub disables scheduled workflows after long repo inactivity — a disabled `drift-check` means no detection. + ## 📥 Importing Existing Repositories To import an **existing GitHub repository** into Terraform: From 74a4b4e67f523cab5f7f50670114f083437af8d2 Mon Sep 17 00:00:00 2001 From: milosjovanovic Date: Fri, 19 Jun 2026 15:25:56 +0200 Subject: [PATCH 8/9] Improve drift detection by adding `drift-detected` input to avoid false positives --- .github/actions/drift-pr/action.yaml | 20 +++++++++++++++----- .github/workflows/drift-check.yaml | 3 +++ 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/.github/actions/drift-pr/action.yaml b/.github/actions/drift-pr/action.yaml index c8de59d..f07f872 100644 --- a/.github/actions/drift-pr/action.yaml +++ b/.github/actions/drift-pr/action.yaml @@ -12,6 +12,10 @@ inputs: github-token: description: "Token used to push and manage the PR" required: true + drift-detected: + description: "Whether `terraform plan` reported drift (exit code 2). A PR is only opened when terraform confirms real drift — this avoids false positives from config differences terraform treats as no-ops (e.g. empty vs null fields)." + required: false + default: "false" runs: using: "composite" steps: @@ -29,6 +33,7 @@ runs: BRANCH: ${{ inputs.branch-name }} REVIEWERS: ${{ inputs.reviewers }} BASE: ${{ github.ref_name }} + DRIFT_DETECTED: ${{ inputs.drift-detected }} run: | set -euo pipefail SRC="importer_tmp_dir" @@ -37,19 +42,24 @@ runs: detected=$(find "$SRC" -maxdepth 1 -type f \( -name '*.yaml' -o -name '.*.yaml' \) 2>/dev/null | wc -l | tr -d ' ') existing_pr=$(gh pr list --head "$BRANCH" --state open --json number --jq '.[0].number // empty') - # No drift: close any open drift PR (and its branch) so it doesn't go stale. - if [[ "$detected" -eq 0 ]]; then + # Only raise a PR when terraform confirms drift AND we have config changes + # to show. terraform is the authority on whether drift is real; `compare` + # only builds the PR content. This avoids false positives from differences + # terraform treats as no-ops (e.g. empty vs null fields). Drift terraform + # sees but compare can't represent (deleted/archived repos) is surfaced by + # the workflow's `Inspect drift` warning instead. + if [[ "$DRIFT_DETECTED" != "true" || "$detected" -eq 0 ]]; then if [[ -n "$existing_pr" ]]; then gh pr close "$existing_pr" --delete-branch \ --comment "No drift detected — resolved. Closing automatically." - echo "Closed drift PR #$existing_pr (no drift remains)." + echo "Closed drift PR #$existing_pr (no actionable drift remains)." else - echo "No drift detected; nothing to do." + echo "No actionable drift; nothing to do." fi exit 0 fi - echo "Detected $detected changed repository config(s)." + echo "Terraform confirmed drift; $detected changed repository config(s) to surface." # Rebuild the drift branch from the base each run so the PR only ever # reflects the current set of changes (untracked importer_tmp_dir files diff --git a/.github/workflows/drift-check.yaml b/.github/workflows/drift-check.yaml index d01ac96..ff255de 100644 --- a/.github/workflows/drift-check.yaml +++ b/.github/workflows/drift-check.yaml @@ -126,3 +126,6 @@ jobs: branch-name: "drift/detected-changes" reviewers: ${{ inputs.reviewers }} github-token: ${{ steps.generate-token.outputs.token }} + # terraform is the authority on whether drift is real (avoids false + # positives from config differences it treats as no-ops). + drift-detected: ${{ steps.graformer.outputs.plan-exitcode == '2' }} From 5af866c55137d96e6024878d62128b9647501dae Mon Sep 17 00:00:00 2001 From: milosjovanovic Date: Fri, 19 Jun 2026 15:57:14 +0200 Subject: [PATCH 9/9] Normalize optional free-text fields in GitHub repo importer to improve drift detection consistency --- .../github-repo-importer/pkg/github/github.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/feature/github-repo-importer/pkg/github/github.go b/feature/github-repo-importer/pkg/github/github.go index 6c94581..a8d3d55 100644 --- a/feature/github-repo-importer/pkg/github/github.go +++ b/feature/github-repo-importer/pkg/github/github.go @@ -55,6 +55,18 @@ func DecodeAppsList() (*AppsList, error) { return &appsList, nil } +// nilIfEmpty normalises a pointer to an empty string to nil. GitHub returns +// optional free-text fields (e.g. description, homepage) inconsistently as "" +// or null over a repo's lifetime, and Terraform treats them as equivalent. +// Collapsing "" to nil keeps a fresh import byte-stable against the committed +// config so drift detection doesn't report spurious changes. +func nilIfEmpty(s *string) *string { + if s == nil || *s == "" { + return nil + } + return s +} + func ImportRepo(repoName string) (*Repository, error) { fmt.Println("Importing repository: ", repoName) @@ -172,9 +184,9 @@ func ImportRepo(repoName string) (*Repository, error) { return &Repository{ Name: repo.GetName(), Owner: repo.GetOwner().GetLogin(), - Description: repo.Description, + Description: nilIfEmpty(repo.Description), Visibility: repo.GetVisibility(), - HomepageURL: repo.Homepage, + HomepageURL: nilIfEmpty(repo.Homepage), DefaultBranch: repo.GetDefaultBranch(), HasIssues: repo.HasIssues, HasProjects: repo.HasProjects,