diff --git a/.github/actions/compare/action.yaml b/.github/actions/compare/action.yaml index 96adfdd..d1019bc 100644 --- a/.github/actions/compare/action.yaml +++ b/.github/actions/compare/action.yaml @@ -10,6 +10,10 @@ inputs: target-directory: description: "Target directory to compare" required: true + keep: + description: "Which configs to keep in the source dir: 'changed' (new + modified, default) or 'new-only' (only repos that have no existing config)" + required: false + default: "changed" runs: using: "composite" steps: @@ -19,16 +23,26 @@ runs: env: SOURCE_DIRECTORY: ${{ inputs.source-directory }} TARGET_DIRECTORY: ${{ inputs.target-directory }} + KEEP: ${{ inputs.keep }} run: | set -euo pipefail dirA="${SOURCE_DIRECTORY}" dirB="${TARGET_DIRECTORY}" result=$(just compare "$dirA" "$dirB") - files_to_delete=$(echo "$result" | jq -r '.identical[]?') + + # Always drop unchanged (identical) configs. In 'new-only' mode also drop + # modified (different) configs, leaving only repos that have no config yet. + case "${KEEP}" in + new-only) jq_filter='(.identical[]?, .different[]?)' ;; + changed) jq_filter='.identical[]?' ;; + *) echo "Invalid 'keep' value: ${KEEP} (expected 'changed' or 'new-only')" >&2; exit 1 ;; + esac + files_to_delete=$(echo "$result" | jq -r "$jq_filter") + if [[ -z "$files_to_delete" ]]; then - echo "No identical (unchanged) files to remove." + echo "No files to remove (keep mode: ${KEEP})." else - echo "Removing unchanged configs from $dirA so the PR only shows real changes:" + echo "Removing configs from $dirA (keep mode: ${KEEP}) so the PR only shows the intended changes:" while IFS= read -r relpath; do [[ -z "$relpath" ]] && continue rm "$dirA/$relpath" diff --git a/.github/actions/drift-pr/action.yaml b/.github/actions/drift-pr/action.yaml new file mode 100644 index 0000000..19a7b90 --- /dev/null +++ b/.github/actions/drift-pr/action.yaml @@ -0,0 +1,99 @@ +name: "Drift PR for unmanaged repositories" +description: "Opens, updates, or closes a single PR listing repos that exist in the org but are not yet managed by GCSS" +inputs: + branch-name: + description: "Stable branch used for the drift PR" + required: false + default: "drift/unmanaged-repos" + reviewers: + description: "Comma-separated reviewers (users or org/team slugs) to request on the PR" + required: false + default: "" + github-token: + description: "Token used to push and manage the PR" + required: true +runs: + using: "composite" + steps: + - name: Configure git + shell: bash + run: | + git config --global user.name 'github-actions[bot]' + git config --global user.email 'github-actions[bot]@users.noreply.github.com' + + - name: Open, update or close the drift PR + shell: bash + working-directory: feature/github-repo-provisioning/gcss_config + env: + GH_TOKEN: ${{ inputs.github-token }} + BRANCH: ${{ inputs.branch-name }} + REVIEWERS: ${{ inputs.reviewers }} + BASE: ${{ github.ref_name }} + run: | + set -euo pipefail + SRC="importer_tmp_dir" + + # How many unmanaged repo configs did the compare step leave behind? + detected=$(find "$SRC" -maxdepth 1 -type f \( -name '*.yaml' -o -name '.*.yaml' \) 2>/dev/null | wc -l | tr -d ' ') + existing_pr=$(gh pr list --head "$BRANCH" --state open --json number --jq '.[0].number // empty') + + # No drift: close any open drift PR (and its branch) so it doesn't go stale. + if [[ "$detected" -eq 0 ]]; then + if [[ -n "$existing_pr" ]]; then + gh pr close "$existing_pr" --delete-branch \ + --comment "No unmanaged repositories detected — drift resolved. Closing automatically." + echo "Closed drift PR #$existing_pr (no unmanaged repositories remain)." + else + echo "No unmanaged repositories detected; nothing to do." + fi + exit 0 + fi + + echo "Detected $detected unmanaged repository config(s)." + + # Rebuild the drift branch from the base each run so the PR only ever + # reflects the current set of unmanaged repos (untracked importer_tmp_dir + # files are preserved across the checkout). + git fetch origin "$BASE" + git checkout -B "$BRANCH" "origin/$BASE" + git add "$SRC" + + # Nothing staged means the detected configs are already present on the + # base branch (e.g. a drift PR was just merged and is awaiting + # promotion); there is nothing to surface, so exit cleanly instead of + # failing on an empty commit under `set -e`. + if git diff --cached --quiet; then + echo "Detected configs already present on $BASE (pending promotion); nothing to surface." + exit 0 + fi + git commit -m "Detected unmanaged repositories (GCSS-1132)" + + # Push only when the remote branch isn't already identical (avoids + # re-notifying reviewers every run), but always continue so that a PR is + # (re)opened below if one isn't already present — e.g. when a previous + # drift PR was closed without deleting the branch. + if git ls-remote --exit-code --heads origin "$BRANCH" >/dev/null 2>&1; then + git fetch origin "$BRANCH" + if git diff --quiet "origin/$BRANCH" HEAD; then + echo "Remote drift branch already up to date; skipping push." + else + git push --force-with-lease origin "$BRANCH" + fi + else + git push origin "$BRANCH" + fi + + # Ensure exactly one open drift PR exists. + if [[ -z "$existing_pr" ]]; then + pr_url=$(gh pr create \ + --title "Unmanaged repositories detected" \ + --body "These repositories exist in the organisation but are not yet managed by GCSS. Merge this PR to bring them under management, or add them to \`ignored_repos\` in \`config/import-config.yaml\` to acknowledge them." \ + --base "$BASE" \ + --head "$BRANCH") + echo "Opened new drift PR: $pr_url" + if [[ -n "${REVIEWERS}" ]]; then + gh pr edit "$pr_url" --add-reviewer "${REVIEWERS}" || echo "Warning: could not request reviewers '${REVIEWERS}'" + fi + else + echo "Updated existing drift PR #$existing_pr." + fi diff --git a/.github/workflows/drift-check.yaml b/.github/workflows/drift-check.yaml index 51b9af3..2465f3c 100644 --- a/.github/workflows/drift-check.yaml +++ b/.github/workflows/drift-check.yaml @@ -12,12 +12,31 @@ on: type: string description: "Terraform Cloud organization" required: true + reviewers: + type: string + description: "Comma-separated reviewers (users or org/team slugs) to request on the unmanaged-repos drift PR" + required: false + default: "" + detect_environment: + type: string + description: >- + Deployment environment that provides APP_ID for the management app, + used by the unmanaged-repos detection job. Because this runs unattended + on a schedule, the chosen environment MUST NOT have required-reviewer + or wait-timer protection rules, or every run will stall pending manual + approval. Defaults to 'schedule'. + required: false + default: "schedule" secrets: tfc_token: description: "Terraform Cloud API token" required: true + app_private_key: + description: "GitHub App private key for the management app (used by the unmanaged-repos detection job)" + required: true jobs: + # Detects config drift on repositories already managed by Terraform. drift-check: name: "Drift Check" runs-on: ubuntu-latest @@ -53,4 +72,73 @@ jobs: exit 1 else echo "No drift detected." - fi \ No newline at end of file + fi + + # Detects repositories that exist in the organisation but are not yet managed + # by GCSS (e.g. created manually in the GitHub UI) — invisible to `terraform + # plan` above — and surfaces them in a single, continuously-updated PR. + # See GCSS-1132. Runs independently of the terraform-plan drift job. + detect-unmanaged-repos: + name: "Detect unmanaged repositories" + runs-on: ubuntu-latest + # Environment supplies APP_ID for the management app, which must have + # org-wide ("All repositories") access or new repos are invisible to the + # importer (see GCSS-1132). Must be free of approval/protection rules — see + # the 'detect_environment' input description. + environment: ${{ inputs.detect_environment }} + permissions: + contents: write + pull-requests: write + # A full org import can run longer than the schedule interval; prevent + # overlapping runs from racing on the shared drift branch / PR. + concurrency: + group: detect-unmanaged-repos + cancel-in-progress: false + steps: + - name: Generate a token + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + id: generate-token + with: + app-id: ${{ vars.APP_ID }} + private-key: ${{ secrets.app_private_key }} + owner: ${{ github.repository_owner }} + + - name: Checkout GCSS + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + repository: G-Research/github-terraformer + ref: ${{ inputs.gcss_ref }} + persist-credentials: false + + - name: GCSS config setup + uses: ./.github/actions/gcss-config-setup + with: + checkout-sha: ${{ github.sha }} + checkout-token: ${{ steps.generate-token.outputs.token }} + + - name: Setup Just + uses: extractions/setup-just@f8a3cce218d9f83db3a2ecd90e41ac3de6cdfd9b # v3.1.0 + with: + just-version: '1.4.0' + + - name: Import repos + working-directory: feature/github-repo-importer + run: just import-repos + env: + GITHUB_TOKEN: ${{ steps.generate-token.outputs.token }} + OWNER: ${{ github.repository_owner }} + + - name: Filter to unmanaged repos only + uses: ./.github/actions/compare + with: + working-directory: feature/github-repo-importer + source-directory: "../github-repo-provisioning/gcss_config/importer_tmp_dir/" + target-directory: "../github-repo-provisioning/gcss_config/repos/" + keep: "new-only" + + - name: Open or update the drift PR + uses: ./.github/actions/drift-pr + with: + branch-name: "drift/unmanaged-repos" + reviewers: ${{ inputs.reviewers }} + github-token: ${{ steps.generate-token.outputs.token }}