Run dependency explanations separately from main AMD canaries

.github/workflows/dependency-upgrade-report.yml

@@ -0,0 +1,167 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+name: Dependency upgrade report
+# Both the job gate and the API validation reject PRs and foreign repositories. Only
+# completed main schedule/dispatch runs are executed, with a read-only token and no secrets.
+"on":  # zizmor: ignore[dangerous-triggers]
+  workflow_run:
+    workflows: ["Tests (AMD)"]
+    types: [completed]
+    branches: [main]
+permissions:
+  contents: read
+  actions: read
+concurrency:
+  group: dependency-report-${{ github.event.workflow_run.id }}
+  cancel-in-progress: false
+jobs:
+  plan:
+    runs-on: ubuntu-slim
+    timeout-minutes: 5
+    if: >-
+      github.event.workflow_run.head_repository.full_name == github.repository &&
+      (github.event.workflow_run.event == 'schedule' ||
+       github.event.workflow_run.event == 'workflow_dispatch')
+    outputs:
+      matrix: ${{ steps.plan.outputs.matrix }}
+      has-reports: ${{ steps.plan.outputs.has-reports }}
+      source-sha: ${{ steps.plan.outputs.source-sha }}
+      source-run-id: ${{ steps.plan.outputs.source-run-id }}
+    steps:
+      - name: "Select inputs from the original canary run"
+        id: plan
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3  # v9.0.0
+        env:
+          SOURCE_RUN_ID: ${{ github.event.workflow_run.id }}
+        with:
+          script: |
+            const rawId = process.env.SOURCE_RUN_ID;
+            if (!/^[1-9][0-9]*$/.test(rawId)) throw new Error('Invalid source run ID');
+            const run_id = Number(rawId);
+            if (!Number.isSafeInteger(run_id)) throw new Error('Invalid source run ID');
+            const repo = context.repo;
+            const { data: run } = await github.rest.actions.getWorkflowRun({ ...repo, run_id });
+            if (run.head_repository.full_name !== `${repo.owner}/${repo.repo}` ||
+                run.head_branch !== 'main' || run.path !== '.github/workflows/ci-amd.yml' ||
+                !['schedule', 'workflow_dispatch'].includes(run.event) || run.status !== 'completed') {
+              throw new Error('Only completed main AMD canary runs are supported');
+            }
+            const artifacts = await github.paginate(github.rest.actions.listWorkflowRunArtifacts,
+              { ...repo, run_id, per_page: 100 });
+            const include = [];
+            const inputName = new RegExp('^dependency-report-inputs-(3\\.[0-9]+)-' +
+              '(constraints(?:-source-providers|-no-providers)?)-(true|false)$');
+            for (const input of artifacts) {
+              const match = inputName.exec(input.name);
+              if (!match || match[1] === '3.10') continue;
+              const [, python, mode, useUv] = match;
+              const image = artifacts.find(a => a.name === `ci-image-save-v3-linux_amd64-${python}-main`);
+              if (input.expired || !image || image.expired) {
+                throw new Error(`Original inputs or CI image unavailable for ${python}:${mode}`);
+              }
+              include.push({ python, mode, 'use-uv': useUv,
+                'inputs-id': input.id, 'image-id': image.id });
+            }
+            core.setOutput('matrix', JSON.stringify({ include }));
+            core.setOutput('has-reports', include.length > 0 ? 'true' : 'false');
+            core.setOutput('source-sha', run.head_sha);
+            core.setOutput('source-run-id', rawId);
+            if (!include.length) {
+              if (Number(process.env.GITHUB_RUN_ATTEMPT) > 1) {
+                throw new Error('Saved report inputs are missing or expired');
+              }
+              core.notice('No saved report inputs: this run did not request independent diagnostics');
+            }
+            await core.summary.addHeading('Dependency upgrade diagnostics')
+              .addLink('Source canary run', run.html_url)
+              .addRaw(`\n\nSource revision: ${run.head_sha}\n\nReports: ${include.length}\n`)
+              .write();
+
+  explain:
+    needs: plan
+    if: needs.plan.outputs.has-reports == 'true'
+    name: "Explain ${{ matrix.python }}:${{ matrix.mode }}"
+    runs-on: ubuntu-22.04
+    timeout-minutes: 90
+    strategy:
+      fail-fast: false
+      max-parallel: 3
+      matrix: ${{ fromJSON(needs.plan.outputs.matrix) }}
+    env:
+      GITHUB_REPOSITORY: ${{ github.repository }}
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      GITHUB_USERNAME: ${{ github.actor }}
+      VERBOSE: "false"
+      USE_UV: ${{ matrix.use-uv }}
+    steps:
+      - name: "Checkout the tested revision"
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        with:
+          ref: ${{ needs.plan.outputs.source-sha }}
+          persist-credentials: false
+      - name: "Install Breeze"
+        uses: ./.github/actions/breeze
+      - name: "Make /mnt writeable"
+        run: ./scripts/ci/make_mnt_writeable.sh
+      - name: "Download the original CI image"
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c  # v8.0.1
+        with:
+          artifact-ids: ${{ matrix.image-id }}
+          run-id: ${{ needs.plan.outputs.source-run-id }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          merge-multiple: true
+          path: /mnt
+      - name: "Load the original CI image"
+        env:
+          PYTHON_VERSION: ${{ matrix.python }}
+        run: breeze ci-image load --platform linux/amd64 --python "${PYTHON_VERSION}" --image-file-dir /mnt
+      - name: "Download the constraints used by the quick summary"
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c  # v8.0.1
+        with:
+          artifact-ids: ${{ matrix.inputs-id }}
+          run-id: ${{ needs.plan.outputs.source-run-id }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          merge-multiple: true
+          path: files/dependency-report
+      - name: "Explain outdated dependencies"
+        shell: bash
+        env:
+          PYTHON_VERSION: ${{ matrix.python }}
+          CONSTRAINTS_MODE: ${{ matrix.mode }}
+          SOURCE_RUN_ID: ${{ needs.plan.outputs.source-run-id }}
+        run: |
+          {
+            echo "## Dependency upgrade explanations"
+            echo "Source canary run: ${SOURCE_RUN_ID}"
+            echo "Constraints commit: $(cat files/dependency-report/constraints-commit.txt)"
+            echo "Python: ${PYTHON_VERSION}; mode: ${CONSTRAINTS_MODE}"
+          } >> "${GITHUB_STEP_SUMMARY}"
+          breeze release-management constraints-version-check \
+            --python "${PYTHON_VERSION}" --airflow-constraints-mode "${CONSTRAINTS_MODE}" \
+            --constraints-file files/dependency-report/constraints.txt --explain-why \
+            2>&1 | tee files/dependency-report/explanations.txt
+      - name: "Save diagnostic output, including partial output on failure"
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
+        with:
+          name: dependency-explanations-${{ matrix.python }}-${{ matrix.mode }}
+          path: files/dependency-report/
+          if-no-files-found: ignore
+          retention-days: 7
+          overwrite: true

.github/workflows/finalize-tests.yml

@@ -149,19 +149,58 @@ jobs:
           python: ${{ matrix.python-version }}
           use-uv: ${{ inputs.use-uv }}
           make-mnt-writeable-and-cleanup: true
+      - name: "Save constraints for the independent dependency report"
+        shell: bash
+        if: >-
+          github.ref == 'refs/heads/main' &&
+          (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          CONSTRAINTS_BRANCH: ${{ inputs.constraints-branch }}
+          PYTHON_VERSION: ${{ matrix.python-version }}
+          CONSTRAINTS_MODE: ${{ matrix.constraints-mode }}
+        run: |
+          mkdir -p files/dependency-report
+          REF_API="repos/apache/airflow/git/ref/heads/${CONSTRAINTS_BRANCH}"
+          CONSTRAINTS_SHA=$(gh api "${REF_API}" --jq '.object.sha')
+          FILE_API="repos/apache/airflow/contents/${CONSTRAINTS_MODE}-${PYTHON_VERSION}.txt"
+          gh api "${FILE_API}?ref=${CONSTRAINTS_SHA}" \
+            --jq '.content' | base64 --decode > files/dependency-report/constraints.txt
+          echo "${CONSTRAINTS_SHA}" > files/dependency-report/constraints-commit.txt
+      - name: "Upload dependency report inputs"
+        if: >-
+          github.ref == 'refs/heads/main' &&
+          (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
+        with:
+          name: "dependency-report-inputs-${{ matrix.python-version }}-${{ matrix.constraints-mode }}\
+            -${{ inputs.use-uv }}"
+          path: files/dependency-report/
+          if-no-files-found: error
+          retention-days: 2
+          overwrite: true
       - name: "Deps: ${{ matrix.python-version }}:${{ matrix.constraints-mode }}"
         shell: bash
-        run: >
-          breeze release-management constraints-version-check
-          --python "${MATRIX_PYTHON_VERSION}"
-          --airflow-constraints-mode "${MATRIX_CONSTRAINTS_MODE}" "${EXPLAIN_WHY_FLAG}"
+        run: |
+          CONSTRAINTS_ARGS=()
+          if [[ -f files/dependency-report/constraints.txt ]]; then
+            CONSTRAINTS_ARGS=(--constraints-file files/dependency-report/constraints.txt)
+          fi
+          breeze release-management constraints-version-check \
+            --python "${MATRIX_PYTHON_VERSION}" \
+            --airflow-constraints-mode "${MATRIX_CONSTRAINTS_MODE}" "${EXPLAIN_WHY_FLAG}" \
+            "${CONSTRAINTS_ARGS[@]}"
         env:
           MATRIX_PYTHON_VERSION: "${{ matrix.python-version }}"
           MATRIX_CONSTRAINTS_MODE: "${{ matrix.constraints-mode }}"
           # Explaining an outdated package resolves its dependency tree; on Python 3.10 more and more
           # dependencies have dropped support, so there is far more to explain and the job takes much
           # longer. Not worth it weeks before 3.10 support is dropped - remove this along with 3.10.
-          EXPLAIN_WHY_FLAG: ${{ matrix.python-version == '3.10' && '--no-explain-why' || '--explain-why' }}
+          EXPLAIN_WHY_FLAG: >-
+            ${{ (matrix.python-version == '3.10' ||
+                 (github.ref == 'refs/heads/main' &&
+                  (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')))
+                && '--no-explain-why' || '--explain-why' }}
           VERBOSE: "false"
 
   push-buildx-cache-to-github-registry:

dev/breeze/doc/ci/05_workflows.md

@@ -368,6 +368,35 @@ unprivileged, which constrains what can go on it:
 When adding a job, reach for `ubuntu-slim` if it only shuffles metadata around, and
 `ubuntu-22.04` otherwise.
 
+### Dependency upgrade diagnostics
+
+Scheduled and manually dispatched AMD canaries on `main` keep the dependency freshness
+summary in `finalize-tests.yml`, but run it with `--no-explain-why`. Tests, constraint
+generation and validation, and image publication still run in the canary. Release-branch
+and other run types retain their existing explanation behavior.
+
+After the canary completes, `dependency-upgrade-report.yml` runs the detailed explanations
+independently, including when tests failed but the report inputs were saved. It checks out
+the original source SHA and downloads the original image and saved constraints by artifact
+ID from that run. It never substitutes the latest branch image or constraints. Python 3.10
+remains summary-only. PyPI release metadata is fetched when the report executes, so the
+available upgrade targets can change between the summary and the explanations.
+
+The workflow has read-only repository permissions and runs only on `workflow_run`, which
+cannot write to the default branch's GitHub Actions cache. Reports use the saved image artifact.
+
+The report runs at most three jobs concurrently. Each job has a 90-minute limit and saves
+its output, including partial output on failure. A failed or timed-out report is incomplete;
+it does not invalidate the canary's test result. Find diagnostics in the **Dependency
+upgrade report** workflow, whose summary links to the source canary. To repeat diagnostics,
+use **Re-run jobs** on that report before its two-day input/image retention expires (or run
+`gh run rerun REPORT_RUN_ID`). Reruns keep the original source canary ID. Missing or expired
+original inputs fail the rerun rather than using a newer image or constraints.
+
+`breeze release-management constraints-version-check --constraints-file PATH` can also
+analyze a saved constraints file locally. Detailed explanations reuse one baseline resolution
+and skip the pinned resolution when that baseline already selects the target version.
+
 ## Implementation Details
 
 Here's how the composite workflow system is organized in practice.