[TEMP] Ack event slow investigation 2

.github/workflows/build-publish.yml

@@ -10,7 +10,7 @@ permissions: {}
 jobs:
   checks:
     name: "Checks"
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-latest
     permissions:
       contents: read
     outputs:
@@ -20,7 +20,7 @@ jobs:
       prerelease-phase: ${{ steps.detect-prerelease-phase.outputs.prerelease-phase }}
       is-hotfix: ${{ steps.detect-hotfix.outputs.is-hotfix }}
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
         with:
           persist-credentials: false
 
@@ -98,7 +98,7 @@ jobs:
     permissions:
       contents: read
       id-token: write
-    uses: smartcontractkit/.github/.github/workflows/reusable-docker-build-publish.yml@f7223bc10928ba2ac7028fe3fae1d435b3bbbf6a # reusable-docker-build-publish/1.2.0
+    uses: smartcontractkit/.github/.github/workflows/reusable-docker-build-publish.yml@b7c347810a36a2d42dc039e3efd5fb86c7f7fa97 # v1.8.0 , intentionally pinned to commit for security
     with:
       aws-ecr-name: chainlink
       aws-region-ecr: us-east-1
@@ -138,7 +138,7 @@ jobs:
     permissions:
       contents: read
       id-token: write
-    uses: smartcontractkit/.github/.github/workflows/reusable-docker-build-publish.yml@f7223bc10928ba2ac7028fe3fae1d435b3bbbf6a # reusable-docker-build-publish/1.2.0
+    uses: smartcontractkit/.github/.github/workflows/reusable-docker-build-publish.yml@b7c347810a36a2d42dc039e3efd5fb86c7f7fa97 # v1.8.0 , intentionally pinned to commit for security
     with:
       aws-ecr-name: ccip
       aws-region-ecr: us-east-1
@@ -184,7 +184,7 @@ jobs:
       contents: read
       id-token: write
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
         with:
           persist-credentials: false
 
@@ -288,4 +288,4 @@ jobs:
             --repo "$GITHUB_REPOSITORY" \
             -f chainlink_version="$TARGET_TAG" \
             -f chainlink_image_tag="$DOCKER_MANIFEST_TAG" \
-            -f slack_thread_ts="$SLACK_THREAD_TS"
+            -f slack_thread_ts="$SLACK_THREAD_TS"

.github/workflows/docker-build.yml

@@ -20,74 +20,68 @@
 
 jobs:
   init:
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-latest
     outputs:
       # To get an image from a feature branch, do one of the following:
       # 1. Use a workflow dispatch
       # 2. Add the `build-publish` label to your PR, and re-run the workflow (or push a commit)
       should-run: >-
         ${{
           steps.pr-labels.outputs.check-label-found == 'true' ||
           github.event_name == 'push' ||
           github.event_name == 'schedule' ||
           github.event_name == 'workflow_dispatch'
         }}
       runner-arm64: ${{ steps.runner-labels.outputs.runner-arm64 }}
       runner-amd64: ${{ steps.runner-labels.outputs.runner-amd64 }}
       checked-out-sha: ${{ steps.checkout-sha.outputs.checked-out-sha }}
       version-tag: ${{ steps.version-info.outputs.version-tag }}
     steps:
       - name: Get PR Labels
         id: pr-labels
         uses: smartcontractkit/.github/actions/get-pr-labels@get-pr-labels/v1
         with:
           check-label: "build-publish"
           skip-merge-group: "true"
 
       - name: Set runner labels
         id: runner-labels
         shell: bash
         env:
-          SH_RUNNER_LABEL_ARM64: runs-on=${{ github.run_id
-            }}/cpu=16/ram=64/family=m7g/spot=false/image=ubuntu24-full-arm64/extras=s3-cache+tmpfs
-          SH_RUNNER_LABEL_AMD64: runs-on=${{ github.run_id
-            }}/cpu=16/ram=64/family=m6i+m7i/spot=false/image=ubuntu24-full-x64/extras=s3-cache+tmpfs
-          GH_RUNNER_LABEL_ARM64: ubuntu-24.04-arm
-          GH_RUNNER_LABEL_AMD64: ubuntu-24.04
+          # All events use runs-on (s3-cache backend) so every build shares caches.
+          # spot=co: runs-on auto-retries interrupted jobs on-demand once.
+          RUNNER_LABEL_ARM64: runs-on=${{ github.run_id
+            }}/cpu=16/ram=64/family=m8g+m9g/spot=co/image=ubuntu24-full-arm64/volume=100gb:gp3:500mbs:4000iops/extras=s3-cache
+          RUNNER_LABEL_AMD64: runs-on=${{ github.run_id
+            }}/cpu=16/ram=64/family=m7i+m8i/spot=co/image=ubuntu24-full-x64/volume=100gb:gp3:500mbs:4000iops/extras=s3-cache
         run: |
-          if [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then
-            echo "runner-arm64=${SH_RUNNER_LABEL_ARM64}" | tee -a "$GITHUB_OUTPUT"
-            echo "runner-amd64=${SH_RUNNER_LABEL_AMD64}" | tee -a "$GITHUB_OUTPUT"
-          else
-            # Use GitHub runner labels for non-PR events
-            echo "runner-arm64=${GH_RUNNER_LABEL_ARM64}" | tee -a "$GITHUB_OUTPUT"
-            echo "runner-amd64=${GH_RUNNER_LABEL_AMD64}" | tee -a "$GITHUB_OUTPUT"
-          fi
+          echo "runner-arm64=${RUNNER_LABEL_ARM64}" | tee -a "$GITHUB_OUTPUT"
+          echo "runner-amd64=${RUNNER_LABEL_AMD64}" | tee -a "$GITHUB_OUTPUT"
 
       - name: Checkout repository
-        uses: actions/checkout@v6
+        uses: actions/checkout@v7
         with:
           ref: ${{ inputs.git-ref || github.sha }}
 
       - name: Resolve checked out SHA
         id: checkout-sha
         shell: bash
         run: echo "checked-out-sha=$(git rev-parse HEAD)" | tee -a "$GITHUB_OUTPUT"
 
       - name: Version Info
         id: version-info
         run: |
           pkg_version=$(jq -r '.version' ./package.json)
           version_tag="${pkg_version}-dev"
           echo "version-tag=${version_tag}" | tee -a "$GITHUB_OUTPUT"
 
   docker-core:
-    needs: [ init ]
+    needs: [init]
     if: ${{ needs.init.outputs.should-run == 'true' }}
     permissions:
       contents: read
       id-token: write
-    uses: smartcontractkit/.github/.github/workflows/reusable-docker-build-publish.yml@reusable-docker-build-publish/v1
+    uses: smartcontractkit/.github/.github/workflows/reusable-docker-build-publish.yml@b7c347810a36a2d42dc039e3efd5fb86c7f7fa97 # v1.8.0 , intentionally pinned to commit for security
     with:
       aws-ecr-name: chainlink
       aws-region-ecr: us-west-2
@@ -100,7 +94,10 @@
         COMMIT_SHA=${{ needs.init.outputs.checked-out-sha }}
         VERSION_TAG=${{ needs.init.outputs.version-tag }}
       docker-manifest-sign: true
-      docker-cache-behaviour: "enable"
+      docker-cache-behaviour: "write-on-trunk"
+      # go-build-cache only; see integration-tests.yml build-chainlink for rationale.
+      # Object form with explicit id is required so the cache id matches the Dockerfile mount.
+      docker-cache-map: '{"cache-mount/go-build-cache": {"id": "go-build-cache", "target": "/var/cache-target"}}'
       git-sha: ${{ needs.init.outputs.checked-out-sha }}
       github-event-name: ${{ github.event_name }}
       github-ref-name: ${{ github.ref_name }}
@@ -116,12 +113,12 @@
       AWS_LAMBDA_GATI_URL: ${{ secrets.AWS_INFRA_RELENG_TOKEN_ISSUER_LAMBDA_URL }}
 
   docker-core-plugins:
-    needs: [ init ]
+    needs: [init]
     if: ${{ needs.init.outputs.should-run == 'true' }}
     permissions:
       contents: read
       id-token: write
-    uses: smartcontractkit/.github/.github/workflows/reusable-docker-build-publish.yml@reusable-docker-build-publish/v1
+    uses: smartcontractkit/.github/.github/workflows/reusable-docker-build-publish.yml@b7c347810a36a2d42dc039e3efd5fb86c7f7fa97 # v1.8.0 , intentionally pinned to commit for security
     with:
       aws-ecr-name: chainlink
       aws-region-ecr: us-west-2
@@ -136,7 +133,10 @@
         CL_INSTALL_PRIVATE_PLUGINS=true
       docker-manifest-sign: true
       docker-tag-custom-suffix: "-plugins"
-      docker-cache-behaviour: "enable"
+      docker-cache-behaviour: "write-on-trunk"
+      # go-build-cache only; see integration-tests.yml build-chainlink for rationale.
+      # Object form with explicit id is required so the cache id matches the Dockerfile mount.
+      docker-cache-map: '{"cache-mount/go-build-cache": {"id": "go-build-cache", "target": "/var/cache-target"}}'
       git-sha: ${{ needs.init.outputs.checked-out-sha }}
       github-event-name: ${{ github.event_name }}
       github-ref-name: ${{ github.ref_name }}
@@ -152,12 +152,12 @@
       AWS_LAMBDA_GATI_URL: ${{ secrets.AWS_INFRA_RELENG_TOKEN_ISSUER_LAMBDA_URL }}
 
   docker-core-plugins-testing:
-    needs: [ init ]
+    needs: [init]
     if: ${{ needs.init.outputs.should-run == 'true' }}
     permissions:
       contents: read
       id-token: write
-    uses: smartcontractkit/.github/.github/workflows/reusable-docker-build-publish.yml@reusable-docker-build-publish/v1
+    uses: smartcontractkit/.github/.github/workflows/reusable-docker-build-publish.yml@b7c347810a36a2d42dc039e3efd5fb86c7f7fa97 # v1.8.0 , intentionally pinned to commit for security
     with:
       aws-ecr-name: chainlink
       aws-region-ecr: us-west-2
@@ -173,7 +173,10 @@
         CL_INSTALL_TESTING_PLUGINS=true
       docker-manifest-sign: true
       docker-tag-custom-suffix: "-plugins-testing"
-      docker-cache-behaviour: "enable"
+      docker-cache-behaviour: "write-on-trunk"
+      # go-build-cache only; see integration-tests.yml build-chainlink for rationale.
+      # Object form with explicit id is required so the cache id matches the Dockerfile mount.
+      docker-cache-map: '{"cache-mount/go-build-cache": {"id": "go-build-cache", "target": "/var/cache-target"}}'
       git-sha: ${{ needs.init.outputs.checked-out-sha }}
       github-event-name: ${{ github.event_name }}
       github-ref-name: ${{ github.ref_name }}
@@ -189,12 +192,12 @@
       AWS_LAMBDA_GATI_URL: ${{ secrets.AWS_INFRA_RELENG_TOKEN_ISSUER_LAMBDA_URL }}
 
   docker-ccip:
-    needs: [ init ]
+    needs: [init]
     if: ${{ needs.init.outputs.should-run == 'true' }}
     permissions:
       contents: read
       id-token: write
-    uses: smartcontractkit/.github/.github/workflows/reusable-docker-build-publish.yml@reusable-docker-build-publish/v1
+    uses: smartcontractkit/.github/.github/workflows/reusable-docker-build-publish.yml@b7c347810a36a2d42dc039e3efd5fb86c7f7fa97 # v1.8.0 , intentionally pinned to commit for security
     with:
       aws-ecr-name: ccip
       aws-region-ecr: us-west-2
@@ -210,7 +213,10 @@
         CL_CHAIN_DEFAULTS=/ccip-config
         CL_SOLANA_CMD=
       docker-manifest-sign: true
-      docker-cache-behaviour: "enable"
+      docker-cache-behaviour: "write-on-trunk"
+      # go-build-cache only; see integration-tests.yml build-chainlink for rationale.
+      # Object form with explicit id is required so the cache id matches the Dockerfile mount.
+      docker-cache-map: '{"cache-mount/go-build-cache": {"id": "go-build-cache", "target": "/var/cache-target"}}'
       git-sha: ${{ needs.init.outputs.checked-out-sha }}
       github-event-name: ${{ github.event_name }}
       github-ref-name: ${{ github.ref_name }}
@@ -226,12 +232,12 @@
       AWS_LAMBDA_GATI_URL: ${{ secrets.AWS_INFRA_RELENG_TOKEN_ISSUER_LAMBDA_URL }}
 
   docker-ccip-plugins:
-    needs: [ init ]
+    needs: [init]
     if: ${{ needs.init.outputs.should-run == 'true' }}
     permissions:
       contents: read
       id-token: write
-    uses: smartcontractkit/.github/.github/workflows/reusable-docker-build-publish.yml@reusable-docker-build-publish/v1
+    uses: smartcontractkit/.github/.github/workflows/reusable-docker-build-publish.yml@b7c347810a36a2d42dc039e3efd5fb86c7f7fa97 # v1.8.0 , intentionally pinned to commit for security
     with:
       aws-ecr-name: ccip
       aws-region-ecr: us-west-2
@@ -247,7 +253,10 @@
         CL_CHAIN_DEFAULTS=/ccip-config
       docker-manifest-sign: true
       docker-tag-custom-suffix: "-plugins"
-      docker-cache-behaviour: "enable"
+      docker-cache-behaviour: "write-on-trunk"
+      # go-build-cache only; see integration-tests.yml build-chainlink for rationale.
+      # Object form with explicit id is required so the cache id matches the Dockerfile mount.
+      docker-cache-map: '{"cache-mount/go-build-cache": {"id": "go-build-cache", "target": "/var/cache-target"}}'
       git-sha: ${{ needs.init.outputs.checked-out-sha }}
       github-event-name: ${{ github.event_name }}
       github-ref-name: ${{ github.ref_name }}
@@ -265,7 +274,7 @@
   call-post-docker-build:
     if: ${{ github.event_name == 'schedule' || github.event_name ==
       'workflow_dispatch' }}
-    needs: [ init, docker-core ]
+    needs: [init, docker-core]
     uses: ./.github/workflows/post-docker-build.yml
     permissions:
       actions: read
@@ -280,13 +289,13 @@
     if: false
     # temporarily disabled
     # if: ${{ github.event_name == 'schedule' }}
-    needs: [ docker-core ]
+    needs: [docker-core]
     permissions:
       contents: read
       id-token: write
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
       - name: Deploy
         uses: ./.github/actions/deploy-image
         with:
@@ -295,21 +304,22 @@
           aws-region: ${{ secrets.AWS_REGION }}
           repo-destination: ${{ secrets.REPO_K8S_DEPLOY }}
           oci-image-tag: ${{ needs.docker-core.outputs.docker-manifest-tag }}
-          oci-repository-url: ${{ format('{0}.dkr.ecr.{1}.amazonaws.com/chainlink',
+          oci-repository-url:
+            ${{ format('{0}.dkr.ecr.{1}.amazonaws.com/chainlink',
             secrets.AWS_ACCOUNT_ID_SDLC, secrets.AWS_REGION) }}
           pr-close-enabled: false
           products: |
             cre
 
   deploy-nightly-ccip:
     if: ${{ github.event_name == 'schedule' }}
-    needs: [ docker-ccip ]
+    needs: [docker-ccip]
     permissions:
       contents: read
       id-token: write
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
       - name: Deploy
         uses: ./.github/actions/deploy-image
         with:
@@ -322,4 +332,4 @@
             secrets.AWS_ACCOUNT_ID_SDLC, secrets.AWS_REGION) }}
           pr-close-enabled: false
           products: |
-            ccip-nightlies
+            ccip-nightlies

core/capabilities/remote/trigger_publisher.go

@@ -194,7 +194,7 @@ func (p *triggerPublisher) initMetrics() error {
 	if err != nil {
 		return fmt.Errorf("failed to register platform_trigger_publisher_ack_event_total: %w", err)
 	}
-	durationBuckets := metric.WithExplicitBucketBoundaries(1, 50, 250, 1_000, 5_000, 30_000)
+	durationBuckets := metric.WithExplicitBucketBoundaries(1, 50, 250, 1_000, 5_000, 30_000, 120_000)
 	p.metrics.registerTriggerDurationMs, err = beholder.GetMeter().Int64Histogram("platform_trigger_publisher_register_trigger_duration_ms", durationBuckets)
 	if err != nil {
 		return fmt.Errorf("failed to register platform_trigger_publisher_register_trigger_duration_ms: %w", err)

core/scripts/go.mod

@@ -45,7 +45,7 @@ require (
 	github.com/smartcontractkit/chain-selectors v1.0.108
 	github.com/smartcontractkit/chainlink-automation v0.8.1
 	github.com/smartcontractkit/chainlink-ccip/chains/evm v0.0.0-20260821001950-7520b255725e
-	github.com/smartcontractkit/chainlink-common v0.11.2-0.20260828143819-eb996f0a6c8b
+	github.com/smartcontractkit/chainlink-common v0.11.2-0.20260904110241-1c7703887d99
 	github.com/smartcontractkit/chainlink-common/keystore v1.3.0
 	github.com/smartcontractkit/chainlink-deployments-framework v0.119.0
 	github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260810110946-8174b6bb7fc9

deployment/go.mod

@@ -49,7 +49,7 @@ require (
 	github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260624154507-ea7ff77a0ddb
 	github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260624154507-ea7ff77a0ddb
 	github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20260821001950-7520b255725e
-	github.com/smartcontractkit/chainlink-common v0.11.2-0.20260828143819-eb996f0a6c8b
+	github.com/smartcontractkit/chainlink-common v0.11.2-0.20260904110241-1c7703887d99
 	github.com/smartcontractkit/chainlink-common/keystore v1.3.0
 	github.com/smartcontractkit/chainlink-data-streams v1.1.1
 	github.com/smartcontractkit/chainlink-deployments-framework v0.119.0

go.mod

@@ -83,7 +83,7 @@ require (
 	github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260415165642-49f23e4d76cc
 	github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260415165642-49f23e4d76cc
 	github.com/smartcontractkit/chainlink-ccv v0.6.1-0.20260831194426-4d0dcce7c395
-	github.com/smartcontractkit/chainlink-common v0.11.2-0.20260828143819-eb996f0a6c8b
+	github.com/smartcontractkit/chainlink-common v0.11.2-0.20260904110241-1c7703887d99
 	github.com/smartcontractkit/chainlink-common/keystore v1.3.0
 	github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260724142814-45996a1bcb72
 	github.com/smartcontractkit/chainlink-data-streams v1.1.1