From 01d47536522c5082127757b8d28fea529e6752be Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Thu, 11 Jun 2026 17:21:04 -0700
Subject: [PATCH 01/84] added unit test sharding

---
 .github/workflows/unittest.yml | 43 ++++++++++++++++++++++++++++------
 ci/run_conditional_tests.sh    | 34 ++++++++++++++++++++++++++-
 2 files changed, 69 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml
index 83a8280132e9..0c7882624447 100644
--- a/.github/workflows/unittest.yml
+++ b/.github/workflows/unittest.yml
@@ -13,11 +13,40 @@ permissions:
   contents: read
 
 jobs:
+  discover:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 0
+    - name: Setup Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: "3.10"
+    - name: Get package shards
+      id: set-matrix
+      env:
+        BUILD_TYPE: presubmit
+        TARGET_BRANCH: ${{ github.base_ref || github.event.merge_group.base_ref }}
+      run: |
+        if [ -n "$TARGET_BRANCH" ]; then
+          git fetch origin "$TARGET_BRANCH" --deepen=200
+        fi
+        echo "matrix=$(python ci/get_package_shards.py)" >> $GITHUB_OUTPUT
+
   unit:
+    needs: discover
+    if: needs.discover.outputs.matrix != '[]' && needs.discover.outputs.matrix != ''
     runs-on: ubuntu-22.04
     strategy:
+      fail-fast: false
       matrix:
         python: ['3.9', '3.10', "3.11", "3.12", "3.13", "3.14"]
+        package_shard: ${{ fromJson(needs.discover.outputs.matrix) }}
+    name: unit (${{ matrix.package_shard.name }}, ${{ matrix.python }})
     steps:
     - name: Checkout
       uses: actions/checkout@v4
@@ -36,17 +65,18 @@ jobs:
         python -m pip install nox
     - name: Run unit tests
       env:
-        COVERAGE_FILE: .coverage-${{ matrix.python }}
+        COVERAGE_FILE: ${{ github.workspace }}/.coverage-${{ matrix.python }}
         BUILD_TYPE: presubmit
         TARGET_BRANCH: ${{ github.base_ref || github.event.merge_group.base_ref }}
         TEST_TYPE: unit
         PY_VERSION: ${{ matrix.python }}
+        PACKAGE_LIST: ${{ matrix.package_shard.packages }}
       run: |
         ci/run_conditional_tests.sh
     - name: Upload coverage results
       uses: actions/upload-artifact@v4
       with:
-        name: coverage-artifact-${{ '{{' }} matrix.python {{ '}}' }}
+        name: coverage-artifact-${{ matrix.python }}-${{ matrix.package_shard.index }}
         path: .coverage-${{ matrix.python }}
 
   cover:
@@ -67,20 +97,19 @@ jobs:
         python-version: "3.10"
     - name: Set number of files changes in packages directory
       id: packages
-      run: echo "::set-output name=num_files_changed::$(git diff HEAD~1 -- packages | wc -l)"
+      run: echo "num_files_changed=$(git diff HEAD~1 -- packages | wc -l)" >> $GITHUB_OUTPUT
     - name: Install coverage
-      if: steps.packages.num_files_changed > 0
+      if: steps.packages.outputs.num_files_changed > 0
       run: |
         python -m pip install --upgrade setuptools pip wheel
         python -m pip install coverage
     - name: Download coverage results
-      if: ${{ steps.date.packages.num_files_changed > 0 }}
+      if: steps.packages.outputs.num_files_changed > 0
       uses: actions/download-artifact@v4
       with:
         path: .coverage-results/
     - name: Report coverage results
-      if: ${{ steps.date.packages.num_files_changed > 0 }}
+      if: steps.packages.outputs.num_files_changed > 0
       run: |
-        find .coverage-results -type f -name '*.zip' -exec unzip {} \;
         coverage combine .coverage-results/**/.coverage*
         coverage report --show-missing --fail-under=100
diff --git a/ci/run_conditional_tests.sh b/ci/run_conditional_tests.sh
index 9b8eaee52e5b..12b02c127ae9 100755
--- a/ci/run_conditional_tests.sh
+++ b/ci/run_conditional_tests.sh
@@ -82,10 +82,42 @@ subdirs=(
     packages
 )
 
+if [ -n "${PACKAGE_LIST}" ]; then
+    echo "Using provided PACKAGE_LIST"
+    to_test=(${PACKAGE_LIST})
+    RETVAL=0
+    for d in ${to_test[@]}; do
+        echo "running test in ${d}"
+        pushd ${d}
+        set +e
+        ${test_script}
+        ret=$?
+        set -e
+        if [ ${ret} -ne 0 ]; then
+            RETVAL=${ret}
+        fi
+        popd
+    done
+    exit ${RETVAL}
+fi
+
+# Sharding logic (fallback for manual runs)
+TOTAL_SHARDS="${TOTAL_SHARDS:-1}"
+SHARD_INDEX="${SHARD_INDEX:-1}"
+count=0
+
 RETVAL=0
 
 for subdir in ${subdirs[@]}; do
-    for d in `ls -d ${subdir}/*/`; do
+    # Sort the directories to ensure consistent sharding across jobs
+    for d in `ls -d ${subdir}/*/ | sort`; do
+        # Sharding logic: only process directories that belong to this shard
+        if (( count % TOTAL_SHARDS != SHARD_INDEX - 1 )); then
+            ((count++))
+            continue
+        fi
+        ((count++))
+
         should_test=false
         if [ -n "${GIT_DIFF_ARG}" ]; then
             echo "checking changes with 'git diff --quiet ${GIT_DIFF_ARG} ${d}'"

From a7502d4e8a0969010a2eb6a641092a9e879326f9 Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Thu, 11 Jun 2026 17:21:14 -0700
Subject: [PATCH 02/84] run system tests concurrently

---
 .kokoro/system.sh | 88 +++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 86 insertions(+), 2 deletions(-)

diff --git a/.kokoro/system.sh b/.kokoro/system.sh
index 469d0e81c7fa..52550987340b 100755
--- a/.kokoro/system.sh
+++ b/.kokoro/system.sh
@@ -39,6 +39,9 @@ run_package_test() {
   local package_name=$1
   local package_path="packages/${package_name}"
   
+  # Isolate gcloud config for parallel execution
+  export CLOUDSDK_CONFIG=$(mktemp -d)
+
   # Declare local overrides to prevent bleeding into the next loop iteration
   local PROJECT_ID
   local GOOGLE_APPLICATION_CREDENTIALS
@@ -87,12 +90,52 @@ run_package_test() {
   set -e
   popd > /dev/null
   
+  # Clean up isolated gcloud config
+  rm -rf "${CLOUDSDK_CONFIG}"
+
   return $res
 }
 
 # A file for running system tests
 system_test_script="${PROJECT_ROOT}/.kokoro/system-single.sh"
 
+# Parallel execution settings
+MAX_PARALLEL=4
+running_pids=()
+declare -A pid_to_pkg
+declare -A pid_to_log
+declare -A pid_to_resfile
+
+# Array to keep track of results for the final summary
+results=()
+
+handle_finished_job() {
+  local pid=$1
+  local pkg=${pid_to_pkg[$pid]}
+  local log=${pid_to_log[$pid]}
+  local resfile=${pid_to_resfile[$pid]}
+  
+  # wait $pid might fail if it was already reaped by wait -n, 
+  # so we ignore its exit code and use the resfile.
+  wait "$pid" 2>/dev/null || true
+  
+  local res=$(cat "$resfile")
+  rm "$resfile"
+  
+  echo "------------------------------------------------------------"
+  echo "System tests for ${pkg} finished (Exit code: ${res})"
+  echo "------------------------------------------------------------"
+  cat "$log"
+  rm "$log"
+  
+  if [ "${res}" -ne 0 ]; then
+    RETVAL=${res}
+    results+=("${pkg}: FAILED")
+  else
+    results+=("${pkg}: PASSED")
+  fi
+}
+
 # Run system tests for each package with directory packages/*/tests/system
 for path in `find 'packages' \
   \( -type d -wholename 'packages/*/tests/system' \) -o \
@@ -140,10 +183,51 @@ for path in `find 'packages' \
   set -e
 
   if [[ "${package_modified}" -gt 0 || "$KOKORO_BUILD_ARTIFACTS_SUBDIR" == *"continuous"* ]]; then
-      # Call the function - its internal exports won't affect the next loop
-      run_package_test "$package_name" || RETVAL=$?
+      # Wait if we have reached MAX_PARALLEL
+      while [[ ${#running_pids[@]} -ge $MAX_PARALLEL ]]; do
+        wait -n
+        # Find which job finished
+        new_pids=()
+        for pid in "${running_pids[@]}"; do
+          if kill -0 "$pid" 2>/dev/null; then
+            new_pids+=("$pid")
+          else
+            handle_finished_job "$pid"
+          fi
+        done
+        running_pids=("${new_pids[@]}")
+      done
+
+      # Start the next test in the background
+      log_file=$(mktemp)
+      res_file=$(mktemp)
+      (
+        run_package_test "$package_name" > "$log_file" 2>&1
+        echo $? > "$res_file"
+      ) &
+      pid=$!
+      running_pids+=($pid)
+      pid_to_pkg[$pid]=$package_name
+      pid_to_log[$pid]=$log_file
+      pid_to_resfile[$pid]=$res_file
+      echo "Started system tests for ${package_name} (PID: ${pid})"
   else
       echo "No changes in ${package_name} and not a continuous build, skipping."
   fi
 done
+
+# Wait for all remaining jobs
+for pid in "${running_pids[@]}"; do
+  handle_finished_job "$pid"
+done
+
+echo "------------------------------------------------------------"
+echo "System Test Summary"
+echo "------------------------------------------------------------"
+for res in "${results[@]}"; do
+  echo "$res"
+done
+echo "------------------------------------------------------------"
+
 exit ${RETVAL}
+

From 9ad0f0c2abb24862dbbd6ea8e1212130acf12b5d Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Thu, 11 Jun 2026 17:24:57 -0700
Subject: [PATCH 03/84] added changes to many packages for testing

---
 packages/django-google-spanner/SHARD_TEST.txt                    | 1 +
 packages/google-cloud-appoptimize/SHARD_TEST.txt                 | 1 +
 packages/google-cloud-artifact-registry/SHARD_TEST.txt           | 1 +
 packages/google-cloud-asset/SHARD_TEST.txt                       | 1 +
 packages/google-cloud-assured-workloads/SHARD_TEST.txt           | 1 +
 packages/google-cloud-audit-log/SHARD_TEST.txt                   | 1 +
 packages/google-cloud-auditmanager/SHARD_TEST.txt                | 1 +
 packages/google-cloud-automl/SHARD_TEST.txt                      | 1 +
 packages/google-cloud-backupdr/SHARD_TEST.txt                    | 1 +
 packages/google-cloud-bare-metal-solution/SHARD_TEST.txt         | 1 +
 packages/google-cloud-batch/SHARD_TEST.txt                       | 1 +
 packages/google-cloud-beyondcorp-appconnections/SHARD_TEST.txt   | 1 +
 packages/google-cloud-beyondcorp-appconnectors/SHARD_TEST.txt    | 1 +
 packages/google-cloud-beyondcorp-appgateways/SHARD_TEST.txt      | 1 +
 .../SHARD_TEST.txt                                               | 1 +
 packages/google-cloud-beyondcorp-clientgateways/SHARD_TEST.txt   | 1 +
 packages/google-cloud-biglake-hive/SHARD_TEST.txt                | 1 +
 packages/google-cloud-biglake/SHARD_TEST.txt                     | 1 +
 packages/google-cloud-bigquery-analyticshub/SHARD_TEST.txt       | 1 +
 packages/google-cloud-bigquery-biglake/SHARD_TEST.txt            | 1 +
 packages/google-cloud-bigquery-connection/SHARD_TEST.txt         | 1 +
 packages/google-cloud-bigquery-data-exchange/SHARD_TEST.txt      | 1 +
 packages/google-cloud-bigquery-datapolicies/SHARD_TEST.txt       | 1 +
 packages/google-cloud-bigquery-datatransfer/SHARD_TEST.txt       | 1 +
 packages/google-cloud-bigquery-logging/SHARD_TEST.txt            | 1 +
 packages/google-cloud-bigquery-migration/SHARD_TEST.txt          | 1 +
 packages/google-cloud-bigquery-reservation/SHARD_TEST.txt        | 1 +
 packages/google-cloud-bigquery-storage/SHARD_TEST.txt            | 1 +
 packages/google-cloud-bigquery/SHARD_TEST.txt                    | 1 +
 packages/google-cloud-bigtable/SHARD_TEST.txt                    | 1 +
 packages/google-cloud-container/SHARD_TEST.txt                   | 1 +
 packages/google-cloud-containeranalysis/SHARD_TEST.txt           | 1 +
 packages/google-cloud-contentwarehouse/SHARD_TEST.txt            | 1 +
 packages/google-cloud-core/SHARD_TEST.txt                        | 1 +
 packages/google-cloud-data-fusion/SHARD_TEST.txt                 | 1 +
 packages/google-cloud-data-qna/SHARD_TEST.txt                    | 1 +
 packages/google-cloud-databasecenter/SHARD_TEST.txt              | 1 +
 .../SHARD_TEST.txt                                               | 1 +
 packages/google-cloud-datacatalog-lineage/SHARD_TEST.txt         | 1 +
 packages/google-cloud-datacatalog/SHARD_TEST.txt                 | 1 +
 packages/google-cloud-dataflow-client/SHARD_TEST.txt             | 1 +
 packages/google-cloud-dataform/SHARD_TEST.txt                    | 1 +
 packages/google-cloud-datalabeling/SHARD_TEST.txt                | 1 +
 packages/google-cloud-dataplex/SHARD_TEST.txt                    | 1 +
 packages/google-cloud-dataproc-metastore/SHARD_TEST.txt          | 1 +
 packages/google-cloud-dataproc/SHARD_TEST.txt                    | 1 +
 packages/google-cloud-datastore/SHARD_TEST.txt                   | 1 +
 packages/google-cloud-datastream/SHARD_TEST.txt                  | 1 +
 packages/google-cloud-deploy/SHARD_TEST.txt                      | 1 +
 packages/google-cloud-developerconnect/SHARD_TEST.txt            | 1 +
 packages/google-cloud-essential-contacts/SHARD_TEST.txt          | 1 +
 packages/google-cloud-eventarc-publishing/SHARD_TEST.txt         | 1 +
 packages/google-cloud-eventarc/SHARD_TEST.txt                    | 1 +
 packages/google-cloud-filestore/SHARD_TEST.txt                   | 1 +
 packages/google-cloud-financialservices/SHARD_TEST.txt           | 1 +
 packages/google-cloud-firestore/SHARD_TEST.txt                   | 1 +
 packages/google-cloud-functions/SHARD_TEST.txt                   | 1 +
 packages/google-cloud-gdchardwaremanagement/SHARD_TEST.txt       | 1 +
 packages/google-cloud-geminidataanalytics/SHARD_TEST.txt         | 1 +
 packages/google-cloud-gke-backup/SHARD_TEST.txt                  | 1 +
 packages/google-cloud-gke-connect-gateway/SHARD_TEST.txt         | 1 +
 packages/google-cloud-gke-hub/SHARD_TEST.txt                     | 1 +
 packages/google-cloud-gke-multicloud/SHARD_TEST.txt              | 1 +
 packages/google-cloud-gkerecommender/SHARD_TEST.txt              | 1 +
 packages/google-cloud-gsuiteaddons/SHARD_TEST.txt                | 1 +
 packages/google-cloud-hypercomputecluster/SHARD_TEST.txt         | 1 +
 packages/google-cloud-iam-logging/SHARD_TEST.txt                 | 1 +
 packages/google-cloud-iam/SHARD_TEST.txt                         | 1 +
 packages/google-cloud-iamconnectorcredentials/SHARD_TEST.txt     | 1 +
 packages/google-cloud-iap/SHARD_TEST.txt                         | 1 +
 packages/google-cloud-ids/SHARD_TEST.txt                         | 1 +
 packages/google-cloud-kms-inventory/SHARD_TEST.txt               | 1 +
 packages/google-cloud-kms/SHARD_TEST.txt                         | 1 +
 packages/google-cloud-pubsub/SHARD_TEST.txt                      | 1 +
 packages/google-cloud-quotas/SHARD_TEST.txt                      | 1 +
 packages/google-cloud-rapidmigrationassessment/SHARD_TEST.txt    | 1 +
 packages/google-cloud-recaptcha-enterprise/SHARD_TEST.txt        | 1 +
 packages/google-cloud-recommendations-ai/SHARD_TEST.txt          | 1 +
 packages/google-cloud-recommender/SHARD_TEST.txt                 | 1 +
 packages/google-cloud-redis-cluster/SHARD_TEST.txt               | 1 +
 packages/google-cloud-redis/SHARD_TEST.txt                       | 1 +
 packages/google-cloud-resource-manager/SHARD_TEST.txt            | 1 +
 packages/google-cloud-retail/SHARD_TEST.txt                      | 1 +
 packages/google-cloud-run/SHARD_TEST.txt                         | 1 +
 packages/sqlalchemy-bigquery/SHARD_TEST.txt                      | 1 +
 packages/sqlalchemy-spanner/SHARD_TEST.txt                       | 1 +
 86 files changed, 86 insertions(+)
 create mode 100644 packages/django-google-spanner/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-appoptimize/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-artifact-registry/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-asset/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-assured-workloads/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-audit-log/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-auditmanager/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-automl/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-backupdr/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-bare-metal-solution/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-batch/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-beyondcorp-appconnections/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-beyondcorp-appconnectors/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-beyondcorp-appgateways/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-beyondcorp-clientconnectorservices/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-beyondcorp-clientgateways/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-biglake-hive/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-biglake/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-bigquery-analyticshub/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-bigquery-biglake/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-bigquery-connection/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-bigquery-data-exchange/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-bigquery-datapolicies/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-bigquery-datatransfer/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-bigquery-logging/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-bigquery-migration/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-bigquery-reservation/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-bigquery-storage/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-bigquery/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-bigtable/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-container/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-containeranalysis/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-contentwarehouse/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-core/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-data-fusion/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-data-qna/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-databasecenter/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-datacatalog-lineage-configmanagement/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-datacatalog-lineage/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-datacatalog/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-dataflow-client/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-dataform/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-datalabeling/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-dataplex/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-dataproc-metastore/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-dataproc/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-datastore/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-datastream/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-deploy/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-developerconnect/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-essential-contacts/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-eventarc-publishing/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-eventarc/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-filestore/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-financialservices/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-firestore/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-functions/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-gdchardwaremanagement/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-geminidataanalytics/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-gke-backup/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-gke-connect-gateway/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-gke-hub/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-gke-multicloud/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-gkerecommender/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-gsuiteaddons/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-hypercomputecluster/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-iam-logging/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-iam/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-iamconnectorcredentials/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-iap/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-ids/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-kms-inventory/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-kms/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-pubsub/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-quotas/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-rapidmigrationassessment/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-recaptcha-enterprise/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-recommendations-ai/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-recommender/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-redis-cluster/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-redis/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-resource-manager/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-retail/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-run/SHARD_TEST.txt
 create mode 100644 packages/sqlalchemy-bigquery/SHARD_TEST.txt
 create mode 100644 packages/sqlalchemy-spanner/SHARD_TEST.txt

diff --git a/packages/django-google-spanner/SHARD_TEST.txt b/packages/django-google-spanner/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/django-google-spanner/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-appoptimize/SHARD_TEST.txt b/packages/google-cloud-appoptimize/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-appoptimize/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-artifact-registry/SHARD_TEST.txt b/packages/google-cloud-artifact-registry/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-artifact-registry/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-asset/SHARD_TEST.txt b/packages/google-cloud-asset/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-asset/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-assured-workloads/SHARD_TEST.txt b/packages/google-cloud-assured-workloads/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-assured-workloads/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-audit-log/SHARD_TEST.txt b/packages/google-cloud-audit-log/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-audit-log/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-auditmanager/SHARD_TEST.txt b/packages/google-cloud-auditmanager/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-auditmanager/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-automl/SHARD_TEST.txt b/packages/google-cloud-automl/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-automl/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-backupdr/SHARD_TEST.txt b/packages/google-cloud-backupdr/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-backupdr/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-bare-metal-solution/SHARD_TEST.txt b/packages/google-cloud-bare-metal-solution/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-bare-metal-solution/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-batch/SHARD_TEST.txt b/packages/google-cloud-batch/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-batch/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-beyondcorp-appconnections/SHARD_TEST.txt b/packages/google-cloud-beyondcorp-appconnections/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-beyondcorp-appconnections/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-beyondcorp-appconnectors/SHARD_TEST.txt b/packages/google-cloud-beyondcorp-appconnectors/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-beyondcorp-appconnectors/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-beyondcorp-appgateways/SHARD_TEST.txt b/packages/google-cloud-beyondcorp-appgateways/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-beyondcorp-appgateways/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-beyondcorp-clientconnectorservices/SHARD_TEST.txt b/packages/google-cloud-beyondcorp-clientconnectorservices/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-beyondcorp-clientconnectorservices/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-beyondcorp-clientgateways/SHARD_TEST.txt b/packages/google-cloud-beyondcorp-clientgateways/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-beyondcorp-clientgateways/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-biglake-hive/SHARD_TEST.txt b/packages/google-cloud-biglake-hive/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-biglake-hive/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-biglake/SHARD_TEST.txt b/packages/google-cloud-biglake/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-biglake/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-bigquery-analyticshub/SHARD_TEST.txt b/packages/google-cloud-bigquery-analyticshub/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-bigquery-analyticshub/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-bigquery-biglake/SHARD_TEST.txt b/packages/google-cloud-bigquery-biglake/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-bigquery-biglake/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-bigquery-connection/SHARD_TEST.txt b/packages/google-cloud-bigquery-connection/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-bigquery-connection/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-bigquery-data-exchange/SHARD_TEST.txt b/packages/google-cloud-bigquery-data-exchange/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-bigquery-data-exchange/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-bigquery-datapolicies/SHARD_TEST.txt b/packages/google-cloud-bigquery-datapolicies/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-bigquery-datapolicies/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-bigquery-datatransfer/SHARD_TEST.txt b/packages/google-cloud-bigquery-datatransfer/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-bigquery-datatransfer/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-bigquery-logging/SHARD_TEST.txt b/packages/google-cloud-bigquery-logging/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-bigquery-logging/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-bigquery-migration/SHARD_TEST.txt b/packages/google-cloud-bigquery-migration/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-bigquery-migration/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-bigquery-reservation/SHARD_TEST.txt b/packages/google-cloud-bigquery-reservation/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-bigquery-reservation/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-bigquery-storage/SHARD_TEST.txt b/packages/google-cloud-bigquery-storage/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-bigquery-storage/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-bigquery/SHARD_TEST.txt b/packages/google-cloud-bigquery/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-bigquery/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-bigtable/SHARD_TEST.txt b/packages/google-cloud-bigtable/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-bigtable/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-container/SHARD_TEST.txt b/packages/google-cloud-container/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-container/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-containeranalysis/SHARD_TEST.txt b/packages/google-cloud-containeranalysis/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-containeranalysis/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-contentwarehouse/SHARD_TEST.txt b/packages/google-cloud-contentwarehouse/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-contentwarehouse/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-core/SHARD_TEST.txt b/packages/google-cloud-core/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-core/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-data-fusion/SHARD_TEST.txt b/packages/google-cloud-data-fusion/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-data-fusion/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-data-qna/SHARD_TEST.txt b/packages/google-cloud-data-qna/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-data-qna/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-databasecenter/SHARD_TEST.txt b/packages/google-cloud-databasecenter/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-databasecenter/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-datacatalog-lineage-configmanagement/SHARD_TEST.txt b/packages/google-cloud-datacatalog-lineage-configmanagement/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-datacatalog-lineage-configmanagement/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-datacatalog-lineage/SHARD_TEST.txt b/packages/google-cloud-datacatalog-lineage/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-datacatalog-lineage/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-datacatalog/SHARD_TEST.txt b/packages/google-cloud-datacatalog/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-datacatalog/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-dataflow-client/SHARD_TEST.txt b/packages/google-cloud-dataflow-client/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-dataflow-client/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-dataform/SHARD_TEST.txt b/packages/google-cloud-dataform/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-dataform/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-datalabeling/SHARD_TEST.txt b/packages/google-cloud-datalabeling/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-datalabeling/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-dataplex/SHARD_TEST.txt b/packages/google-cloud-dataplex/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-dataplex/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-dataproc-metastore/SHARD_TEST.txt b/packages/google-cloud-dataproc-metastore/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-dataproc-metastore/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-dataproc/SHARD_TEST.txt b/packages/google-cloud-dataproc/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-dataproc/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-datastore/SHARD_TEST.txt b/packages/google-cloud-datastore/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-datastore/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-datastream/SHARD_TEST.txt b/packages/google-cloud-datastream/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-datastream/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-deploy/SHARD_TEST.txt b/packages/google-cloud-deploy/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-deploy/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-developerconnect/SHARD_TEST.txt b/packages/google-cloud-developerconnect/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-developerconnect/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-essential-contacts/SHARD_TEST.txt b/packages/google-cloud-essential-contacts/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-essential-contacts/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-eventarc-publishing/SHARD_TEST.txt b/packages/google-cloud-eventarc-publishing/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-eventarc-publishing/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-eventarc/SHARD_TEST.txt b/packages/google-cloud-eventarc/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-eventarc/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-filestore/SHARD_TEST.txt b/packages/google-cloud-filestore/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-filestore/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-financialservices/SHARD_TEST.txt b/packages/google-cloud-financialservices/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-financialservices/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-firestore/SHARD_TEST.txt b/packages/google-cloud-firestore/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-firestore/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-functions/SHARD_TEST.txt b/packages/google-cloud-functions/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-functions/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-gdchardwaremanagement/SHARD_TEST.txt b/packages/google-cloud-gdchardwaremanagement/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-gdchardwaremanagement/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-geminidataanalytics/SHARD_TEST.txt b/packages/google-cloud-geminidataanalytics/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-geminidataanalytics/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-gke-backup/SHARD_TEST.txt b/packages/google-cloud-gke-backup/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-gke-backup/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-gke-connect-gateway/SHARD_TEST.txt b/packages/google-cloud-gke-connect-gateway/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-gke-connect-gateway/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-gke-hub/SHARD_TEST.txt b/packages/google-cloud-gke-hub/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-gke-hub/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-gke-multicloud/SHARD_TEST.txt b/packages/google-cloud-gke-multicloud/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-gke-multicloud/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-gkerecommender/SHARD_TEST.txt b/packages/google-cloud-gkerecommender/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-gkerecommender/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-gsuiteaddons/SHARD_TEST.txt b/packages/google-cloud-gsuiteaddons/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-gsuiteaddons/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-hypercomputecluster/SHARD_TEST.txt b/packages/google-cloud-hypercomputecluster/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-hypercomputecluster/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-iam-logging/SHARD_TEST.txt b/packages/google-cloud-iam-logging/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-iam-logging/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-iam/SHARD_TEST.txt b/packages/google-cloud-iam/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-iam/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-iamconnectorcredentials/SHARD_TEST.txt b/packages/google-cloud-iamconnectorcredentials/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-iamconnectorcredentials/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-iap/SHARD_TEST.txt b/packages/google-cloud-iap/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-iap/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-ids/SHARD_TEST.txt b/packages/google-cloud-ids/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-ids/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-kms-inventory/SHARD_TEST.txt b/packages/google-cloud-kms-inventory/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-kms-inventory/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-kms/SHARD_TEST.txt b/packages/google-cloud-kms/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-kms/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-pubsub/SHARD_TEST.txt b/packages/google-cloud-pubsub/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-pubsub/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-quotas/SHARD_TEST.txt b/packages/google-cloud-quotas/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-quotas/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-rapidmigrationassessment/SHARD_TEST.txt b/packages/google-cloud-rapidmigrationassessment/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-rapidmigrationassessment/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-recaptcha-enterprise/SHARD_TEST.txt b/packages/google-cloud-recaptcha-enterprise/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-recaptcha-enterprise/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-recommendations-ai/SHARD_TEST.txt b/packages/google-cloud-recommendations-ai/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-recommendations-ai/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-recommender/SHARD_TEST.txt b/packages/google-cloud-recommender/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-recommender/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-redis-cluster/SHARD_TEST.txt b/packages/google-cloud-redis-cluster/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-redis-cluster/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-redis/SHARD_TEST.txt b/packages/google-cloud-redis/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-redis/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-resource-manager/SHARD_TEST.txt b/packages/google-cloud-resource-manager/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-resource-manager/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-retail/SHARD_TEST.txt b/packages/google-cloud-retail/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-retail/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-run/SHARD_TEST.txt b/packages/google-cloud-run/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-run/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/sqlalchemy-bigquery/SHARD_TEST.txt b/packages/sqlalchemy-bigquery/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/sqlalchemy-bigquery/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/sqlalchemy-spanner/SHARD_TEST.txt b/packages/sqlalchemy-spanner/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/sqlalchemy-spanner/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding

From 02c765377413efc0cc9b807f411f157ff03439e7 Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Thu, 11 Jun 2026 17:25:15 -0700
Subject: [PATCH 04/84] added sharding test file

---
 ci/get_package_shards.py | 94 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 94 insertions(+)
 create mode 100644 ci/get_package_shards.py

diff --git a/ci/get_package_shards.py b/ci/get_package_shards.py
new file mode 100644
index 000000000000..25bec5703efb
--- /dev/null
+++ b/ci/get_package_shards.py
@@ -0,0 +1,94 @@
+import os
+import subprocess
+import json
+import math
+import sys
+
+def get_packages():
+    subdirs = ['packages']
+    packages = []
+    for subdir in subdirs:
+        if not os.path.exists(subdir):
+            continue
+        # Use the same sorting as the shell script
+        pkg_dirs = [os.path.join(subdir, d) + '/' for d in os.listdir(subdir) if os.path.isdir(os.path.join(subdir, d))]
+        packages.extend(sorted(pkg_dirs))
+    return packages
+
+def get_packages_to_test():
+    build_type = os.environ.get('BUILD_TYPE', 'presubmit')
+    target_branch = os.environ.get('TARGET_BRANCH', 'main')
+    
+    all_packages = get_packages()
+    
+    if build_type == 'presubmit':
+        git_diff_arg = f"origin/{target_branch}..."
+    elif build_type == 'continuous':
+        git_diff_arg = "HEAD~.."
+    else:
+        return all_packages
+
+    # Check if ci/ changed
+    try:
+        subprocess.check_call(['git', 'diff', '--quiet', git_diff_arg, 'ci'])
+        ci_changed = False
+    except subprocess.CalledProcessError:
+        ci_changed = True
+    
+    if ci_changed:
+        return all_packages
+    
+    try:
+        res = subprocess.check_output(['git', 'diff', '--name-only', git_diff_arg]).decode('utf-8')
+        changed_files = res.splitlines()
+    except subprocess.CalledProcessError:
+        return all_packages
+    
+    to_test = []
+    for pkg in all_packages:
+        # Check if any changed file starts with the package path
+        if any(f.startswith(pkg) for f in changed_files):
+            to_test.append(pkg)
+    
+    return to_test
+
+def group_packages(packages, max_packages_per_shard=25, max_total_shards=20):
+    if not packages:
+        return []
+    
+    num_packages = len(packages)
+    
+    # Calculate number of shards based on packages per shard
+    num_shards = math.ceil(num_packages / max_packages_per_shard)
+    
+    # Cap the total number of shards
+    num_shards = min(num_shards, max_total_shards)
+    
+    # Recalculate shard size to be as even as possible given the capped shards
+    shard_size = math.ceil(num_packages / num_shards)
+    
+    shards = []
+    for i in range(num_shards):
+        start = i * shard_size
+        end = min((i + 1) * shard_size, num_packages)
+        if start >= num_packages:
+            break
+        
+        shard_packages = packages[start:end]
+        if len(shard_packages) == 1:
+            name = shard_packages[0].strip('/').split('/')[-1]
+        else:
+            name = f"{shard_packages[0].strip('/').split('/')[-1]}...{shard_packages[-1].strip('/').split('/')[-1]}"
+        
+        shards.append({
+            "name": name,
+            "index": i + 1,
+            "packages": " ".join(shard_packages)
+        })
+    return shards
+
+if __name__ == "__main__":
+    packages = get_packages_to_test()
+    # Shard into groups of ~25 libraries, up to 20 parallel jobs
+    shards = group_packages(packages, max_packages_per_shard=25, max_total_shards=20)
+    print(json.dumps(shards))

From 3c8d0577061dfab9008afffd39f8f58a6f1556b1 Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Thu, 11 Jun 2026 17:29:10 -0700
Subject: [PATCH 05/84] added unit-complete to gather all shards

---
 .github/workflows/unittest.yml | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml
index 0c7882624447..49ede47ce769 100644
--- a/.github/workflows/unittest.yml
+++ b/.github/workflows/unittest.yml
@@ -79,11 +79,24 @@ jobs:
         name: coverage-artifact-${{ matrix.python }}-${{ matrix.package_shard.index }}
         path: .coverage-${{ matrix.python }}
 
+  unit-complete:
+    needs: [discover, unit]
+    if: always()
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check unit test results
+        run: |
+          if [[ "${{ needs.unit.result }}" != "success" && "${{ needs.unit.result }}" != "skipped" ]]; then
+            echo "Unit tests failed"
+            exit 1
+          fi
+          echo "All unit tests passed or were skipped"
+
   cover:
     runs-on: ubuntu-latest
     needs:
         - unit
-    steps:
+    ...
     - name: Checkout
       uses: actions/checkout@v4
       # Use a fetch-depth of 2 to avoid error `fatal: origin/main...HEAD: no merge base`

From 73ab0915ed3d3f22e90ef398e59e1a27b5508698 Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Thu, 11 Jun 2026 17:31:58 -0700
Subject: [PATCH 06/84] changed shard params

---
 ci/get_package_shards.py | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/ci/get_package_shards.py b/ci/get_package_shards.py
index 25bec5703efb..880bc689d29b 100644
--- a/ci/get_package_shards.py
+++ b/ci/get_package_shards.py
@@ -52,7 +52,7 @@ def get_packages_to_test():
     
     return to_test
 
-def group_packages(packages, max_packages_per_shard=25, max_total_shards=20):
+def group_packages(packages, max_packages_per_shard=50, max_total_shards=10):
     if not packages:
         return []
     
@@ -75,20 +75,18 @@ def group_packages(packages, max_packages_per_shard=25, max_total_shards=20):
             break
         
         shard_packages = packages[start:end]
-        if len(shard_packages) == 1:
-            name = shard_packages[0].strip('/').split('/')[-1]
-        else:
-            name = f"{shard_packages[0].strip('/').split('/')[-1]}...{shard_packages[-1].strip('/').split('/')[-1]}"
+        index = i + 1
+        name = f"Shard {index}"
         
         shards.append({
             "name": name,
-            "index": i + 1,
+            "index": index,
             "packages": " ".join(shard_packages)
         })
     return shards
 
 if __name__ == "__main__":
     packages = get_packages_to_test()
-    # Shard into groups of ~25 libraries, up to 20 parallel jobs
-    shards = group_packages(packages, max_packages_per_shard=25, max_total_shards=20)
+    # Shard into groups of ~50 libraries, up to 10 parallel jobs
+    shards = group_packages(packages, max_packages_per_shard=50, max_total_shards=10)
     print(json.dumps(shards))

From 74f1ef7d22f12d93ee2712b91e374f0af9406db7 Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Thu, 11 Jun 2026 17:36:13 -0700
Subject: [PATCH 07/84] fixed tests

---
 .github/workflows/unittest.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml
index 49ede47ce769..a30ea6c684f3 100644
--- a/.github/workflows/unittest.yml
+++ b/.github/workflows/unittest.yml
@@ -33,9 +33,9 @@ jobs:
         TARGET_BRANCH: ${{ github.base_ref || github.event.merge_group.base_ref }}
       run: |
         if [ -n "$TARGET_BRANCH" ]; then
-          git fetch origin "$TARGET_BRANCH" --deepen=200
+          git fetch origin "$TARGET_BRANCH" --deepen=200 || true
         fi
-        echo "matrix=$(python ci/get_package_shards.py)" >> $GITHUB_OUTPUT
+        echo "matrix=$(python3 ci/get_package_shards.py)" >> $GITHUB_OUTPUT
 
   unit:
     needs: discover
@@ -96,7 +96,7 @@ jobs:
     runs-on: ubuntu-latest
     needs:
         - unit
-    ...
+    steps:
     - name: Checkout
       uses: actions/checkout@v4
       # Use a fetch-depth of 2 to avoid error `fatal: origin/main...HEAD: no merge base`

From a450254f883140d649ec98cad9fb61ce453c6ef5 Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Thu, 11 Jun 2026 19:01:06 -0700
Subject: [PATCH 08/84] fix coverage

---
 .github/workflows/unittest.yml | 3 +--
 ci/run_conditional_tests.sh    | 7 +++++++
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml
index a30ea6c684f3..1de144540dd6 100644
--- a/.github/workflows/unittest.yml
+++ b/.github/workflows/unittest.yml
@@ -65,7 +65,6 @@ jobs:
         python -m pip install nox
     - name: Run unit tests
       env:
-        COVERAGE_FILE: ${{ github.workspace }}/.coverage-${{ matrix.python }}
         BUILD_TYPE: presubmit
         TARGET_BRANCH: ${{ github.base_ref || github.event.merge_group.base_ref }}
         TEST_TYPE: unit
@@ -77,7 +76,7 @@ jobs:
       uses: actions/upload-artifact@v4
       with:
         name: coverage-artifact-${{ matrix.python }}-${{ matrix.package_shard.index }}
-        path: .coverage-${{ matrix.python }}
+        path: .coverage.${{ matrix.python }}.*
 
   unit-complete:
     needs: [discover, unit]
diff --git a/ci/run_conditional_tests.sh b/ci/run_conditional_tests.sh
index 12b02c127ae9..e047b49cf365 100755
--- a/ci/run_conditional_tests.sh
+++ b/ci/run_conditional_tests.sh
@@ -140,6 +140,13 @@ for subdir in ${subdirs[@]}; do
             pushd ${d}
             # Temporarily allow failure.
             set +e
+            
+            # Ensure unique coverage file per package to avoid DataError
+            # when combining statement and branch coverage.
+            # Strip trailing slash from directory name for the filename.
+            pkg_name_clean=$(echo ${d} | sed 's|/$||' | sed 's|/|_|g')
+            export COVERAGE_FILE="${PROJECT_ROOT}/.coverage.${PY_VERSION}.${pkg_name_clean}"
+            
             ${test_script}
             ret=$?
             set -e

From af4d10c45b1de758d51eb56e3089ec7e22676965 Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Thu, 11 Jun 2026 21:33:04 -0700
Subject: [PATCH 09/84] fix system tests

---
 .kokoro/system.sh | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/.kokoro/system.sh b/.kokoro/system.sh
index 52550987340b..6aedd9885e70 100755
--- a/.kokoro/system.sh
+++ b/.kokoro/system.sh
@@ -119,14 +119,21 @@ handle_finished_job() {
   # so we ignore its exit code and use the resfile.
   wait "$pid" 2>/dev/null || true
   
-  local res=$(cat "$resfile")
-  rm "$resfile"
+  local res=1
+  if [ -f "$resfile" ]; then
+    res=$(cat "$resfile")
+    rm "$resfile"
+  fi
   
   echo "------------------------------------------------------------"
   echo "System tests for ${pkg} finished (Exit code: ${res})"
   echo "------------------------------------------------------------"
-  cat "$log"
-  rm "$log"
+  if [ -f "$log" ]; then
+    cat "$log"
+    rm "$log"
+  else
+    echo "Log file missing for ${pkg}"
+  fi
   
   if [ "${res}" -ne 0 ]; then
     RETVAL=${res}
@@ -185,7 +192,9 @@ for path in `find 'packages' \
   if [[ "${package_modified}" -gt 0 || "$KOKORO_BUILD_ARTIFACTS_SUBDIR" == *"continuous"* ]]; then
       # Wait if we have reached MAX_PARALLEL
       while [[ ${#running_pids[@]} -ge $MAX_PARALLEL ]]; do
+        set +e
         wait -n
+        set -e
         # Find which job finished
         new_pids=()
         for pid in "${running_pids[@]}"; do

From 15916dfd86f16f8b1ce71a8c9bbe2b92bb11d714 Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Fri, 12 Jun 2026 09:26:18 -0700
Subject: [PATCH 10/84] split out system test logs

---
 .kokoro/continuous/common.cfg |  1 +
 .kokoro/presubmit/common.cfg  |  1 +
 .kokoro/system-single.sh      |  8 +++++++-
 .kokoro/system.sh             | 30 ++++++++++++++++--------------
 4 files changed, 25 insertions(+), 15 deletions(-)

diff --git a/.kokoro/continuous/common.cfg b/.kokoro/continuous/common.cfg
index 175a4e8fd7a2..36cd039de187 100644
--- a/.kokoro/continuous/common.cfg
+++ b/.kokoro/continuous/common.cfg
@@ -4,6 +4,7 @@
 action {
   define_artifacts {
     regex: "**/*sponge_log.xml"
+    regex: "**/*sponge_log.log"
   }
 }
 
diff --git a/.kokoro/presubmit/common.cfg b/.kokoro/presubmit/common.cfg
index 175a4e8fd7a2..36cd039de187 100644
--- a/.kokoro/presubmit/common.cfg
+++ b/.kokoro/presubmit/common.cfg
@@ -4,6 +4,7 @@
 action {
   define_artifacts {
     regex: "**/*sponge_log.xml"
+    regex: "**/*sponge_log.log"
   }
 }
 
diff --git a/.kokoro/system-single.sh b/.kokoro/system-single.sh
index 0ec5ae7ebf1b..b75936bda820 100755
--- a/.kokoro/system-single.sh
+++ b/.kokoro/system-single.sh
@@ -32,4 +32,10 @@ NOX_FILE_ARG=""
 
 [[ -z "${NOX_FILE}" ]] || NOX_FILE_ARG="-f ${NOX_FILE}"
 
-python3 -m nox ${NOX_SESSION_ARG} $NOX_FILE_ARG
+NOX_ARGS=()
+if [[ -n "${XML_OUTPUT_FILE}" ]]; then
+  # Pass --junitxml as a positional argument to the underlying pytest call
+  NOX_ARGS+=("--junitxml=${XML_OUTPUT_FILE}")
+fi
+
+python3 -m nox ${NOX_SESSION_ARG} $NOX_FILE_ARG -- "${NOX_ARGS[@]}"
diff --git a/.kokoro/system.sh b/.kokoro/system.sh
index 6aedd9885e70..2ae5b4daea22 100755
--- a/.kokoro/system.sh
+++ b/.kokoro/system.sh
@@ -38,7 +38,15 @@ pwd
 run_package_test() {
   local package_name=$1
   local package_path="packages/${package_name}"
-  
+
+  # Create a dedicated directory for this package's artifacts
+  local artifact_dir="${KOKORO_ARTIFACTS_DIR}/${package_name}"
+  mkdir -p "${artifact_dir}"
+
+  # Define standard output paths for logs and XML results
+  export XML_OUTPUT_FILE="${artifact_dir}/sponge_log.xml"
+  local log_file="${artifact_dir}/sponge_log.log"
+
   # Isolate gcloud config for parallel execution
   export CLOUDSDK_CONFIG=$(mktemp -d)
 
@@ -51,6 +59,8 @@ run_package_test() {
 
   echo "------------------------------------------------------------"
   echo "Configuring environment for: ${package_name}"
+  echo "Log file: ${log_file}"
+  echo "XML results: ${XML_OUTPUT_FILE}"
   echo "------------------------------------------------------------"
 
   case "${package_name}" in
@@ -85,11 +95,12 @@ run_package_test() {
   # Run the actual test
   pushd "${package_path}" > /dev/null
   set +e
-  "${system_test_script}"
+  # *** CRITICAL: system-single.sh MUST use XML_OUTPUT_FILE to generate the JUnit XML ***
+  "${system_test_script}" > "${log_file}" 2>&1
   local res=$?
   set -e
   popd > /dev/null
-  
+
   # Clean up isolated gcloud config
   rm -rf "${CLOUDSDK_CONFIG}"
 
@@ -103,7 +114,6 @@ system_test_script="${PROJECT_ROOT}/.kokoro/system-single.sh"
 MAX_PARALLEL=4
 running_pids=()
 declare -A pid_to_pkg
-declare -A pid_to_log
 declare -A pid_to_resfile
 
 # Array to keep track of results for the final summary
@@ -112,7 +122,6 @@ results=()
 handle_finished_job() {
   local pid=$1
   local pkg=${pid_to_pkg[$pid]}
-  local log=${pid_to_log[$pid]}
   local resfile=${pid_to_resfile[$pid]}
   
   # wait $pid might fail if it was already reaped by wait -n, 
@@ -127,13 +136,8 @@ handle_finished_job() {
   
   echo "------------------------------------------------------------"
   echo "System tests for ${pkg} finished (Exit code: ${res})"
+  echo "Artifacts in: ${KOKORO_ARTIFACTS_DIR}/${pkg}"
   echo "------------------------------------------------------------"
-  if [ -f "$log" ]; then
-    cat "$log"
-    rm "$log"
-  else
-    echo "Log file missing for ${pkg}"
-  fi
   
   if [ "${res}" -ne 0 ]; then
     RETVAL=${res}
@@ -208,16 +212,14 @@ for path in `find 'packages' \
       done
 
       # Start the next test in the background
-      log_file=$(mktemp)
       res_file=$(mktemp)
       (
-        run_package_test "$package_name" > "$log_file" 2>&1
+        run_package_test "$package_name"
         echo $? > "$res_file"
       ) &
       pid=$!
       running_pids+=($pid)
       pid_to_pkg[$pid]=$package_name
-      pid_to_log[$pid]=$log_file
       pid_to_resfile[$pid]=$res_file
       echo "Started system tests for ${package_name} (PID: ${pid})"
   else

From 49cac0fe1f782ef8b88601b635fdc7722a7855bf Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Fri, 12 Jun 2026 10:35:12 -0700
Subject: [PATCH 11/84] attempt fix for lint

---
 ci/run_conditional_tests.sh | 56 ++++++++++++++-----------------------
 1 file changed, 21 insertions(+), 35 deletions(-)

diff --git a/ci/run_conditional_tests.sh b/ci/run_conditional_tests.sh
index e047b49cf365..e8bf7659888f 100755
--- a/ci/run_conditional_tests.sh
+++ b/ci/run_conditional_tests.sh
@@ -47,41 +47,6 @@ git config --global url."${PROJECT_ROOT}".insteadOf "https://github.com/googleap
 # A script file for running the test in a sub project.
 test_script="${PROJECT_ROOT}/ci/run_single_test.sh"
 
-if [[ ${BUILD_TYPE} == "presubmit" ]]; then
-    # For presubmit build, we want to know the difference from the
-    # common commit in the target branch.
-    GIT_DIFF_ARG="origin/$TARGET_BRANCH..."
-
-    # Then fetch enough history for finding the common commit.
-    git fetch origin "$TARGET_BRANCH" --deepen=200
-
-elif [[ ${BUILD_TYPE} == "continuous" ]]; then
-    # For continuous build, we want to know the difference in the last
-    # commit. This assumes we use squash commit when merging PRs.
-    GIT_DIFF_ARG="HEAD~.."
-
-    # Then fetch one last commit for getting the diff.
-    git fetch origin "$TARGET_BRANCH" --deepen=1
-
-else
-    # Run everything.
-    GIT_DIFF_ARG=""
-fi
-
-# Then detect changes in the test scripts.
-
-set +e
-git diff --quiet ${GIT_DIFF_ARG} ci
-changed=$?
-set -e
-
-# Now we have a fixed list, but we can change it to autodetect if
-# necessary.
-
-subdirs=(
-    packages
-)
-
 if [ -n "${PACKAGE_LIST}" ]; then
     echo "Using provided PACKAGE_LIST"
     to_test=(${PACKAGE_LIST})
@@ -101,6 +66,27 @@ if [ -n "${PACKAGE_LIST}" ]; then
     exit ${RETVAL}
 fi
 
+if [[ ${BUILD_TYPE} == "presubmit" ]]; then
+    # For presubmit build, we want to know the difference from the
+    # common commit in the target branch.
+    GIT_DIFF_ARG="origin/$TARGET_BRANCH..."
+
+    # Then fetch enough history for finding the common commit.
+    git fetch origin "$TARGET_BRANCH" --deepen=200 || true
+
+elif [[ ${BUILD_TYPE} == "continuous" ]]; then
+    # For continuous build, we want to know the difference in the last
+    # commit. This assumes we use squash commit when merging PRs.
+    GIT_DIFF_ARG="HEAD~.."
+
+    # Then fetch one last commit for getting the diff.
+    git fetch origin "$TARGET_BRANCH" --deepen=1 || true
+
+else
+    # Run everything.
+    GIT_DIFF_ARG=""
+fi
+
 # Sharding logic (fallback for manual runs)
 TOTAL_SHARDS="${TOTAL_SHARDS:-1}"
 SHARD_INDEX="${SHARD_INDEX:-1}"

From d8dd522a1557ddc070a95ed01bbcbe65bbb7b5a4 Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Fri, 12 Jun 2026 10:38:34 -0700
Subject: [PATCH 12/84] update system tests to show logs for each target

---
 .kokoro/system.sh | 26 +++++++++++++++++---------
 1 file changed, 17 insertions(+), 9 deletions(-)

diff --git a/.kokoro/system.sh b/.kokoro/system.sh
index 2ae5b4daea22..d207d917dce6 100755
--- a/.kokoro/system.sh
+++ b/.kokoro/system.sh
@@ -109,6 +109,8 @@ run_package_test() {
 
 # A file for running system tests
 system_test_script="${PROJECT_ROOT}/.kokoro/system-single.sh"
+# Ensure KOKORO_ARTIFACTS_DIR is set, fallback to a local directory
+export KOKORO_ARTIFACTS_DIR="${KOKORO_ARTIFACTS_DIR:-${PROJECT_ROOT}/.artifacts}"
 
 # Parallel execution settings
 MAX_PARALLEL=4
@@ -123,24 +125,26 @@ handle_finished_job() {
   local pid=$1
   local pkg=${pid_to_pkg[$pid]}
   local resfile=${pid_to_resfile[$pid]}
-  
+
   # wait $pid might fail if it was already reaped by wait -n, 
   # so we ignore its exit code and use the resfile.
   wait "$pid" 2>/dev/null || true
-  
+
   local res=1
-  if [ -f "$resfile" ]; then
+  if [ -s "$resfile" ]; then
     res=$(cat "$resfile")
     rm "$resfile"
+  else
+    # If the file is empty or missing, the subshell crashed early
+    res=1
   fi
-  
+
   echo "------------------------------------------------------------"
   echo "System tests for ${pkg} finished (Exit code: ${res})"
   echo "Artifacts in: ${KOKORO_ARTIFACTS_DIR}/${pkg}"
   echo "------------------------------------------------------------"
-  
-  if [ "${res}" -ne 0 ]; then
-    RETVAL=${res}
+  if [ -z "${res}" ] || [ "${res}" -ne 0 ]; then
+    RETVAL=1
     results+=("${pkg}: FAILED")
   else
     results+=("${pkg}: PASSED")
@@ -214,8 +218,12 @@ for path in `find 'packages' \
       # Start the next test in the background
       res_file=$(mktemp)
       (
-        run_package_test "$package_name"
-        echo $? > "$res_file"
+        if run_package_test "$package_name"; then
+          echo 0 > "$res_file"
+        else
+          res=$?
+          echo $res > "$res_file"
+        fi
       ) &
       pid=$!
       running_pids+=($pid)

From de74a685007361f9e100253f768015a6c6e190a6 Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Fri, 12 Jun 2026 10:42:56 -0700
Subject: [PATCH 13/84] update sharding logic

---
 .github/workflows/unittest.yml |  2 +-
 ci/get_package_shards.py       | 22 +++++++++++++---------
 2 files changed, 14 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml
index 1de144540dd6..9c349028e862 100644
--- a/.github/workflows/unittest.yml
+++ b/.github/workflows/unittest.yml
@@ -46,7 +46,7 @@ jobs:
       matrix:
         python: ['3.9', '3.10', "3.11", "3.12", "3.13", "3.14"]
         package_shard: ${{ fromJson(needs.discover.outputs.matrix) }}
-    name: unit (${{ matrix.package_shard.name }}, ${{ matrix.python }})
+    name: ${{ matrix.package_shard.is_sharded && format('unit ({0}, {1})', matrix.python, matrix.package_shard.name) || format('unit ({0})', matrix.python) }}
     steps:
     - name: Checkout
       uses: actions/checkout@v4
diff --git a/ci/get_package_shards.py b/ci/get_package_shards.py
index 880bc689d29b..c5b8a03f8de7 100644
--- a/ci/get_package_shards.py
+++ b/ci/get_package_shards.py
@@ -52,19 +52,23 @@ def get_packages_to_test():
     
     return to_test
 
-def group_packages(packages, max_packages_per_shard=50, max_total_shards=10):
+def group_packages(packages):
     if not packages:
         return []
     
     num_packages = len(packages)
     
-    # Calculate number of shards based on packages per shard
-    num_shards = math.ceil(num_packages / max_packages_per_shard)
+    # 1. Only shard if > 10 packages are being touched
+    # 2. Only add a new shard if we'd have > 10 in each shard (meaning size >= 11)
+    num_shards = num_packages // 11
     
-    # Cap the total number of shards
-    num_shards = min(num_shards, max_total_shards)
+    # Ensure at least 1 shard if we have packages
+    num_shards = max(1, num_shards)
     
-    # Recalculate shard size to be as even as possible given the capped shards
+    # 3. Top out at 10 shards
+    num_shards = min(10, num_shards)
+    
+    # Distribute packages between them as evenly as possible
     shard_size = math.ceil(num_packages / num_shards)
     
     shards = []
@@ -81,12 +85,12 @@ def group_packages(packages, max_packages_per_shard=50, max_total_shards=10):
         shards.append({
             "name": name,
             "index": index,
-            "packages": " ".join(shard_packages)
+            "packages": " ".join(shard_packages),
+            "is_sharded": num_shards > 1
         })
     return shards
 
 if __name__ == "__main__":
     packages = get_packages_to_test()
-    # Shard into groups of ~50 libraries, up to 10 parallel jobs
-    shards = group_packages(packages, max_packages_per_shard=50, max_total_shards=10)
+    shards = group_packages(packages)
     print(json.dumps(shards))

From 46d158345fa4fad1d12b96177f6d6c3daa015b68 Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Fri, 12 Jun 2026 11:19:01 -0700
Subject: [PATCH 14/84] fixed lint/mypy runs

---
 ci/run_conditional_tests.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/ci/run_conditional_tests.sh b/ci/run_conditional_tests.sh
index e8bf7659888f..5236b2bec292 100755
--- a/ci/run_conditional_tests.sh
+++ b/ci/run_conditional_tests.sh
@@ -88,6 +88,9 @@ else
 fi
 
 # Sharding logic (fallback for manual runs)
+subdirs=(
+    packages
+)
 TOTAL_SHARDS="${TOTAL_SHARDS:-1}"
 SHARD_INDEX="${SHARD_INDEX:-1}"
 count=0

From 6e3531dc303bb7ae4811b3cd9d3e004897366241 Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Fri, 12 Jun 2026 11:25:24 -0700
Subject: [PATCH 15/84] removed sqlalchemy-spanner from touched packages

---
 packages/sqlalchemy-spanner/SHARD_TEST.txt | 1 -
 1 file changed, 1 deletion(-)
 delete mode 100644 packages/sqlalchemy-spanner/SHARD_TEST.txt

diff --git a/packages/sqlalchemy-spanner/SHARD_TEST.txt b/packages/sqlalchemy-spanner/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/sqlalchemy-spanner/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding

From e6167bd04f929910b111b3787cbdc52788714d35 Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Fri, 12 Jun 2026 11:43:25 -0700
Subject: [PATCH 16/84] system tests print all logs in main build log

---
 .kokoro/system.sh | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/.kokoro/system.sh b/.kokoro/system.sh
index d207d917dce6..e24b6b1a5d81 100755
--- a/.kokoro/system.sh
+++ b/.kokoro/system.sh
@@ -120,11 +120,11 @@ declare -A pid_to_resfile
 
 # Array to keep track of results for the final summary
 results=()
-
 handle_finished_job() {
   local pid=$1
   local pkg=${pid_to_pkg[$pid]}
   local resfile=${pid_to_resfile[$pid]}
+  local pkg_log="${KOKORO_ARTIFACTS_DIR}/${pkg}/sponge_log.log"
 
   # wait $pid might fail if it was already reaped by wait -n, 
   # so we ignore its exit code and use the resfile.
@@ -139,10 +139,20 @@ handle_finished_job() {
     res=1
   fi
 
-  echo "------------------------------------------------------------"
+  echo "============================================================"
   echo "System tests for ${pkg} finished (Exit code: ${res})"
   echo "Artifacts in: ${KOKORO_ARTIFACTS_DIR}/${pkg}"
-  echo "------------------------------------------------------------"
+  echo "============================================================"
+
+  if [ -f "${pkg_log}" ]; then
+    echo "--- Start of Logs for ${pkg} ---"
+    cat "${pkg_log}"
+    echo "--- End of Logs for ${pkg} ---"
+  else
+    echo "Console Fallback Warning: Log file not found at ${pkg_log}"
+  fi
+  echo ""
+
   if [ -z "${res}" ] || [ "${res}" -ne 0 ]; then
     RETVAL=1
     results+=("${pkg}: FAILED")
@@ -150,7 +160,6 @@ handle_finished_job() {
     results+=("${pkg}: PASSED")
   fi
 }
-
 # Run system tests for each package with directory packages/*/tests/system
 for path in `find 'packages' \
   \( -type d -wholename 'packages/*/tests/system' \) -o \

From fd5792d883aa05ebbaee275130ccb8d65acc64cc Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Fri, 12 Jun 2026 12:20:31 -0700
Subject: [PATCH 17/84] removed many SHARD_TEST files

---
 packages/django-google-spanner/SHARD_TEST.txt                    | 1 -
 packages/google-cloud-appoptimize/SHARD_TEST.txt                 | 1 -
 packages/google-cloud-artifact-registry/SHARD_TEST.txt           | 1 -
 packages/google-cloud-asset/SHARD_TEST.txt                       | 1 -
 packages/google-cloud-assured-workloads/SHARD_TEST.txt           | 1 -
 packages/google-cloud-audit-log/SHARD_TEST.txt                   | 1 -
 packages/google-cloud-auditmanager/SHARD_TEST.txt                | 1 -
 packages/google-cloud-automl/SHARD_TEST.txt                      | 1 -
 packages/google-cloud-backupdr/SHARD_TEST.txt                    | 1 -
 packages/google-cloud-bare-metal-solution/SHARD_TEST.txt         | 1 -
 packages/google-cloud-batch/SHARD_TEST.txt                       | 1 -
 packages/google-cloud-beyondcorp-appconnections/SHARD_TEST.txt   | 1 -
 packages/google-cloud-beyondcorp-appconnectors/SHARD_TEST.txt    | 1 -
 packages/google-cloud-beyondcorp-appgateways/SHARD_TEST.txt      | 1 -
 .../SHARD_TEST.txt                                               | 1 -
 packages/google-cloud-beyondcorp-clientgateways/SHARD_TEST.txt   | 1 -
 packages/google-cloud-biglake-hive/SHARD_TEST.txt                | 1 -
 packages/google-cloud-biglake/SHARD_TEST.txt                     | 1 -
 packages/google-cloud-bigquery-analyticshub/SHARD_TEST.txt       | 1 -
 packages/google-cloud-bigquery-biglake/SHARD_TEST.txt            | 1 -
 packages/google-cloud-bigquery-connection/SHARD_TEST.txt         | 1 -
 packages/google-cloud-bigquery-data-exchange/SHARD_TEST.txt      | 1 -
 packages/google-cloud-bigquery-datapolicies/SHARD_TEST.txt       | 1 -
 packages/google-cloud-bigquery-datatransfer/SHARD_TEST.txt       | 1 -
 packages/google-cloud-bigquery-logging/SHARD_TEST.txt            | 1 -
 packages/google-cloud-bigquery-migration/SHARD_TEST.txt          | 1 -
 packages/google-cloud-bigquery-reservation/SHARD_TEST.txt        | 1 -
 packages/google-cloud-bigquery-storage/SHARD_TEST.txt            | 1 -
 packages/google-cloud-bigquery/SHARD_TEST.txt                    | 1 -
 packages/google-cloud-bigtable/SHARD_TEST.txt                    | 1 -
 packages/google-cloud-container/SHARD_TEST.txt                   | 1 -
 packages/google-cloud-containeranalysis/SHARD_TEST.txt           | 1 -
 packages/google-cloud-contentwarehouse/SHARD_TEST.txt            | 1 -
 packages/google-cloud-core/SHARD_TEST.txt                        | 1 -
 packages/google-cloud-data-fusion/SHARD_TEST.txt                 | 1 -
 packages/google-cloud-data-qna/SHARD_TEST.txt                    | 1 -
 packages/google-cloud-databasecenter/SHARD_TEST.txt              | 1 -
 .../SHARD_TEST.txt                                               | 1 -
 packages/google-cloud-datacatalog-lineage/SHARD_TEST.txt         | 1 -
 packages/google-cloud-datacatalog/SHARD_TEST.txt                 | 1 -
 packages/google-cloud-dataflow-client/SHARD_TEST.txt             | 1 -
 packages/google-cloud-dataform/SHARD_TEST.txt                    | 1 -
 packages/google-cloud-datalabeling/SHARD_TEST.txt                | 1 -
 packages/google-cloud-dataplex/SHARD_TEST.txt                    | 1 -
 packages/google-cloud-dataproc-metastore/SHARD_TEST.txt          | 1 -
 packages/google-cloud-dataproc/SHARD_TEST.txt                    | 1 -
 packages/google-cloud-datastore/SHARD_TEST.txt                   | 1 -
 packages/google-cloud-datastream/SHARD_TEST.txt                  | 1 -
 packages/google-cloud-deploy/SHARD_TEST.txt                      | 1 -
 packages/google-cloud-developerconnect/SHARD_TEST.txt            | 1 -
 packages/google-cloud-essential-contacts/SHARD_TEST.txt          | 1 -
 packages/google-cloud-eventarc-publishing/SHARD_TEST.txt         | 1 -
 packages/google-cloud-eventarc/SHARD_TEST.txt                    | 1 -
 packages/google-cloud-filestore/SHARD_TEST.txt                   | 1 -
 packages/google-cloud-financialservices/SHARD_TEST.txt           | 1 -
 packages/google-cloud-functions/SHARD_TEST.txt                   | 1 -
 packages/google-cloud-gdchardwaremanagement/SHARD_TEST.txt       | 1 -
 packages/google-cloud-geminidataanalytics/SHARD_TEST.txt         | 1 -
 packages/google-cloud-gke-backup/SHARD_TEST.txt                  | 1 -
 packages/google-cloud-gke-connect-gateway/SHARD_TEST.txt         | 1 -
 packages/google-cloud-gke-hub/SHARD_TEST.txt                     | 1 -
 packages/google-cloud-gke-multicloud/SHARD_TEST.txt              | 1 -
 packages/google-cloud-gkerecommender/SHARD_TEST.txt              | 1 -
 packages/google-cloud-gsuiteaddons/SHARD_TEST.txt                | 1 -
 packages/google-cloud-hypercomputecluster/SHARD_TEST.txt         | 1 -
 packages/google-cloud-iam-logging/SHARD_TEST.txt                 | 1 -
 packages/google-cloud-iam/SHARD_TEST.txt                         | 1 -
 packages/google-cloud-iamconnectorcredentials/SHARD_TEST.txt     | 1 -
 packages/google-cloud-iap/SHARD_TEST.txt                         | 1 -
 packages/google-cloud-ids/SHARD_TEST.txt                         | 1 -
 packages/google-cloud-kms-inventory/SHARD_TEST.txt               | 1 -
 packages/google-cloud-kms/SHARD_TEST.txt                         | 1 -
 packages/google-cloud-pubsub/SHARD_TEST.txt                      | 1 -
 packages/google-cloud-quotas/SHARD_TEST.txt                      | 1 -
 packages/google-cloud-rapidmigrationassessment/SHARD_TEST.txt    | 1 -
 packages/google-cloud-recaptcha-enterprise/SHARD_TEST.txt        | 1 -
 packages/google-cloud-recommendations-ai/SHARD_TEST.txt          | 1 -
 packages/google-cloud-recommender/SHARD_TEST.txt                 | 1 -
 packages/google-cloud-redis-cluster/SHARD_TEST.txt               | 1 -
 packages/google-cloud-redis/SHARD_TEST.txt                       | 1 -
 packages/google-cloud-resource-manager/SHARD_TEST.txt            | 1 -
 packages/google-cloud-retail/SHARD_TEST.txt                      | 1 -
 packages/google-cloud-run/SHARD_TEST.txt                         | 1 -
 packages/sqlalchemy-bigquery/SHARD_TEST.txt                      | 1 -
 84 files changed, 84 deletions(-)
 delete mode 100644 packages/django-google-spanner/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-appoptimize/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-artifact-registry/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-asset/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-assured-workloads/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-audit-log/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-auditmanager/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-automl/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-backupdr/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-bare-metal-solution/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-batch/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-beyondcorp-appconnections/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-beyondcorp-appconnectors/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-beyondcorp-appgateways/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-beyondcorp-clientconnectorservices/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-beyondcorp-clientgateways/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-biglake-hive/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-biglake/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-bigquery-analyticshub/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-bigquery-biglake/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-bigquery-connection/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-bigquery-data-exchange/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-bigquery-datapolicies/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-bigquery-datatransfer/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-bigquery-logging/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-bigquery-migration/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-bigquery-reservation/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-bigquery-storage/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-bigquery/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-bigtable/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-container/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-containeranalysis/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-contentwarehouse/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-core/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-data-fusion/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-data-qna/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-databasecenter/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-datacatalog-lineage-configmanagement/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-datacatalog-lineage/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-datacatalog/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-dataflow-client/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-dataform/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-datalabeling/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-dataplex/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-dataproc-metastore/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-dataproc/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-datastore/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-datastream/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-deploy/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-developerconnect/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-essential-contacts/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-eventarc-publishing/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-eventarc/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-filestore/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-financialservices/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-functions/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-gdchardwaremanagement/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-geminidataanalytics/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-gke-backup/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-gke-connect-gateway/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-gke-hub/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-gke-multicloud/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-gkerecommender/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-gsuiteaddons/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-hypercomputecluster/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-iam-logging/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-iam/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-iamconnectorcredentials/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-iap/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-ids/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-kms-inventory/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-kms/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-pubsub/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-quotas/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-rapidmigrationassessment/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-recaptcha-enterprise/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-recommendations-ai/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-recommender/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-redis-cluster/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-redis/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-resource-manager/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-retail/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-run/SHARD_TEST.txt
 delete mode 100644 packages/sqlalchemy-bigquery/SHARD_TEST.txt

diff --git a/packages/django-google-spanner/SHARD_TEST.txt b/packages/django-google-spanner/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/django-google-spanner/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-appoptimize/SHARD_TEST.txt b/packages/google-cloud-appoptimize/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-appoptimize/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-artifact-registry/SHARD_TEST.txt b/packages/google-cloud-artifact-registry/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-artifact-registry/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-asset/SHARD_TEST.txt b/packages/google-cloud-asset/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-asset/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-assured-workloads/SHARD_TEST.txt b/packages/google-cloud-assured-workloads/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-assured-workloads/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-audit-log/SHARD_TEST.txt b/packages/google-cloud-audit-log/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-audit-log/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-auditmanager/SHARD_TEST.txt b/packages/google-cloud-auditmanager/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-auditmanager/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-automl/SHARD_TEST.txt b/packages/google-cloud-automl/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-automl/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-backupdr/SHARD_TEST.txt b/packages/google-cloud-backupdr/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-backupdr/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-bare-metal-solution/SHARD_TEST.txt b/packages/google-cloud-bare-metal-solution/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-bare-metal-solution/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-batch/SHARD_TEST.txt b/packages/google-cloud-batch/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-batch/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-beyondcorp-appconnections/SHARD_TEST.txt b/packages/google-cloud-beyondcorp-appconnections/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-beyondcorp-appconnections/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-beyondcorp-appconnectors/SHARD_TEST.txt b/packages/google-cloud-beyondcorp-appconnectors/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-beyondcorp-appconnectors/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-beyondcorp-appgateways/SHARD_TEST.txt b/packages/google-cloud-beyondcorp-appgateways/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-beyondcorp-appgateways/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-beyondcorp-clientconnectorservices/SHARD_TEST.txt b/packages/google-cloud-beyondcorp-clientconnectorservices/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-beyondcorp-clientconnectorservices/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-beyondcorp-clientgateways/SHARD_TEST.txt b/packages/google-cloud-beyondcorp-clientgateways/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-beyondcorp-clientgateways/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-biglake-hive/SHARD_TEST.txt b/packages/google-cloud-biglake-hive/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-biglake-hive/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-biglake/SHARD_TEST.txt b/packages/google-cloud-biglake/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-biglake/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-bigquery-analyticshub/SHARD_TEST.txt b/packages/google-cloud-bigquery-analyticshub/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-bigquery-analyticshub/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-bigquery-biglake/SHARD_TEST.txt b/packages/google-cloud-bigquery-biglake/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-bigquery-biglake/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-bigquery-connection/SHARD_TEST.txt b/packages/google-cloud-bigquery-connection/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-bigquery-connection/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-bigquery-data-exchange/SHARD_TEST.txt b/packages/google-cloud-bigquery-data-exchange/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-bigquery-data-exchange/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-bigquery-datapolicies/SHARD_TEST.txt b/packages/google-cloud-bigquery-datapolicies/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-bigquery-datapolicies/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-bigquery-datatransfer/SHARD_TEST.txt b/packages/google-cloud-bigquery-datatransfer/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-bigquery-datatransfer/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-bigquery-logging/SHARD_TEST.txt b/packages/google-cloud-bigquery-logging/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-bigquery-logging/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-bigquery-migration/SHARD_TEST.txt b/packages/google-cloud-bigquery-migration/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-bigquery-migration/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-bigquery-reservation/SHARD_TEST.txt b/packages/google-cloud-bigquery-reservation/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-bigquery-reservation/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-bigquery-storage/SHARD_TEST.txt b/packages/google-cloud-bigquery-storage/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-bigquery-storage/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-bigquery/SHARD_TEST.txt b/packages/google-cloud-bigquery/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-bigquery/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-bigtable/SHARD_TEST.txt b/packages/google-cloud-bigtable/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-bigtable/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-container/SHARD_TEST.txt b/packages/google-cloud-container/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-container/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-containeranalysis/SHARD_TEST.txt b/packages/google-cloud-containeranalysis/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-containeranalysis/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-contentwarehouse/SHARD_TEST.txt b/packages/google-cloud-contentwarehouse/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-contentwarehouse/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-core/SHARD_TEST.txt b/packages/google-cloud-core/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-core/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-data-fusion/SHARD_TEST.txt b/packages/google-cloud-data-fusion/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-data-fusion/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-data-qna/SHARD_TEST.txt b/packages/google-cloud-data-qna/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-data-qna/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-databasecenter/SHARD_TEST.txt b/packages/google-cloud-databasecenter/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-databasecenter/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-datacatalog-lineage-configmanagement/SHARD_TEST.txt b/packages/google-cloud-datacatalog-lineage-configmanagement/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-datacatalog-lineage-configmanagement/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-datacatalog-lineage/SHARD_TEST.txt b/packages/google-cloud-datacatalog-lineage/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-datacatalog-lineage/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-datacatalog/SHARD_TEST.txt b/packages/google-cloud-datacatalog/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-datacatalog/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-dataflow-client/SHARD_TEST.txt b/packages/google-cloud-dataflow-client/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-dataflow-client/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-dataform/SHARD_TEST.txt b/packages/google-cloud-dataform/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-dataform/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-datalabeling/SHARD_TEST.txt b/packages/google-cloud-datalabeling/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-datalabeling/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-dataplex/SHARD_TEST.txt b/packages/google-cloud-dataplex/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-dataplex/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-dataproc-metastore/SHARD_TEST.txt b/packages/google-cloud-dataproc-metastore/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-dataproc-metastore/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-dataproc/SHARD_TEST.txt b/packages/google-cloud-dataproc/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-dataproc/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-datastore/SHARD_TEST.txt b/packages/google-cloud-datastore/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-datastore/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-datastream/SHARD_TEST.txt b/packages/google-cloud-datastream/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-datastream/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-deploy/SHARD_TEST.txt b/packages/google-cloud-deploy/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-deploy/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-developerconnect/SHARD_TEST.txt b/packages/google-cloud-developerconnect/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-developerconnect/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-essential-contacts/SHARD_TEST.txt b/packages/google-cloud-essential-contacts/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-essential-contacts/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-eventarc-publishing/SHARD_TEST.txt b/packages/google-cloud-eventarc-publishing/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-eventarc-publishing/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-eventarc/SHARD_TEST.txt b/packages/google-cloud-eventarc/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-eventarc/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-filestore/SHARD_TEST.txt b/packages/google-cloud-filestore/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-filestore/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-financialservices/SHARD_TEST.txt b/packages/google-cloud-financialservices/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-financialservices/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-functions/SHARD_TEST.txt b/packages/google-cloud-functions/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-functions/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-gdchardwaremanagement/SHARD_TEST.txt b/packages/google-cloud-gdchardwaremanagement/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-gdchardwaremanagement/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-geminidataanalytics/SHARD_TEST.txt b/packages/google-cloud-geminidataanalytics/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-geminidataanalytics/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-gke-backup/SHARD_TEST.txt b/packages/google-cloud-gke-backup/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-gke-backup/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-gke-connect-gateway/SHARD_TEST.txt b/packages/google-cloud-gke-connect-gateway/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-gke-connect-gateway/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-gke-hub/SHARD_TEST.txt b/packages/google-cloud-gke-hub/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-gke-hub/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-gke-multicloud/SHARD_TEST.txt b/packages/google-cloud-gke-multicloud/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-gke-multicloud/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-gkerecommender/SHARD_TEST.txt b/packages/google-cloud-gkerecommender/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-gkerecommender/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-gsuiteaddons/SHARD_TEST.txt b/packages/google-cloud-gsuiteaddons/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-gsuiteaddons/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-hypercomputecluster/SHARD_TEST.txt b/packages/google-cloud-hypercomputecluster/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-hypercomputecluster/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-iam-logging/SHARD_TEST.txt b/packages/google-cloud-iam-logging/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-iam-logging/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-iam/SHARD_TEST.txt b/packages/google-cloud-iam/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-iam/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-iamconnectorcredentials/SHARD_TEST.txt b/packages/google-cloud-iamconnectorcredentials/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-iamconnectorcredentials/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-iap/SHARD_TEST.txt b/packages/google-cloud-iap/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-iap/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-ids/SHARD_TEST.txt b/packages/google-cloud-ids/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-ids/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-kms-inventory/SHARD_TEST.txt b/packages/google-cloud-kms-inventory/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-kms-inventory/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-kms/SHARD_TEST.txt b/packages/google-cloud-kms/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-kms/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-pubsub/SHARD_TEST.txt b/packages/google-cloud-pubsub/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-pubsub/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-quotas/SHARD_TEST.txt b/packages/google-cloud-quotas/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-quotas/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-rapidmigrationassessment/SHARD_TEST.txt b/packages/google-cloud-rapidmigrationassessment/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-rapidmigrationassessment/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-recaptcha-enterprise/SHARD_TEST.txt b/packages/google-cloud-recaptcha-enterprise/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-recaptcha-enterprise/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-recommendations-ai/SHARD_TEST.txt b/packages/google-cloud-recommendations-ai/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-recommendations-ai/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-recommender/SHARD_TEST.txt b/packages/google-cloud-recommender/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-recommender/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-redis-cluster/SHARD_TEST.txt b/packages/google-cloud-redis-cluster/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-redis-cluster/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-redis/SHARD_TEST.txt b/packages/google-cloud-redis/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-redis/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-resource-manager/SHARD_TEST.txt b/packages/google-cloud-resource-manager/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-resource-manager/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-retail/SHARD_TEST.txt b/packages/google-cloud-retail/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-retail/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-run/SHARD_TEST.txt b/packages/google-cloud-run/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-run/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/sqlalchemy-bigquery/SHARD_TEST.txt b/packages/sqlalchemy-bigquery/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/sqlalchemy-bigquery/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding

From 8b825d5a645184860009b3dffef65c82c08d7109 Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Fri, 12 Jun 2026 12:20:49 -0700
Subject: [PATCH 18/84] updated lint and mypy logic

---
 ci/run_conditional_tests.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ci/run_conditional_tests.sh b/ci/run_conditional_tests.sh
index 5236b2bec292..c5321a17bf91 100755
--- a/ci/run_conditional_tests.sh
+++ b/ci/run_conditional_tests.sh
@@ -102,10 +102,10 @@ for subdir in ${subdirs[@]}; do
     for d in `ls -d ${subdir}/*/ | sort`; do
         # Sharding logic: only process directories that belong to this shard
         if (( count % TOTAL_SHARDS != SHARD_INDEX - 1 )); then
-            ((count++))
+            ((++count))
             continue
         fi
-        ((count++))
+        ((++count))
 
         should_test=false
         if [ -n "${GIT_DIFF_ARG}" ]; then

From 32968a9a2222b9414574b0f8e1d8d26f6f0bcf95 Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Fri, 12 Jun 2026 12:33:23 -0700
Subject: [PATCH 19/84] add shard descriptions

---
 .github/workflows/unittest.yml | 2 +-
 ci/get_package_shards.py       | 7 +++++++
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml
index 9c349028e862..95aa8ce0ed9a 100644
--- a/.github/workflows/unittest.yml
+++ b/.github/workflows/unittest.yml
@@ -63,7 +63,7 @@ jobs:
       run: |
         python -m pip install --upgrade setuptools pip wheel
         python -m pip install nox
-    - name: Run unit tests
+    - name: Run unit tests for ${{ matrix.package_shard.description }}
       env:
         BUILD_TYPE: presubmit
         TARGET_BRANCH: ${{ github.base_ref || github.event.merge_group.base_ref }}
diff --git a/ci/get_package_shards.py b/ci/get_package_shards.py
index c5b8a03f8de7..89a99bf7730b 100644
--- a/ci/get_package_shards.py
+++ b/ci/get_package_shards.py
@@ -82,9 +82,16 @@ def group_packages(packages):
         index = i + 1
         name = f"Shard {index}"
         
+        # Calculate a descriptive range for step visibility
+        if len(shard_packages) == 1:
+            desc = shard_packages[0].strip('/').split('/')[-1]
+        else:
+            desc = f"{shard_packages[0].strip('/').split('/')[-1]}...{shard_packages[-1].strip('/').split('/')[-1]}"
+        
         shards.append({
             "name": name,
             "index": index,
+            "description": desc,
             "packages": " ".join(shard_packages),
             "is_sharded": num_shards > 1
         })

From ff1274f39f8f29c5c13663753c3fa83e1cb12e78 Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Fri, 12 Jun 2026 12:37:01 -0700
Subject: [PATCH 20/84] remove global run on ci/ change

---
 ci/get_package_shards.py | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/ci/get_package_shards.py b/ci/get_package_shards.py
index 89a99bf7730b..5147153f8562 100644
--- a/ci/get_package_shards.py
+++ b/ci/get_package_shards.py
@@ -28,16 +28,6 @@ def get_packages_to_test():
     else:
         return all_packages
 
-    # Check if ci/ changed
-    try:
-        subprocess.check_call(['git', 'diff', '--quiet', git_diff_arg, 'ci'])
-        ci_changed = False
-    except subprocess.CalledProcessError:
-        ci_changed = True
-    
-    if ci_changed:
-        return all_packages
-    
     try:
         res = subprocess.check_output(['git', 'diff', '--name-only', git_diff_arg]).decode('utf-8')
         changed_files = res.splitlines()

From c49b06d884e554ef0eca578606afd3fe871bf8c5 Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Fri, 12 Jun 2026 13:17:10 -0700
Subject: [PATCH 21/84] attempt cover fix

---
 .github/workflows/unittest.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml
index 95aa8ce0ed9a..b8680571645f 100644
--- a/.github/workflows/unittest.yml
+++ b/.github/workflows/unittest.yml
@@ -123,5 +123,6 @@ jobs:
     - name: Report coverage results
       if: steps.packages.outputs.num_files_changed > 0
       run: |
-        coverage combine .coverage-results/**/.coverage*
+        find .coverage-results -type f -name '.coverage*' -exec mv {} . \;
+        coverage combine
         coverage report --show-missing --fail-under=100

From 1cd8c1f5baf462387d22408d013577d9eb882119 Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Fri, 12 Jun 2026 13:28:15 -0700
Subject: [PATCH 22/84] attempt fix for cover

---
 .github/workflows/unittest.yml | 12 +++++++++---
 ci/run_conditional_tests.sh    |  7 +++++++
 2 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml
index b8680571645f..f99428713be1 100644
--- a/.github/workflows/unittest.yml
+++ b/.github/workflows/unittest.yml
@@ -123,6 +123,12 @@ jobs:
     - name: Report coverage results
       if: steps.packages.outputs.num_files_changed > 0
       run: |
-        find .coverage-results -type f -name '.coverage*' -exec mv {} . \;
-        coverage combine
-        coverage report --show-missing --fail-under=100
+        if [ -d .coverage-results ]; then
+          find .coverage-results -type f -name '.coverage*' -exec mv {} . \;
+          coverage combine
+          coverage report --show-missing --fail-under=100
+        else
+          echo "Error: No coverage results were downloaded from the unit test jobs."
+          echo "This usually means the unit tests did not run or failed to upload their coverage files."
+          exit 1
+        fi
diff --git a/ci/run_conditional_tests.sh b/ci/run_conditional_tests.sh
index c5321a17bf91..fb68904ead6a 100755
--- a/ci/run_conditional_tests.sh
+++ b/ci/run_conditional_tests.sh
@@ -55,6 +55,13 @@ if [ -n "${PACKAGE_LIST}" ]; then
         echo "running test in ${d}"
         pushd ${d}
         set +e
+        
+        # Ensure unique coverage file per package to avoid DataError
+        # when combining statement and branch coverage.
+        # Strip trailing slash from directory name for the filename.
+        pkg_name_clean=$(echo ${d} | sed 's|/$||' | sed 's|/|_|g')
+        export COVERAGE_FILE="${PROJECT_ROOT}/.coverage.${PY_VERSION}.${pkg_name_clean}"
+        
         ${test_script}
         ret=$?
         set -e

From a0f5c9ada55a7b0c544d807e0e3d4b4311497a6b Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Fri, 12 Jun 2026 13:50:18 -0700
Subject: [PATCH 23/84] allow hidden files for cover

---
 .github/workflows/unittest.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml
index f99428713be1..e806130e0ae5 100644
--- a/.github/workflows/unittest.yml
+++ b/.github/workflows/unittest.yml
@@ -77,6 +77,7 @@ jobs:
       with:
         name: coverage-artifact-${{ matrix.python }}-${{ matrix.package_shard.index }}
         path: .coverage.${{ matrix.python }}.*
+        include-hidden-files: true
 
   unit-complete:
     needs: [discover, unit]

From 71f444a03dca196b8984321de2ec65c9e2093b26 Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Fri, 12 Jun 2026 13:55:50 -0700
Subject: [PATCH 24/84] fail fast

---
 .github/workflows/unittest.yml |  2 +-
 .kokoro/system.sh              | 19 +++++++++++++++++--
 2 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml
index e806130e0ae5..3385b5a127e8 100644
--- a/.github/workflows/unittest.yml
+++ b/.github/workflows/unittest.yml
@@ -42,7 +42,7 @@ jobs:
     if: needs.discover.outputs.matrix != '[]' && needs.discover.outputs.matrix != ''
     runs-on: ubuntu-22.04
     strategy:
-      fail-fast: false
+      fail-fast: true
       matrix:
         python: ['3.9', '3.10', "3.11", "3.12", "3.13", "3.14"]
         package_shard: ${{ fromJson(needs.discover.outputs.matrix) }}
diff --git a/.kokoro/system.sh b/.kokoro/system.sh
index e24b6b1a5d81..a46758e3e11c 100755
--- a/.kokoro/system.sh
+++ b/.kokoro/system.sh
@@ -144,6 +144,7 @@ handle_finished_job() {
   echo "Artifacts in: ${KOKORO_ARTIFACTS_DIR}/${pkg}"
   echo "============================================================"
 
+  # Print the package log to the console
   if [ -f "${pkg_log}" ]; then
     echo "--- Start of Logs for ${pkg} ---"
     cat "${pkg_log}"
@@ -154,8 +155,22 @@ handle_finished_job() {
   echo ""
 
   if [ -z "${res}" ] || [ "${res}" -ne 0 ]; then
-    RETVAL=1
-    results+=("${pkg}: FAILED")
+    echo "============================================================"
+    echo "FAIL-FAST: System tests for ${pkg} failed!"
+    echo "Cancelling all remaining running background jobs..."
+    echo "============================================================"
+    
+    # Kill all other active background processes
+    for active_pid in "${running_pids[@]}"; do
+      if [ "$active_pid" != "$pid" ] && kill -0 "$active_pid" 2>/dev/null; then
+        pkg_to_cancel=${pid_to_pkg[$active_pid]}
+        echo "Cancelling active system tests for ${pkg_to_cancel} (PID: ${active_pid})..."
+        # Send SIGTERM to allow graceful cleanup of resources if possible, or SIGKILL
+        kill -9 "$active_pid" 2>/dev/null || true
+      fi
+    done
+    
+    exit "${res}"
   else
     results+=("${pkg}: PASSED")
   fi

From cf1a1f16dd823558c00148e01f997e280a4fd38c Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Fri, 12 Jun 2026 15:12:07 -0700
Subject: [PATCH 25/84] use individual coverage checks

---
 .github/workflows/unittest.yml | 27 ++++++++++++++++++++++++++-
 1 file changed, 26 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml
index 3385b5a127e8..78c7e20b8345 100644
--- a/.github/workflows/unittest.yml
+++ b/.github/workflows/unittest.yml
@@ -127,7 +127,32 @@ jobs:
         if [ -d .coverage-results ]; then
           find .coverage-results -type f -name '.coverage*' -exec mv {} . \;
           coverage combine
-          coverage report --show-missing --fail-under=100
+          
+          # Find all modified packages
+          modified_packages=$(git diff --name-only HEAD~1 -- packages | cut -d/ -f1,2 | sort -u)
+          
+          for pkg in ${modified_packages}; do
+            if [ -d "${pkg}" ]; then
+              echo "============================================================"
+              echo "Evaluating coverage for package: ${pkg}"
+              echo "============================================================"
+              
+              if [ -f "${pkg}/.coveragerc" ]; then
+                echo "Using package-specific configuration: ${pkg}/.coveragerc"
+                # If fail_under is specified in the package-specific .coveragerc, coverage report
+                # will automatically enforce it. Otherwise, we enforce the default 100%.
+                if grep -q "fail_under" "${pkg}/.coveragerc"; then
+                  coverage report --rcfile="${pkg}/.coveragerc"
+                else
+                  echo "No fail_under specified in ${pkg}/.coveragerc, enforcing default of 100%"
+                  coverage report --rcfile="${pkg}/.coveragerc" --fail-under=100
+                fi
+              else
+                echo "No .coveragerc found for ${pkg}, enforcing default of 100%"
+                coverage report --include="${pkg}/**" --fail-under=100
+              fi
+            fi
+          done
         else
           echo "Error: No coverage results were downloaded from the unit test jobs."
           echo "This usually means the unit tests did not run or failed to upload their coverage files."

From b00967f00800ad6b41a887ec7d1112bddc8c594d Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Fri, 12 Jun 2026 15:17:58 -0700
Subject: [PATCH 26/84] loosen firestore coverage requirement

---
 packages/google-cloud-firestore/.coveragerc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/google-cloud-firestore/.coveragerc b/packages/google-cloud-firestore/.coveragerc
index 4c355f6455c1..66bbde8955a9 100644
--- a/packages/google-cloud-firestore/.coveragerc
+++ b/packages/google-cloud-firestore/.coveragerc
@@ -2,6 +2,7 @@
 branch = True
 
 [report]
+fail_under = 99
 show_missing = True
 omit =
     google/cloud/firestore/__init__.py

From 7bb323ceb3bf4b25af8903874e460f1c0b0f17c6 Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Fri, 12 Jun 2026 16:43:46 -0700
Subject: [PATCH 27/84] activated more packages

---
 packages/google-backstory/SHARD_TEST.txt         | 1 +
 packages/google-cloud-apiregistry/SHARD_TEST.txt | 1 +
 packages/google-cloud-bigtable/SHARD_TEST.txt    | 1 +
 packages/google-cloud-ces/SHARD_TEST.txt         | 1 +
 packages/google-cloud-chronicle/SHARD_TEST.txt   | 1 +
 packages/google-cloud-dataproc/SHARD_TEST.txt    | 1 +
 6 files changed, 6 insertions(+)
 create mode 100644 packages/google-backstory/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-apiregistry/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-bigtable/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-ces/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-chronicle/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-dataproc/SHARD_TEST.txt

diff --git a/packages/google-backstory/SHARD_TEST.txt b/packages/google-backstory/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-backstory/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-apiregistry/SHARD_TEST.txt b/packages/google-cloud-apiregistry/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-apiregistry/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-bigtable/SHARD_TEST.txt b/packages/google-cloud-bigtable/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-bigtable/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-ces/SHARD_TEST.txt b/packages/google-cloud-ces/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-ces/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-chronicle/SHARD_TEST.txt b/packages/google-cloud-chronicle/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-chronicle/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-dataproc/SHARD_TEST.txt b/packages/google-cloud-dataproc/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-dataproc/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding

From 363308980061fa18dfff9ae63fe801e5613b1120 Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Fri, 12 Jun 2026 17:07:00 -0700
Subject: [PATCH 28/84] 10 packages total

---
 packages/google-cloud-managedkafka/SHARD_TEST.txt | 1 +
 packages/google-cloud-optimization/SHARD_TEST.txt | 1 +
 packages/google-cloud-retail/SHARD_TEST.txt       | 1 +
 3 files changed, 3 insertions(+)
 create mode 100644 packages/google-cloud-managedkafka/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-optimization/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-retail/SHARD_TEST.txt

diff --git a/packages/google-cloud-managedkafka/SHARD_TEST.txt b/packages/google-cloud-managedkafka/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-managedkafka/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-optimization/SHARD_TEST.txt b/packages/google-cloud-optimization/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-optimization/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding
diff --git a/packages/google-cloud-retail/SHARD_TEST.txt b/packages/google-cloud-retail/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-retail/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding

From 980d8eb2bb2f0e4f4a69946ab639d4fd98dcc135 Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Fri, 12 Jun 2026 17:10:15 -0700
Subject: [PATCH 29/84] change default for coverage percent

---
 .github/workflows/unittest.yml | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml
index 78c7e20b8345..a2f4e9363b12 100644
--- a/.github/workflows/unittest.yml
+++ b/.github/workflows/unittest.yml
@@ -123,33 +123,37 @@ jobs:
         path: .coverage-results/
     - name: Report coverage results
       if: steps.packages.outputs.num_files_changed > 0
+      env:
+        # TODO: default to 100% coverage after next gapic-generator release
+        # https://github.com/googleapis/google-cloud-python/issues/17459
+        DEFAULT_FAIL_UNDER: 99
       run: |
         if [ -d .coverage-results ]; then
           find .coverage-results -type f -name '.coverage*' -exec mv {} . \;
           coverage combine
-          
+
           # Find all modified packages
           modified_packages=$(git diff --name-only HEAD~1 -- packages | cut -d/ -f1,2 | sort -u)
-          
+
           for pkg in ${modified_packages}; do
             if [ -d "${pkg}" ]; then
               echo "============================================================"
               echo "Evaluating coverage for package: ${pkg}"
               echo "============================================================"
-              
+
               if [ -f "${pkg}/.coveragerc" ]; then
                 echo "Using package-specific configuration: ${pkg}/.coveragerc"
                 # If fail_under is specified in the package-specific .coveragerc, coverage report
-                # will automatically enforce it. Otherwise, we enforce the default 100%.
+                # will automatically enforce it. Otherwise, we enforce the default.
                 if grep -q "fail_under" "${pkg}/.coveragerc"; then
                   coverage report --rcfile="${pkg}/.coveragerc"
                 else
-                  echo "No fail_under specified in ${pkg}/.coveragerc, enforcing default of 100%"
-                  coverage report --rcfile="${pkg}/.coveragerc" --fail-under=100
+                  echo "No fail_under specified in ${pkg}/.coveragerc, enforcing default"
+                  coverage report --rcfile="${pkg}/.coveragerc" --fail-under="${DEFAULT_FAIL_UNDER}"
                 fi
               else
-                echo "No .coveragerc found for ${pkg}, enforcing default of 100%"
-                coverage report --include="${pkg}/**" --fail-under=100
+                echo "No .coveragerc found for ${pkg}, enforcing default
+                coverage report --include="${pkg}/**" --fail-under="${DEFAULT_FAIL_UNDER}"
               fi
             fi
           done

From a5cc4979647c3c353f8cbcd8c0c4131bc7a7a98b Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Fri, 12 Jun 2026 17:29:36 -0700
Subject: [PATCH 30/84] activated 11th package (enable sharding)

---
 packages/google-cloud-gke-multicloud/SHARD_TEST.txt | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 packages/google-cloud-gke-multicloud/SHARD_TEST.txt

diff --git a/packages/google-cloud-gke-multicloud/SHARD_TEST.txt b/packages/google-cloud-gke-multicloud/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/google-cloud-gke-multicloud/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding

From b36b559c0afa45812f9e4ab094b70ba6e3e85be6 Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Fri, 12 Jun 2026 17:35:40 -0700
Subject: [PATCH 31/84] fixed typo

---
 .github/workflows/unittest.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml
index a2f4e9363b12..31f306bd9bf7 100644
--- a/.github/workflows/unittest.yml
+++ b/.github/workflows/unittest.yml
@@ -152,7 +152,7 @@ jobs:
                   coverage report --rcfile="${pkg}/.coveragerc" --fail-under="${DEFAULT_FAIL_UNDER}"
                 fi
               else
-                echo "No .coveragerc found for ${pkg}, enforcing default
+                echo "No .coveragerc found for ${pkg}, enforcing default"
                 coverage report --include="${pkg}/**" --fail-under="${DEFAULT_FAIL_UNDER}"
               fi
             fi

From c3cf49eb027638e57cc0f68821ad6e54771b079a Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Fri, 12 Jun 2026 17:43:10 -0700
Subject: [PATCH 32/84] change shard logic

---
 ci/get_package_shards.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ci/get_package_shards.py b/ci/get_package_shards.py
index 5147153f8562..d410697c824b 100644
--- a/ci/get_package_shards.py
+++ b/ci/get_package_shards.py
@@ -49,8 +49,8 @@ def group_packages(packages):
     num_packages = len(packages)
     
     # 1. Only shard if > 10 packages are being touched
-    # 2. Only add a new shard if we'd have > 10 in each shard (meaning size >= 11)
-    num_shards = num_packages // 11
+    # 2. Only add a new shard if we'd have > 10 packages in at least one shard (meaning ceil(N / S) >= 11)
+    num_shards = (num_packages - 1) // 10
     
     # Ensure at least 1 shard if we have packages
     num_shards = max(1, num_shards)

From acdfb1890618a9fa2052d71967e8850f47cd7478 Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Fri, 12 Jun 2026 17:49:45 -0700
Subject: [PATCH 33/84] iterating on shard logic

---
 ci/get_package_shards.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/ci/get_package_shards.py b/ci/get_package_shards.py
index d410697c824b..6c72ede87aaf 100644
--- a/ci/get_package_shards.py
+++ b/ci/get_package_shards.py
@@ -49,8 +49,10 @@ def group_packages(packages):
     num_packages = len(packages)
     
     # 1. Only shard if > 10 packages are being touched
-    # 2. Only add a new shard if we'd have > 10 packages in at least one shard (meaning ceil(N / S) >= 11)
-    num_shards = (num_packages - 1) // 10
+    # 2. Only add a new shard if any shard would have > 10 packages.
+    # To guarantee that no shard contains more than 10 packages (when distributed evenly),
+    # we need S >= N / 10, which means S = ceil(N / 10).
+    num_shards = math.ceil(num_packages / 10)
     
     # Ensure at least 1 shard if we have packages
     num_shards = max(1, num_shards)

From 9c80a5105d48dce2c60852427b308305d99e549d Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Fri, 12 Jun 2026 18:00:18 -0700
Subject: [PATCH 34/84] rename tests

---
 .github/workflows/unittest.yml | 12 ++++++------
 ci/get_package_shards.py       | 27 +++++++++++++--------------
 2 files changed, 19 insertions(+), 20 deletions(-)

diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml
index 31f306bd9bf7..f5ee10281c5d 100644
--- a/.github/workflows/unittest.yml
+++ b/.github/workflows/unittest.yml
@@ -13,7 +13,7 @@ permissions:
   contents: read
 
 jobs:
-  discover:
+  initialize:
     runs-on: ubuntu-latest
     outputs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
@@ -38,14 +38,14 @@ jobs:
         echo "matrix=$(python3 ci/get_package_shards.py)" >> $GITHUB_OUTPUT
 
   unit:
-    needs: discover
-    if: needs.discover.outputs.matrix != '[]' && needs.discover.outputs.matrix != ''
+    needs: initialize
+    if: needs.initialize.outputs.matrix != '[]' && needs.initialize.outputs.matrix != ''
     runs-on: ubuntu-22.04
     strategy:
       fail-fast: true
       matrix:
         python: ['3.9', '3.10', "3.11", "3.12", "3.13", "3.14"]
-        package_shard: ${{ fromJson(needs.discover.outputs.matrix) }}
+        package_shard: ${{ fromJson(needs.initialize.outputs.matrix) }}
     name: ${{ matrix.package_shard.is_sharded && format('unit ({0}, {1})', matrix.python, matrix.package_shard.name) || format('unit ({0})', matrix.python) }}
     steps:
     - name: Checkout
@@ -79,8 +79,8 @@ jobs:
         path: .coverage.${{ matrix.python }}.*
         include-hidden-files: true
 
-  unit-complete:
-    needs: [discover, unit]
+  all-tests:
+    needs: [initialize, unit]
     if: always()
     runs-on: ubuntu-latest
     steps:
diff --git a/ci/get_package_shards.py b/ci/get_package_shards.py
index 6c72ede87aaf..4de99c64cf75 100644
--- a/ci/get_package_shards.py
+++ b/ci/get_package_shards.py
@@ -18,9 +18,9 @@ def get_packages():
 def get_packages_to_test():
     build_type = os.environ.get('BUILD_TYPE', 'presubmit')
     target_branch = os.environ.get('TARGET_BRANCH', 'main')
-    
+
     all_packages = get_packages()
-    
+
     if build_type == 'presubmit':
         git_diff_arg = f"origin/{target_branch}..."
     elif build_type == 'continuous':
@@ -33,53 +33,52 @@ def get_packages_to_test():
         changed_files = res.splitlines()
     except subprocess.CalledProcessError:
         return all_packages
-    
+
     to_test = []
     for pkg in all_packages:
         # Check if any changed file starts with the package path
         if any(f.startswith(pkg) for f in changed_files):
             to_test.append(pkg)
-    
+
     return to_test
 
 def group_packages(packages):
     if not packages:
         return []
-    
+
     num_packages = len(packages)
-    
+
     # 1. Only shard if > 10 packages are being touched
     # 2. Only add a new shard if any shard would have > 10 packages.
     # To guarantee that no shard contains more than 10 packages (when distributed evenly),
     # we need S >= N / 10, which means S = ceil(N / 10).
     num_shards = math.ceil(num_packages / 10)
-    
+
     # Ensure at least 1 shard if we have packages
     num_shards = max(1, num_shards)
-    
+
     # 3. Top out at 10 shards
     num_shards = min(10, num_shards)
-    
+
     # Distribute packages between them as evenly as possible
     shard_size = math.ceil(num_packages / num_shards)
-    
+
     shards = []
     for i in range(num_shards):
         start = i * shard_size
         end = min((i + 1) * shard_size, num_packages)
         if start >= num_packages:
             break
-        
         shard_packages = packages[start:end]
         index = i + 1
         name = f"Shard {index}"
-        
+        num_in_shard = len(shard_packages)
         # Calculate a descriptive range for step visibility
         if len(shard_packages) == 1:
             desc = shard_packages[0].strip('/').split('/')[-1]
         else:
-            desc = f"{shard_packages[0].strip('/').split('/')[-1]}...{shard_packages[-1].strip('/').split('/')[-1]}"
-        
+            desc = f"{shard_packages[0].strip('/').split('/')[-1]}...{shard_packages[-1].strip('/').split('/')[-1]} ({num_in_shard})"
+
         shards.append({
             "name": name,
             "index": index,

From cf554cabbf84e1fdc87f38b2e9f096a45892e1ce Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Fri, 12 Jun 2026 18:02:52 -0700
Subject: [PATCH 35/84] add label to number in shard

---
 ci/get_package_shards.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/get_package_shards.py b/ci/get_package_shards.py
index 4de99c64cf75..a5599146abd5 100644
--- a/ci/get_package_shards.py
+++ b/ci/get_package_shards.py
@@ -77,7 +77,7 @@ def group_packages(packages):
         if len(shard_packages) == 1:
             desc = shard_packages[0].strip('/').split('/')[-1]
         else:
-            desc = f"{shard_packages[0].strip('/').split('/')[-1]}...{shard_packages[-1].strip('/').split('/')[-1]} ({num_in_shard})"
+            desc = f"{shard_packages[0].strip('/').split('/')[-1]}...{shard_packages[-1].strip('/').split('/')[-1]} ({num_in_shard} packages)"
 
         shards.append({
             "name": name,

From 886d6d9d9040bb47171de313cc82fef16e195aa3 Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Fri, 12 Jun 2026 18:07:20 -0700
Subject: [PATCH 36/84] unit tests fail if initialize fails

---
 .github/workflows/unittest.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml
index f5ee10281c5d..f1685cbb62d4 100644
--- a/.github/workflows/unittest.yml
+++ b/.github/workflows/unittest.yml
@@ -86,6 +86,12 @@ jobs:
     steps:
       - name: Check unit test results
         run: |
+          # 1. Check initialize job
+          if [[ "${{ needs.initialize.result }}" != "success" ]]; then
+            echo "Error: The initialize job status was: ${{ needs.initialize.result }}"
+            exit 1
+          fi
+          # 2. Check unit test shards
           if [[ "${{ needs.unit.result }}" != "success" && "${{ needs.unit.result }}" != "skipped" ]]; then
             echo "Unit tests failed"
             exit 1

From 83ca04a45cc756898ca93ef25dfc5446e1a27eb6 Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Fri, 12 Jun 2026 18:10:46 -0700
Subject: [PATCH 37/84] added known-bad package

---
 packages/sqlalchemy-spanner/SHARD_TEST.txt | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 packages/sqlalchemy-spanner/SHARD_TEST.txt

diff --git a/packages/sqlalchemy-spanner/SHARD_TEST.txt b/packages/sqlalchemy-spanner/SHARD_TEST.txt
new file mode 100644
index 000000000000..b7ac1b35614f
--- /dev/null
+++ b/packages/sqlalchemy-spanner/SHARD_TEST.txt
@@ -0,0 +1 @@
+Test change for sharding

From ae05642c77ed529b0a6782fd6e51477b919cb2c3 Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Fri, 12 Jun 2026 19:09:16 -0700
Subject: [PATCH 38/84] added summary to cover step

---
 .github/workflows/unittest.yml | 33 +++++++++++++++++++++++++++++++--
 1 file changed, 31 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml
index f1685cbb62d4..7c97e149daa1 100644
--- a/.github/workflows/unittest.yml
+++ b/.github/workflows/unittest.yml
@@ -141,28 +141,57 @@ jobs:
           # Find all modified packages
           modified_packages=$(git diff --name-only HEAD~1 -- packages | cut -d/ -f1,2 | sort -u)
 
+          failed_packages=()
+          passed_packages=()
+
           for pkg in ${modified_packages}; do
             if [ -d "${pkg}" ]; then
               echo "============================================================"
               echo "Evaluating coverage for package: ${pkg}"
               echo "============================================================"
 
+              set +e
               if [ -f "${pkg}/.coveragerc" ]; then
                 echo "Using package-specific configuration: ${pkg}/.coveragerc"
                 # If fail_under is specified in the package-specific .coveragerc, coverage report
                 # will automatically enforce it. Otherwise, we enforce the default.
                 if grep -q "fail_under" "${pkg}/.coveragerc"; then
-                  coverage report --rcfile="${pkg}/.coveragerc"
+                  coverage report --rcfile="${pkg}/.coveragerc" --include="${pkg}/**"
                 else
                   echo "No fail_under specified in ${pkg}/.coveragerc, enforcing default"
-                  coverage report --rcfile="${pkg}/.coveragerc" --fail-under="${DEFAULT_FAIL_UNDER}"
+                  coverage report --rcfile="${pkg}/.coveragerc" --include="${pkg}/**" --fail-under="${DEFAULT_FAIL_UNDER}"
                 fi
               else
                 echo "No .coveragerc found for ${pkg}, enforcing default"
                 coverage report --include="${pkg}/**" --fail-under="${DEFAULT_FAIL_UNDER}"
               fi
+              status=$?
+              set -e
+
+              if [ ${status} -ne 0 ]; then
+                failed_packages+=("${pkg}")
+              else
+                passed_packages+=("${pkg}")
+              fi
             fi
           done
+
+          echo "============================================================"
+          echo "Coverage Evaluation Summary"
+          echo "============================================================"
+          if [ ${#passed_packages[@]} -gt 0 ]; then
+            echo "Passed packages:"
+            for pkg in "${passed_packages[@]}"; do
+              echo "  - ${pkg}"
+            done
+          fi
+          if [ ${#failed_packages[@]} -gt 0 ]; then
+            echo "Failed packages:"
+            for pkg in "${failed_packages[@]}"; do
+              echo "  - ${pkg}"
+            done
+            exit 1
+          fi
         else
           echo "Error: No coverage results were downloaded from the unit test jobs."
           echo "This usually means the unit tests did not run or failed to upload their coverage files."

From 7b565a4f4941c6159decfdba84394a17db50e9d6 Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Fri, 12 Jun 2026 19:09:44 -0700
Subject: [PATCH 39/84] added no-fail .coveragerc to sqlalchemy-spanner

---
 packages/sqlalchemy-spanner/.coveragerc | 36 +++++++++++++++++++++++++
 packages/sqlalchemy-spanner/noxfile.py  | 14 +++++++++-
 2 files changed, 49 insertions(+), 1 deletion(-)
 create mode 100644 packages/sqlalchemy-spanner/.coveragerc

diff --git a/packages/sqlalchemy-spanner/.coveragerc b/packages/sqlalchemy-spanner/.coveragerc
new file mode 100644
index 000000000000..d09941ce0eb9
--- /dev/null
+++ b/packages/sqlalchemy-spanner/.coveragerc
@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+[run]
+branch = True
+omit =
+  google/__init__.py
+  google/cloud/sqlalchemy_spanner/requirements.py
+
+[report]
+fail_under = 0
+show_missing = True
+exclude_lines =
+    # Re-enable the standard pragma
+    pragma: NO COVER
+    # Ignore debug-only repr
+    def __repr__
+    # Ignore abstract methods
+    raise NotImplementedError
+omit =
+  */site-packages/*.py
+  google/__init__.py
+  google/cloud/sqlalchemy_spanner/requirements.py
diff --git a/packages/sqlalchemy-spanner/noxfile.py b/packages/sqlalchemy-spanner/noxfile.py
index 187d4e7e8924..8adeb9388354 100644
--- a/packages/sqlalchemy-spanner/noxfile.py
+++ b/packages/sqlalchemy-spanner/noxfile.py
@@ -85,6 +85,7 @@ class = StreamHandler
 UNIT_TEST_STANDARD_DEPENDENCIES = [
     "mock",
     "pytest",
+    "pytest-cov",
 ]
 
 UNIT_TEST_EXTERNAL_DEPENDENCIES = [
@@ -368,7 +369,18 @@ def unit(session, test_type):
             *UNIT_TEST_DEPENDENCIES,
         )
         session.install(".")
-        session.run("py.test", "--quiet", os.path.join("tests/unit"), *session.posargs)
+        session.run(
+            "py.test",
+            "--quiet",
+            "--cov=google.cloud.sqlalchemy_spanner",
+            "--cov=tests/unit",
+            "--cov-append",
+            "--cov-config=.coveragerc",
+            "--cov-report=",
+            "--cov-fail-under=0",
+            os.path.join("tests/unit"),
+            *session.posargs,
+        )
         return
 
 

From e34b6543f3a38734688eccffff115536ffb1668f Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Thu, 18 Jun 2026 17:03:08 -0700
Subject: [PATCH 40/84] reverted systen test changes

---
 .kokoro/continuous/common.cfg |   1 -
 .kokoro/presubmit/common.cfg  |   1 -
 .kokoro/system-single.sh      |   8 +-
 .kokoro/system.sh             | 137 ++--------------------------------
 4 files changed, 6 insertions(+), 141 deletions(-)

diff --git a/.kokoro/continuous/common.cfg b/.kokoro/continuous/common.cfg
index 36cd039de187..175a4e8fd7a2 100644
--- a/.kokoro/continuous/common.cfg
+++ b/.kokoro/continuous/common.cfg
@@ -4,7 +4,6 @@
 action {
   define_artifacts {
     regex: "**/*sponge_log.xml"
-    regex: "**/*sponge_log.log"
   }
 }
 
diff --git a/.kokoro/presubmit/common.cfg b/.kokoro/presubmit/common.cfg
index 36cd039de187..175a4e8fd7a2 100644
--- a/.kokoro/presubmit/common.cfg
+++ b/.kokoro/presubmit/common.cfg
@@ -4,7 +4,6 @@
 action {
   define_artifacts {
     regex: "**/*sponge_log.xml"
-    regex: "**/*sponge_log.log"
   }
 }
 
diff --git a/.kokoro/system-single.sh b/.kokoro/system-single.sh
index b75936bda820..0ec5ae7ebf1b 100755
--- a/.kokoro/system-single.sh
+++ b/.kokoro/system-single.sh
@@ -32,10 +32,4 @@ NOX_FILE_ARG=""
 
 [[ -z "${NOX_FILE}" ]] || NOX_FILE_ARG="-f ${NOX_FILE}"
 
-NOX_ARGS=()
-if [[ -n "${XML_OUTPUT_FILE}" ]]; then
-  # Pass --junitxml as a positional argument to the underlying pytest call
-  NOX_ARGS+=("--junitxml=${XML_OUTPUT_FILE}")
-fi
-
-python3 -m nox ${NOX_SESSION_ARG} $NOX_FILE_ARG -- "${NOX_ARGS[@]}"
+python3 -m nox ${NOX_SESSION_ARG} $NOX_FILE_ARG
diff --git a/.kokoro/system.sh b/.kokoro/system.sh
index a46758e3e11c..469d0e81c7fa 100755
--- a/.kokoro/system.sh
+++ b/.kokoro/system.sh
@@ -38,18 +38,7 @@ pwd
 run_package_test() {
   local package_name=$1
   local package_path="packages/${package_name}"
-
-  # Create a dedicated directory for this package's artifacts
-  local artifact_dir="${KOKORO_ARTIFACTS_DIR}/${package_name}"
-  mkdir -p "${artifact_dir}"
-
-  # Define standard output paths for logs and XML results
-  export XML_OUTPUT_FILE="${artifact_dir}/sponge_log.xml"
-  local log_file="${artifact_dir}/sponge_log.log"
-
-  # Isolate gcloud config for parallel execution
-  export CLOUDSDK_CONFIG=$(mktemp -d)
-
+  
   # Declare local overrides to prevent bleeding into the next loop iteration
   local PROJECT_ID
   local GOOGLE_APPLICATION_CREDENTIALS
@@ -59,8 +48,6 @@ run_package_test() {
 
   echo "------------------------------------------------------------"
   echo "Configuring environment for: ${package_name}"
-  echo "Log file: ${log_file}"
-  echo "XML results: ${XML_OUTPUT_FILE}"
   echo "------------------------------------------------------------"
 
   case "${package_name}" in
@@ -95,86 +82,17 @@ run_package_test() {
   # Run the actual test
   pushd "${package_path}" > /dev/null
   set +e
-  # *** CRITICAL: system-single.sh MUST use XML_OUTPUT_FILE to generate the JUnit XML ***
-  "${system_test_script}" > "${log_file}" 2>&1
+  "${system_test_script}"
   local res=$?
   set -e
   popd > /dev/null
-
-  # Clean up isolated gcloud config
-  rm -rf "${CLOUDSDK_CONFIG}"
-
+  
   return $res
 }
 
 # A file for running system tests
 system_test_script="${PROJECT_ROOT}/.kokoro/system-single.sh"
-# Ensure KOKORO_ARTIFACTS_DIR is set, fallback to a local directory
-export KOKORO_ARTIFACTS_DIR="${KOKORO_ARTIFACTS_DIR:-${PROJECT_ROOT}/.artifacts}"
-
-# Parallel execution settings
-MAX_PARALLEL=4
-running_pids=()
-declare -A pid_to_pkg
-declare -A pid_to_resfile
-
-# Array to keep track of results for the final summary
-results=()
-handle_finished_job() {
-  local pid=$1
-  local pkg=${pid_to_pkg[$pid]}
-  local resfile=${pid_to_resfile[$pid]}
-  local pkg_log="${KOKORO_ARTIFACTS_DIR}/${pkg}/sponge_log.log"
-
-  # wait $pid might fail if it was already reaped by wait -n, 
-  # so we ignore its exit code and use the resfile.
-  wait "$pid" 2>/dev/null || true
-
-  local res=1
-  if [ -s "$resfile" ]; then
-    res=$(cat "$resfile")
-    rm "$resfile"
-  else
-    # If the file is empty or missing, the subshell crashed early
-    res=1
-  fi
-
-  echo "============================================================"
-  echo "System tests for ${pkg} finished (Exit code: ${res})"
-  echo "Artifacts in: ${KOKORO_ARTIFACTS_DIR}/${pkg}"
-  echo "============================================================"
 
-  # Print the package log to the console
-  if [ -f "${pkg_log}" ]; then
-    echo "--- Start of Logs for ${pkg} ---"
-    cat "${pkg_log}"
-    echo "--- End of Logs for ${pkg} ---"
-  else
-    echo "Console Fallback Warning: Log file not found at ${pkg_log}"
-  fi
-  echo ""
-
-  if [ -z "${res}" ] || [ "${res}" -ne 0 ]; then
-    echo "============================================================"
-    echo "FAIL-FAST: System tests for ${pkg} failed!"
-    echo "Cancelling all remaining running background jobs..."
-    echo "============================================================"
-    
-    # Kill all other active background processes
-    for active_pid in "${running_pids[@]}"; do
-      if [ "$active_pid" != "$pid" ] && kill -0 "$active_pid" 2>/dev/null; then
-        pkg_to_cancel=${pid_to_pkg[$active_pid]}
-        echo "Cancelling active system tests for ${pkg_to_cancel} (PID: ${active_pid})..."
-        # Send SIGTERM to allow graceful cleanup of resources if possible, or SIGKILL
-        kill -9 "$active_pid" 2>/dev/null || true
-      fi
-    done
-    
-    exit "${res}"
-  else
-    results+=("${pkg}: PASSED")
-  fi
-}
 # Run system tests for each package with directory packages/*/tests/system
 for path in `find 'packages' \
   \( -type d -wholename 'packages/*/tests/system' \) -o \
@@ -222,55 +140,10 @@ for path in `find 'packages' \
   set -e
 
   if [[ "${package_modified}" -gt 0 || "$KOKORO_BUILD_ARTIFACTS_SUBDIR" == *"continuous"* ]]; then
-      # Wait if we have reached MAX_PARALLEL
-      while [[ ${#running_pids[@]} -ge $MAX_PARALLEL ]]; do
-        set +e
-        wait -n
-        set -e
-        # Find which job finished
-        new_pids=()
-        for pid in "${running_pids[@]}"; do
-          if kill -0 "$pid" 2>/dev/null; then
-            new_pids+=("$pid")
-          else
-            handle_finished_job "$pid"
-          fi
-        done
-        running_pids=("${new_pids[@]}")
-      done
-
-      # Start the next test in the background
-      res_file=$(mktemp)
-      (
-        if run_package_test "$package_name"; then
-          echo 0 > "$res_file"
-        else
-          res=$?
-          echo $res > "$res_file"
-        fi
-      ) &
-      pid=$!
-      running_pids+=($pid)
-      pid_to_pkg[$pid]=$package_name
-      pid_to_resfile[$pid]=$res_file
-      echo "Started system tests for ${package_name} (PID: ${pid})"
+      # Call the function - its internal exports won't affect the next loop
+      run_package_test "$package_name" || RETVAL=$?
   else
       echo "No changes in ${package_name} and not a continuous build, skipping."
   fi
 done
-
-# Wait for all remaining jobs
-for pid in "${running_pids[@]}"; do
-  handle_finished_job "$pid"
-done
-
-echo "------------------------------------------------------------"
-echo "System Test Summary"
-echo "------------------------------------------------------------"
-for res in "${results[@]}"; do
-  echo "$res"
-done
-echo "------------------------------------------------------------"
-
 exit ${RETVAL}
-

From 6c1be8b5fe54f899d1d534b7dd543ea93dda2498 Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Mon, 6 Jul 2026 11:30:23 -0700
Subject: [PATCH 41/84] added github tag to test all packages

---
 .github/workflows/unittest.yml                | 30 ++++++++++++++++---
 ci/get_package_shards.py                      |  4 +++
 packages/google-backstory/SHARD_TEST.txt      |  1 -
 .../google-cloud-apiregistry/SHARD_TEST.txt   |  1 -
 packages/google-cloud-bigtable/SHARD_TEST.txt |  1 -
 packages/google-cloud-ces/SHARD_TEST.txt      |  1 -
 .../google-cloud-chronicle/SHARD_TEST.txt     |  1 -
 packages/google-cloud-dataproc/SHARD_TEST.txt |  1 -
 .../google-cloud-firestore/SHARD_TEST.txt     |  1 -
 .../SHARD_TEST.txt                            |  1 -
 .../google-cloud-managedkafka/SHARD_TEST.txt  |  1 -
 .../google-cloud-optimization/SHARD_TEST.txt  |  1 -
 packages/google-cloud-retail/SHARD_TEST.txt   |  1 -
 packages/sqlalchemy-spanner/SHARD_TEST.txt    |  1 -
 14 files changed, 30 insertions(+), 16 deletions(-)
 delete mode 100644 packages/google-backstory/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-apiregistry/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-bigtable/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-ces/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-chronicle/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-dataproc/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-firestore/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-gke-multicloud/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-managedkafka/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-optimization/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-retail/SHARD_TEST.txt
 delete mode 100644 packages/sqlalchemy-spanner/SHARD_TEST.txt

diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml
index 3d347fcc017a..a43636f0aead 100644
--- a/.github/workflows/unittest.yml
+++ b/.github/workflows/unittest.yml
@@ -17,11 +17,20 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
+      is_full_run: ${{ steps.check-label.outputs.is_full_run }}
     steps:
     - name: Checkout
       uses: actions/checkout@v4
       with:
         fetch-depth: 0
+    - name: Check for unit_test:all_packages label
+      id: check-label
+      run: |
+        if [[ "${{ contains(github.event.pull_request.labels.*.name, 'unit_test:all_packages') }}" == "true" ]]; then
+          echo "is_full_run=true" >> $GITHUB_OUTPUT
+        else
+          echo "is_full_run=false" >> $GITHUB_OUTPUT
+        fi
     - name: Setup Python
       uses: actions/setup-python@v5
       with:
@@ -31,6 +40,7 @@ jobs:
       env:
         BUILD_TYPE: presubmit
         TARGET_BRANCH: ${{ github.base_ref || github.event.merge_group.base_ref }}
+        TEST_ALL_PACKAGES: ${{ steps.check-label.outputs.is_full_run }}
       run: |
         if [ -n "$TARGET_BRANCH" ]; then
           git fetch origin "$TARGET_BRANCH" --deepen=200 || true
@@ -102,6 +112,7 @@ jobs:
     runs-on: ubuntu-latest
     needs:
         - unit
+        - initialize
     steps:
     - name: Checkout
       uses: actions/checkout@v4
@@ -116,10 +127,16 @@ jobs:
         python-version: "3.10"
     - name: Set number of files changes in packages directory
       id: packages
+      env:
+        TEST_ALL_PACKAGES: ${{ needs.initialize.outputs.is_full_run }}
       run: |
-        git diff HEAD~1 -- packages > /dev/null
-        num_files_changed=$(git diff HEAD~1 -- packages | wc -l | tr -d ' ')
-        echo "num_files_changed=${num_files_changed}" >> "$GITHUB_OUTPUT"
+        if [[ "${TEST_ALL_PACKAGES}" == "true" ]]; then
+          echo "num_files_changed=1" >> "$GITHUB_OUTPUT"
+        else
+          git diff HEAD~1 -- packages > /dev/null
+          num_files_changed=$(git diff HEAD~1 -- packages | wc -l | tr -d ' ')
+          echo "num_files_changed=${num_files_changed}" >> "$GITHUB_OUTPUT"
+        fi
     - name: Install coverage
       if: ${{ steps.packages.outputs.num_files_changed > 0 }}
       run: |
@@ -136,6 +153,7 @@ jobs:
         # TODO: default to 100% coverage after next gapic-generator release
         # https://github.com/googleapis/google-cloud-python/issues/17459
         DEFAULT_FAIL_UNDER: 99
+        TEST_ALL_PACKAGES: ${{ needs.initialize.outputs.is_full_run }}
       run: |
         if [ -d .coverage-results ]; then
           # Unzip any zipped coverage results
@@ -152,7 +170,11 @@ jobs:
           fi
 
           # Find all modified packages
-          modified_packages=$(git diff --name-only HEAD~1 -- packages | cut -d/ -f1,2 | sort -u)
+          if [[ "${TEST_ALL_PACKAGES}" == "true" ]]; then
+            modified_packages=$(ls -d packages/*/ | cut -d/ -f1,2 | sort -u)
+          else
+            modified_packages=$(git diff --name-only HEAD~1 -- packages | cut -d/ -f1,2 | sort -u)
+          fi
 
           failed_packages=()
           passed_packages=()
diff --git a/ci/get_package_shards.py b/ci/get_package_shards.py
index a5599146abd5..a7cc621abcb7 100644
--- a/ci/get_package_shards.py
+++ b/ci/get_package_shards.py
@@ -18,9 +18,13 @@ def get_packages():
 def get_packages_to_test():
     build_type = os.environ.get('BUILD_TYPE', 'presubmit')
     target_branch = os.environ.get('TARGET_BRANCH', 'main')
+    test_all_packages = os.environ.get('TEST_ALL_PACKAGES', 'false').lower() == 'true'
 
     all_packages = get_packages()
 
+    if test_all_packages:
+        return all_packages
+
     if build_type == 'presubmit':
         git_diff_arg = f"origin/{target_branch}..."
     elif build_type == 'continuous':
diff --git a/packages/google-backstory/SHARD_TEST.txt b/packages/google-backstory/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-backstory/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-apiregistry/SHARD_TEST.txt b/packages/google-cloud-apiregistry/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-apiregistry/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-bigtable/SHARD_TEST.txt b/packages/google-cloud-bigtable/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-bigtable/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-ces/SHARD_TEST.txt b/packages/google-cloud-ces/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-ces/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-chronicle/SHARD_TEST.txt b/packages/google-cloud-chronicle/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-chronicle/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-dataproc/SHARD_TEST.txt b/packages/google-cloud-dataproc/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-dataproc/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-firestore/SHARD_TEST.txt b/packages/google-cloud-firestore/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-firestore/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-gke-multicloud/SHARD_TEST.txt b/packages/google-cloud-gke-multicloud/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-gke-multicloud/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-managedkafka/SHARD_TEST.txt b/packages/google-cloud-managedkafka/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-managedkafka/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-optimization/SHARD_TEST.txt b/packages/google-cloud-optimization/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-optimization/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/google-cloud-retail/SHARD_TEST.txt b/packages/google-cloud-retail/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/google-cloud-retail/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding
diff --git a/packages/sqlalchemy-spanner/SHARD_TEST.txt b/packages/sqlalchemy-spanner/SHARD_TEST.txt
deleted file mode 100644
index b7ac1b35614f..000000000000
--- a/packages/sqlalchemy-spanner/SHARD_TEST.txt
+++ /dev/null
@@ -1 +0,0 @@
-Test change for sharding

From 8fb68d3a886ab88fb3b5d185ce3d7ceaa8d1086a Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Mon, 6 Jul 2026 11:37:32 -0700
Subject: [PATCH 42/84] re-run when label added

---
 .github/workflows/unittest.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml
index a43636f0aead..efe426fc157d 100644
--- a/.github/workflows/unittest.yml
+++ b/.github/workflows/unittest.yml
@@ -1,5 +1,6 @@
 on:
   pull_request:
+    types: [opened, synchronize, reopened, labeled, unlabeled]
     branches:
       - main
       - preview
@@ -15,6 +16,7 @@ permissions:
 jobs:
   initialize:
     runs-on: ubuntu-latest
+    if: github.event.action != 'labeled' || github.event.label.name == 'unit_test:all_packages'
     outputs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
       is_full_run: ${{ steps.check-label.outputs.is_full_run }}

From fb908604dfbc2bdc5bc0662f1b47d30f12332eef Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Mon, 6 Jul 2026 11:48:19 -0700
Subject: [PATCH 43/84] don't trigger on label changes

---
 .github/workflows/unittest.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml
index efe426fc157d..a43636f0aead 100644
--- a/.github/workflows/unittest.yml
+++ b/.github/workflows/unittest.yml
@@ -1,6 +1,5 @@
 on:
   pull_request:
-    types: [opened, synchronize, reopened, labeled, unlabeled]
     branches:
       - main
       - preview
@@ -16,7 +15,6 @@ permissions:
 jobs:
   initialize:
     runs-on: ubuntu-latest
-    if: github.event.action != 'labeled' || github.event.label.name == 'unit_test:all_packages'
     outputs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
       is_full_run: ${{ steps.check-label.outputs.is_full_run }}

From 6123e7a2965888aef19cc6db03288efbb1d28dd1 Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Mon, 6 Jul 2026 12:22:47 -0700
Subject: [PATCH 44/84] revered unrelated changes

---
 packages/google-cloud-firestore/.coveragerc |  1 -
 packages/sqlalchemy-spanner/.coveragerc     | 36 ---------------------
 packages/sqlalchemy-spanner/noxfile.py      | 14 +-------
 3 files changed, 1 insertion(+), 50 deletions(-)
 delete mode 100644 packages/sqlalchemy-spanner/.coveragerc

diff --git a/packages/google-cloud-firestore/.coveragerc b/packages/google-cloud-firestore/.coveragerc
index 66bbde8955a9..4c355f6455c1 100644
--- a/packages/google-cloud-firestore/.coveragerc
+++ b/packages/google-cloud-firestore/.coveragerc
@@ -2,7 +2,6 @@
 branch = True
 
 [report]
-fail_under = 99
 show_missing = True
 omit =
     google/cloud/firestore/__init__.py
diff --git a/packages/sqlalchemy-spanner/.coveragerc b/packages/sqlalchemy-spanner/.coveragerc
deleted file mode 100644
index d09941ce0eb9..000000000000
--- a/packages/sqlalchemy-spanner/.coveragerc
+++ /dev/null
@@ -1,36 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-# Copyright 2026 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-[run]
-branch = True
-omit =
-  google/__init__.py
-  google/cloud/sqlalchemy_spanner/requirements.py
-
-[report]
-fail_under = 0
-show_missing = True
-exclude_lines =
-    # Re-enable the standard pragma
-    pragma: NO COVER
-    # Ignore debug-only repr
-    def __repr__
-    # Ignore abstract methods
-    raise NotImplementedError
-omit =
-  */site-packages/*.py
-  google/__init__.py
-  google/cloud/sqlalchemy_spanner/requirements.py
diff --git a/packages/sqlalchemy-spanner/noxfile.py b/packages/sqlalchemy-spanner/noxfile.py
index 8adeb9388354..187d4e7e8924 100644
--- a/packages/sqlalchemy-spanner/noxfile.py
+++ b/packages/sqlalchemy-spanner/noxfile.py
@@ -85,7 +85,6 @@ class = StreamHandler
 UNIT_TEST_STANDARD_DEPENDENCIES = [
     "mock",
     "pytest",
-    "pytest-cov",
 ]
 
 UNIT_TEST_EXTERNAL_DEPENDENCIES = [
@@ -369,18 +368,7 @@ def unit(session, test_type):
             *UNIT_TEST_DEPENDENCIES,
         )
         session.install(".")
-        session.run(
-            "py.test",
-            "--quiet",
-            "--cov=google.cloud.sqlalchemy_spanner",
-            "--cov=tests/unit",
-            "--cov-append",
-            "--cov-config=.coveragerc",
-            "--cov-report=",
-            "--cov-fail-under=0",
-            os.path.join("tests/unit"),
-            *session.posargs,
-        )
+        session.run("py.test", "--quiet", os.path.join("tests/unit"), *session.posargs)
         return
 
 

From c99a693be64913b2e127107af6b3aa60af6f4d20 Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Mon, 6 Jul 2026 12:27:23 -0700
Subject: [PATCH 45/84] increased shard count

---
 ci/get_package_shards.py | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/ci/get_package_shards.py b/ci/get_package_shards.py
index a7cc621abcb7..342d29dc6adb 100644
--- a/ci/get_package_shards.py
+++ b/ci/get_package_shards.py
@@ -1,3 +1,17 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import os
 import subprocess
 import json
@@ -61,8 +75,8 @@ def group_packages(packages):
     # Ensure at least 1 shard if we have packages
     num_shards = max(1, num_shards)
 
-    # 3. Top out at 10 shards
-    num_shards = min(10, num_shards)
+    # 3. Top out at 16 shards
+    num_shards = min(16, num_shards)
 
     # Distribute packages between them as evenly as possible
     shard_size = math.ceil(num_packages / num_shards)

From ee7cabcda9a589a531d32353e6e5c282037220c3 Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Mon, 6 Jul 2026 12:35:03 -0700
Subject: [PATCH 46/84] isolate specific packages

---
 ci/get_package_shards.py | 109 +++++++++++++++++++++++++++------------
 1 file changed, 76 insertions(+), 33 deletions(-)

diff --git a/ci/get_package_shards.py b/ci/get_package_shards.py
index 342d29dc6adb..5d4dd0bf1447 100644
--- a/ci/get_package_shards.py
+++ b/ci/get_package_shards.py
@@ -18,6 +18,16 @@
 import math
 import sys
 
+# Define set of long-running unit tests to run in isolated shard
+ISOLATED_PACKAGES = {
+    "google-cloud-compute",
+    "google-cloud-compute-v1beta",
+    "google-cloud-dialogflow",
+    "google-cloud-dialogflow-cx",
+    "google-cloud-retail",
+}
+
+
 def get_packages():
     subdirs = ['packages']
     packages = []
@@ -60,50 +70,83 @@ def get_packages_to_test():
 
     return to_test
 
+
 def group_packages(packages):
     if not packages:
         return []
 
-    num_packages = len(packages)
-
-    # 1. Only shard if > 10 packages are being touched
-    # 2. Only add a new shard if any shard would have > 10 packages.
-    # To guarantee that no shard contains more than 10 packages (when distributed evenly),
-    # we need S >= N / 10, which means S = ceil(N / 10).
-    num_shards = math.ceil(num_packages / 10)
-
-    # Ensure at least 1 shard if we have packages
-    num_shards = max(1, num_shards)
+    isolated_to_test = []
+    normal_to_test = []
 
-    # 3. Top out at 16 shards
-    num_shards = min(16, num_shards)
-
-    # Distribute packages between them as evenly as possible
-    shard_size = math.ceil(num_packages / num_shards)
+    for pkg in packages:
+        pkg_name = pkg.strip('/').split('/')[-1]
+        if pkg_name in ISOLATED_PACKAGES:
+            isolated_to_test.append(pkg)
+        else:
+            normal_to_test.append(pkg)
 
     shards = []
-    for i in range(num_shards):
-        start = i * shard_size
-        end = min((i + 1) * shard_size, num_packages)
-        if start >= num_packages:
-            break
-        shard_packages = packages[start:end]
-        index = i + 1
-        name = f"Shard {index}"
-        num_in_shard = len(shard_packages)
-        # Calculate a descriptive range for step visibility
-        if len(shard_packages) == 1:
-            desc = shard_packages[0].strip('/').split('/')[-1]
-        else:
-            desc = f"{shard_packages[0].strip('/').split('/')[-1]}...{shard_packages[-1].strip('/').split('/')[-1]} ({num_in_shard} packages)"
+    index = 1
 
+    # Add isolated packages to their own shards
+    for pkg in isolated_to_test:
+        pkg_name = pkg.strip('/').split('/')[-1]
+        clean_name = pkg_name.replace("google-cloud-", "")
         shards.append({
-            "name": name,
+            "name": clean_name,
             "index": index,
-            "description": desc,
-            "packages": " ".join(shard_packages),
-            "is_sharded": num_shards > 1
+            "description": pkg_name,
+            "packages": pkg,
+            "is_sharded": True
         })
+        index += 1
+
+    # Group the remaining packages
+    if normal_to_test:
+        num_packages = len(normal_to_test)
+        
+        # 1. Only shard if > 10 packages are being touched
+        # 2. Only add a new shard if any shard would have > 10 packages.
+        # To guarantee that no shard contains more than 10 packages (when distributed evenly),
+        # we need S >= N / 10, which means S = ceil(N / 10).
+        num_shards = math.ceil(num_packages / 10)
+        
+        # Ensure at least 1 shard if we have packages
+        num_shards = max(1, num_shards)
+        
+        # 3. Top out at 16 shards
+        num_shards = min(16, num_shards)
+        
+        # Distribute packages between them as evenly as possible
+        shard_size = math.ceil(num_packages / num_shards)
+
+        for i in range(num_shards):
+            start = i * shard_size
+            end = min((i + 1) * shard_size, num_packages)
+            if start >= num_packages:
+                break
+            shard_packages = normal_to_test[start:end]
+            name = f"Shard {index}"
+            num_in_shard = len(shard_packages)
+            if len(shard_packages) == 1:
+                desc = shard_packages[0].strip('/').split('/')[-1]
+            else:
+                desc = f"{shard_packages[0].strip('/').split('/')[-1]}...{shard_packages[-1].strip('/').split('/')[-1]} ({num_in_shard} packages)"
+
+            shards.append({
+                "name": name,
+                "index": index,
+                "description": desc,
+                "packages": " ".join(shard_packages),
+                "is_sharded": True
+            })
+            index += 1
+
+    # Set is_sharded dynamically based on the total number of shards
+    total_shards = len(shards)
+    for shard in shards:
+        shard["is_sharded"] = total_shards > 1
+
     return shards
 
 if __name__ == "__main__":

From 40d31e357608aa1e7890df103d7ab17202d10bdb Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Mon, 6 Jul 2026 13:12:41 -0700
Subject: [PATCH 47/84] isolated shards should be included in max_shard count

---
 ci/get_package_shards.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/ci/get_package_shards.py b/ci/get_package_shards.py
index 5d4dd0bf1447..5fca3c977429 100644
--- a/ci/get_package_shards.py
+++ b/ci/get_package_shards.py
@@ -24,7 +24,6 @@
     "google-cloud-compute-v1beta",
     "google-cloud-dialogflow",
     "google-cloud-dialogflow-cx",
-    "google-cloud-retail",
 }
 
 
@@ -114,8 +113,9 @@ def group_packages(packages):
         # Ensure at least 1 shard if we have packages
         num_shards = max(1, num_shards)
         
-        # 3. Top out at 16 shards
-        num_shards = min(16, num_shards)
+        # 3. Top out at the remaining budget (total 16 shards minus isolated shards)
+        max_normal_shards = max(1, 16 - len(isolated_to_test))
+        num_shards = min(max_normal_shards, num_shards)
         
         # Distribute packages between them as evenly as possible
         shard_size = math.ceil(num_packages / num_shards)
@@ -126,7 +126,7 @@ def group_packages(packages):
             if start >= num_packages:
                 break
             shard_packages = normal_to_test[start:end]
-            name = f"Shard {index}"
+            name = f"Shard {i + 1}"
             num_in_shard = len(shard_packages)
             if len(shard_packages) == 1:
                 desc = shard_packages[0].strip('/').split('/')[-1]

From 89e549918ebb2c174d2c1674bf6bb49a3c4d1711 Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Mon, 6 Jul 2026 13:16:30 -0700
Subject: [PATCH 48/84] changed shard names

---
 ci/get_package_shards.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ci/get_package_shards.py b/ci/get_package_shards.py
index 5fca3c977429..7c458b83a50a 100644
--- a/ci/get_package_shards.py
+++ b/ci/get_package_shards.py
@@ -92,7 +92,7 @@ def group_packages(packages):
         pkg_name = pkg.strip('/').split('/')[-1]
         clean_name = pkg_name.replace("google-cloud-", "")
         shards.append({
-            "name": clean_name,
+            "name": f"Shard {index}: {clean_name}",
             "index": index,
             "description": pkg_name,
             "packages": pkg,
@@ -126,7 +126,7 @@ def group_packages(packages):
             if start >= num_packages:
                 break
             shard_packages = normal_to_test[start:end]
-            name = f"Shard {i + 1}"
+            name = f"Shard {index}"
             num_in_shard = len(shard_packages)
             if len(shard_packages) == 1:
                 desc = shard_packages[0].strip('/').split('/')[-1]

From a1fab952cfc5e6999efa3880413670b478529669 Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Mon, 6 Jul 2026 14:19:16 -0700
Subject: [PATCH 49/84] use pacakge weights instead of isolation

---
 ci/get_package_shards.py | 121 +++++++++++++++++++--------------------
 1 file changed, 59 insertions(+), 62 deletions(-)

diff --git a/ci/get_package_shards.py b/ci/get_package_shards.py
index 7c458b83a50a..fe11de590829 100644
--- a/ci/get_package_shards.py
+++ b/ci/get_package_shards.py
@@ -18,12 +18,17 @@
 import math
 import sys
 
-# Define set of long-running unit tests to run in isolated shard
-ISOLATED_PACKAGES = {
-    "google-cloud-compute",
-    "google-cloud-compute-v1beta",
-    "google-cloud-dialogflow",
-    "google-cloud-dialogflow-cx",
+# Define weights for long-running packages to balance shard execution time.
+# Default weight for any package not listed is 1.
+PACKAGE_WEIGHTS = {
+    "bigframes": 6,
+    "google-auth": 5,
+    "google-cloud-compute": 12,
+    "google-cloud-compute-v1beta": 12,
+    "google-cloud-dialogflow": 6,
+    "google-cloud-dialogflow-cx": 6,
+    "google-cloud-discoveryengine": 8,
+    "google-cloud-retail": 5,
 }
 
 
@@ -38,6 +43,7 @@ def get_packages():
         packages.extend(sorted(pkg_dirs))
     return packages
 
+
 def get_packages_to_test():
     build_type = os.environ.get('BUILD_TYPE', 'presubmit')
     target_branch = os.environ.get('TARGET_BRANCH', 'main')
@@ -74,74 +80,64 @@ def group_packages(packages):
     if not packages:
         return []
 
-    isolated_to_test = []
-    normal_to_test = []
-
+    # Map packages to their weights
+    package_weights = []
+    total_weight = 0
     for pkg in packages:
         pkg_name = pkg.strip('/').split('/')[-1]
-        if pkg_name in ISOLATED_PACKAGES:
-            isolated_to_test.append(pkg)
-        else:
-            normal_to_test.append(pkg)
+        weight = PACKAGE_WEIGHTS.get(pkg_name, 1)
+        package_weights.append((pkg, weight))
+        total_weight += weight
+
+    # Determine number of shards based on total weight
+    # 1. Base shard count on weight capacity of 10 per shard
+    num_shards = math.ceil(total_weight / 10)
+
+    # Ensure at least 1 shard if we have packages
+    num_shards = max(1, num_shards)
+
+    # 2. Top out at 16 shards
+    num_shards = min(16, num_shards)
+
+    # Initialize shards as empty lists of packages with their current total weight
+    shard_buckets = [{"packages": [], "weight": 0} for _ in range(num_shards)]
+
+    # Sort packages descending by weight (LPT heuristic)
+    # If weights are equal, sort alphabetically for stability
+    sorted_packages = sorted(package_weights, key=lambda x: (-x[1], x[0]))
 
+    # Distribute greedily to the bucket with the minimum current weight
+    for pkg, weight in sorted_packages:
+        min_bucket = min(shard_buckets, key=lambda b: b["weight"])
+        min_bucket["packages"].append(pkg)
+        min_bucket["weight"] += weight
+
+    # Construct the final shards output list
     shards = []
     index = 1
+    for bucket in shard_buckets:
+        shard_packages = bucket["packages"]
+        if not shard_packages:
+            continue
+        name = f"Shard {index}"
+        num_in_shard = len(shard_packages)
+
+        # Sort packages alphabetically for a cleaner description
+        sorted_shard_pkgs = sorted(shard_packages)
+        if len(sorted_shard_pkgs) == 1:
+            desc = sorted_shard_pkgs[0].strip('/').split('/')[-1]
+        else:
+            desc = f"{sorted_shard_pkgs[0].strip('/').split('/')[-1]}...{sorted_shard_pkgs[-1].strip('/').split('/')[-1]} ({num_in_shard} packages)"
 
-    # Add isolated packages to their own shards
-    for pkg in isolated_to_test:
-        pkg_name = pkg.strip('/').split('/')[-1]
-        clean_name = pkg_name.replace("google-cloud-", "")
         shards.append({
-            "name": f"Shard {index}: {clean_name}",
+            "name": name,
             "index": index,
-            "description": pkg_name,
-            "packages": pkg,
+            "description": desc,
+            "packages": " ".join(shard_packages),
             "is_sharded": True
         })
         index += 1
 
-    # Group the remaining packages
-    if normal_to_test:
-        num_packages = len(normal_to_test)
-        
-        # 1. Only shard if > 10 packages are being touched
-        # 2. Only add a new shard if any shard would have > 10 packages.
-        # To guarantee that no shard contains more than 10 packages (when distributed evenly),
-        # we need S >= N / 10, which means S = ceil(N / 10).
-        num_shards = math.ceil(num_packages / 10)
-        
-        # Ensure at least 1 shard if we have packages
-        num_shards = max(1, num_shards)
-        
-        # 3. Top out at the remaining budget (total 16 shards minus isolated shards)
-        max_normal_shards = max(1, 16 - len(isolated_to_test))
-        num_shards = min(max_normal_shards, num_shards)
-        
-        # Distribute packages between them as evenly as possible
-        shard_size = math.ceil(num_packages / num_shards)
-
-        for i in range(num_shards):
-            start = i * shard_size
-            end = min((i + 1) * shard_size, num_packages)
-            if start >= num_packages:
-                break
-            shard_packages = normal_to_test[start:end]
-            name = f"Shard {index}"
-            num_in_shard = len(shard_packages)
-            if len(shard_packages) == 1:
-                desc = shard_packages[0].strip('/').split('/')[-1]
-            else:
-                desc = f"{shard_packages[0].strip('/').split('/')[-1]}...{shard_packages[-1].strip('/').split('/')[-1]} ({num_in_shard} packages)"
-
-            shards.append({
-                "name": name,
-                "index": index,
-                "description": desc,
-                "packages": " ".join(shard_packages),
-                "is_sharded": True
-            })
-            index += 1
-
     # Set is_sharded dynamically based on the total number of shards
     total_shards = len(shards)
     for shard in shards:
@@ -149,6 +145,7 @@ def group_packages(packages):
 
     return shards
 
+
 if __name__ == "__main__":
     packages = get_packages_to_test()
     shards = group_packages(packages)

From 92626bd5815b50fabb7569f9e8d73bb730c86665 Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Mon, 6 Jul 2026 14:23:17 -0700
Subject: [PATCH 50/84] parallelize final cover check

---
 .github/workflows/unittest.yml | 61 ++++++++++++++++++++++++----------
 1 file changed, 43 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml
index a43636f0aead..2136140561a1 100644
--- a/.github/workflows/unittest.yml
+++ b/.github/workflows/unittest.yml
@@ -176,34 +176,59 @@ jobs:
             modified_packages=$(git diff --name-only HEAD~1 -- packages | cut -d/ -f1,2 | sort -u)
           fi
 
+          mkdir -p /tmp/coverage-logs
+          MAX_JOBS=8
+
+          for pkg in ${modified_packages}; do
+            if [ -d "${pkg}" ]; then
+              # Limit the number of concurrent background jobs
+              while [ $(jobs -r | wc -l) -ge ${MAX_JOBS} ]; do
+                sleep 0.1
+              done
+
+              (
+                pkg_log="/tmp/coverage-logs/${pkg//\//_}.log"
+                pkg_status="/tmp/coverage-logs/${pkg//\//_}.status"
+
+                set +e
+                if [ -f "${pkg}/.coveragerc" ]; then
+                  echo "Using package-specific configuration: ${pkg}/.coveragerc" > "${pkg_log}"
+                  if grep -q "fail_under" "${pkg}/.coveragerc"; then
+                    coverage report --rcfile="${pkg}/.coveragerc" --include="${pkg}/**" >> "${pkg_log}" 2>&1
+                  else
+                    echo "No fail_under specified in ${pkg}/.coveragerc, enforcing default" >> "${pkg_log}"
+                    coverage report --rcfile="${pkg}/.coveragerc" --include="${pkg}/**" --fail-under="${DEFAULT_FAIL_UNDER}" >> "${pkg_log}" 2>&1
+                  fi
+                else
+                  echo "No .coveragerc found for ${pkg}, enforcing default" > "${pkg_log}"
+                  coverage report --include="${pkg}/**" --fail-under="${DEFAULT_FAIL_UNDER}" >> "${pkg_log}" 2>&1
+                fi
+                echo $? > "${pkg_status}"
+                set -e
+              ) &
+            fi
+          done
+
+          # Wait for all background checks to finish
+          wait
+
           failed_packages=()
           passed_packages=()
 
           for pkg in ${modified_packages}; do
             if [ -d "${pkg}" ]; then
+              pkg_log="/tmp/coverage-logs/${pkg//\//_}.log"
+              pkg_status="/tmp/coverage-logs/${pkg//\//_}.status"
+
               echo "============================================================"
               echo "Evaluating coverage for package: ${pkg}"
               echo "============================================================"
-
-              set +e
-              if [ -f "${pkg}/.coveragerc" ]; then
-                echo "Using package-specific configuration: ${pkg}/.coveragerc"
-                # If fail_under is specified in the package-specific .coveragerc, coverage report
-                # will automatically enforce it. Otherwise, we enforce the default.
-                if grep -q "fail_under" "${pkg}/.coveragerc"; then
-                  coverage report --rcfile="${pkg}/.coveragerc" --include="${pkg}/**"
-                else
-                  echo "No fail_under specified in ${pkg}/.coveragerc, enforcing default"
-                  coverage report --rcfile="${pkg}/.coveragerc" --include="${pkg}/**" --fail-under="${DEFAULT_FAIL_UNDER}"
-                fi
-              else
-                echo "No .coveragerc found for ${pkg}, enforcing default"
-                coverage report --include="${pkg}/**" --fail-under="${DEFAULT_FAIL_UNDER}"
+              if [ -f "${pkg_log}" ]; then
+                cat "${pkg_log}"
               fi
-              status=$?
-              set -e
 
-              if [ ${status} -ne 0 ]; then
+              status=$(cat "${pkg_status}")
+              if [ "${status}" -ne 0 ]; then
                 failed_packages+=("${pkg}")
               else
                 passed_packages+=("${pkg}")

From 8c4d2ddf05dca98a39acdc615f8cba74e2e671ec Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Mon, 6 Jul 2026 14:25:39 -0700
Subject: [PATCH 51/84] resolve merge conflict

---
 .github/workflows/unittest.yml | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml
index 2136140561a1..4dbef14740f5 100644
--- a/.github/workflows/unittest.yml
+++ b/.github/workflows/unittest.yml
@@ -191,19 +191,21 @@ jobs:
                 pkg_status="/tmp/coverage-logs/${pkg//\//_}.status"
 
                 set +e
-                if [ -f "${pkg}/.coveragerc" ]; then
+                pushd "${pkg}" > /dev/null
+                if [ -f ".coveragerc" ]; then
                   echo "Using package-specific configuration: ${pkg}/.coveragerc" > "${pkg_log}"
-                  if grep -q "fail_under" "${pkg}/.coveragerc"; then
-                    coverage report --rcfile="${pkg}/.coveragerc" --include="${pkg}/**" >> "${pkg_log}" 2>&1
+                  if grep -q "fail_under" ".coveragerc"; then
+                    COVERAGE_FILE=../../.coverage coverage report --rcfile=".coveragerc" --include="$PWD/**" >> "${pkg_log}" 2>&1
                   else
                     echo "No fail_under specified in ${pkg}/.coveragerc, enforcing default" >> "${pkg_log}"
-                    coverage report --rcfile="${pkg}/.coveragerc" --include="${pkg}/**" --fail-under="${DEFAULT_FAIL_UNDER}" >> "${pkg_log}" 2>&1
+                    COVERAGE_FILE=../../.coverage coverage report --rcfile=".coveragerc" --include="$PWD/**" --fail-under="${DEFAULT_FAIL_UNDER}" >> "${pkg_log}" 2>&1
                   fi
                 else
                   echo "No .coveragerc found for ${pkg}, enforcing default" > "${pkg_log}"
-                  coverage report --include="${pkg}/**" --fail-under="${DEFAULT_FAIL_UNDER}" >> "${pkg_log}" 2>&1
+                  COVERAGE_FILE=../../.coverage coverage report --include="$PWD/**" --fail-under="${DEFAULT_FAIL_UNDER}" >> "${pkg_log}" 2>&1
                 fi
                 echo $? > "${pkg_status}"
+                popd > /dev/null
                 set -e
               ) &
             fi

From 2e9bafce26377016a74ae551eb5a34972d9f170e Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Mon, 6 Jul 2026 15:11:57 -0700
Subject: [PATCH 52/84] added additional weights

---
 ci/get_package_shards.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/ci/get_package_shards.py b/ci/get_package_shards.py
index fe11de590829..aae208b5dd5f 100644
--- a/ci/get_package_shards.py
+++ b/ci/get_package_shards.py
@@ -22,6 +22,7 @@
 # Default weight for any package not listed is 1.
 PACKAGE_WEIGHTS = {
     "bigframes": 6,
+    "google-ai-generativelanguage": 4
     "google-auth": 5,
     "google-cloud-compute": 12,
     "google-cloud-compute-v1beta": 12,
@@ -29,6 +30,8 @@
     "google-cloud-dialogflow-cx": 6,
     "google-cloud-discoveryengine": 8,
     "google-cloud-retail": 5,
+    "google-shopping-merchant-accounts": 4,
+
 }
 
 

From f14d489f3d1f71962ebe36a84f305718ea0222dd Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Mon, 6 Jul 2026 15:18:01 -0700
Subject: [PATCH 53/84] pack shards alphabetically

---
 ci/get_package_shards.py | 63 +++++++++++++++++++---------------------
 1 file changed, 30 insertions(+), 33 deletions(-)

diff --git a/ci/get_package_shards.py b/ci/get_package_shards.py
index aae208b5dd5f..f7172fdb9a2c 100644
--- a/ci/get_package_shards.py
+++ b/ci/get_package_shards.py
@@ -92,45 +92,43 @@ def group_packages(packages):
         package_weights.append((pkg, weight))
         total_weight += weight
 
-    # Determine number of shards based on total weight
-    # 1. Base shard count on weight capacity of 10 per shard
-    num_shards = math.ceil(total_weight / 10)
-
-    # Ensure at least 1 shard if we have packages
-    num_shards = max(1, num_shards)
-
-    # 2. Top out at 16 shards
-    num_shards = min(16, num_shards)
-
-    # Initialize shards as empty lists of packages with their current total weight
-    shard_buckets = [{"packages": [], "weight": 0} for _ in range(num_shards)]
-
-    # Sort packages descending by weight (LPT heuristic)
-    # If weights are equal, sort alphabetically for stability
-    sorted_packages = sorted(package_weights, key=lambda x: (-x[1], x[0]))
+    # Dynamically determine target weight to balance across max 16 shards.
+    # 1. Base shard count has a target weight capacity of 10 per shard.
+    # 2. To avoid exceeding 16 shards, we scale the target weight dynamically
+    #    based on the total workload.
+    max_shards = 16
+    target_weight = max(10, math.ceil(total_weight / max_shards))
+
+    shards_list = []
+    current_shard_packages = []
+    current_shard_weight = 0
+
+    # Pack packages alphabetically and contiguously.
+    for pkg, weight in package_weights:
+        # Check if adding this package would exceed the target weight
+        if current_shard_packages and (current_shard_weight + weight > target_weight):
+            shards_list.append(current_shard_packages)
+            current_shard_packages = [pkg]
+            current_shard_weight = weight
+        else:
+            current_shard_packages.append(pkg)
+            current_shard_weight += weight
 
-    # Distribute greedily to the bucket with the minimum current weight
-    for pkg, weight in sorted_packages:
-        min_bucket = min(shard_buckets, key=lambda b: b["weight"])
-        min_bucket["packages"].append(pkg)
-        min_bucket["weight"] += weight
+    if current_shard_packages:
+        shards_list.append(current_shard_packages)
 
     # Construct the final shards output list
     shards = []
-    index = 1
-    for bucket in shard_buckets:
-        shard_packages = bucket["packages"]
-        if not shard_packages:
-            continue
+    for i, shard_packages in enumerate(shards_list):
+        index = i + 1
         name = f"Shard {index}"
         num_in_shard = len(shard_packages)
-
-        # Sort packages alphabetically for a cleaner description
-        sorted_shard_pkgs = sorted(shard_packages)
-        if len(sorted_shard_pkgs) == 1:
-            desc = sorted_shard_pkgs[0].strip('/').split('/')[-1]
+        
+        # Calculate contiguous range description
+        if len(shard_packages) == 1:
+            desc = shard_packages[0].strip('/').split('/')[-1]
         else:
-            desc = f"{sorted_shard_pkgs[0].strip('/').split('/')[-1]}...{sorted_shard_pkgs[-1].strip('/').split('/')[-1]} ({num_in_shard} packages)"
+            desc = f"{shard_packages[0].strip('/').split('/')[-1]}...{shard_packages[-1].strip('/').split('/')[-1]} ({num_in_shard} packages)"
 
         shards.append({
             "name": name,
@@ -139,7 +137,6 @@ def group_packages(packages):
             "packages": " ".join(shard_packages),
             "is_sharded": True
         })
-        index += 1
 
     # Set is_sharded dynamically based on the total number of shards
     total_shards = len(shards)

From e4495b88bc7d82e19913131f4b22036eab330b08 Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Mon, 6 Jul 2026 16:45:48 -0700
Subject: [PATCH 54/84] fixed typo

---
 ci/get_package_shards.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/get_package_shards.py b/ci/get_package_shards.py
index f7172fdb9a2c..62ad719a3ae9 100644
--- a/ci/get_package_shards.py
+++ b/ci/get_package_shards.py
@@ -22,7 +22,7 @@
 # Default weight for any package not listed is 1.
 PACKAGE_WEIGHTS = {
     "bigframes": 6,
-    "google-ai-generativelanguage": 4
+    "google-ai-generativelanguage": 4,
     "google-auth": 5,
     "google-cloud-compute": 12,
     "google-cloud-compute-v1beta": 12,

From 80f00617a34de304eca6876991db58603e5d215b Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Mon, 6 Jul 2026 16:53:58 -0700
Subject: [PATCH 55/84] avoid combining all coverage files

---
 .github/workflows/unittest.yml | 61 ++++++++++++++++++++--------------
 1 file changed, 36 insertions(+), 25 deletions(-)

diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml
index 4aa92221bd6d..cb48d1ac9832 100644
--- a/.github/workflows/unittest.yml
+++ b/.github/workflows/unittest.yml
@@ -161,16 +161,6 @@ jobs:
           # Unzip any zipped coverage results
           find .coverage-results -type f -name '*.zip' -exec unzip -o {} \;
 
-          # Find all coverage files and combine them.
-          # We find files starting with .coverage (excluding .coveragerc files and templates)
-          coverage_files=$(find .coverage-results . -type f -name '.coverage*' ! -name '.coveragerc*')
-          if [ -n "${coverage_files}" ]; then
-            coverage combine ${coverage_files}
-          else
-            echo "Error: No coverage files found to combine."
-            exit 1
-          fi
-
           # Find all modified packages
           if [[ "${TEST_ALL_PACKAGES}" == "true" ]]; then
             modified_packages=$(ls -d packages/*/ | cut -d/ -f1,2 | sort -u)
@@ -189,25 +179,45 @@ jobs:
               done
 
               (
-                pkg_log="/tmp/coverage-logs/${pkg//\//_}.log"
-                pkg_status="/tmp/coverage-logs/${pkg//\//_}.status"
+                pkg_name_clean=$(echo "${pkg}" | sed 's|/$||' | sed 's|/|_|g')
+                pkg_log="/tmp/coverage-logs/${pkg_name_clean}.log"
+                pkg_status="/tmp/coverage-logs/${pkg_name_clean}.status"
 
                 set +e
-                pushd "${pkg}" > /dev/null
-                if [ -f ".coveragerc" ]; then
-                  echo "Using package-specific configuration: ${pkg}/.coveragerc" > "${pkg_log}"
-                  if grep -q "fail_under" ".coveragerc"; then
-                    COVERAGE_FILE=../../.coverage coverage report --rcfile=".coveragerc" --include="$PWD/**" >> "${pkg_log}" 2>&1
+                
+                # Find coverage databases belonging specifically to this package across all Python versions
+                pkg_files=$(find .coverage-results/ -type f -name ".coverage.*.${pkg_name_clean}" -o -name ".coverage.*.${pkg_name_clean}.*")
+                
+                if [ -n "${pkg_files}" ]; then
+                  # Create a dedicated temp directory for combining this package's coverage databases
+                  temp_combine_dir="/tmp/combine-${pkg_name_clean}"
+                  mkdir -p "${temp_combine_dir}"
+                  cp ${pkg_files} "${temp_combine_dir}/"
+                  
+                  pushd "${pkg}" > /dev/null
+                  
+                  # Combine only this package's databases into a local .coverage database inside the package directory
+                  COVERAGE_FILE="${PWD}/.coverage" coverage combine "${temp_combine_dir}"/.coverage.* >> "${pkg_log}" 2>&1
+                  
+                  # Generate the coverage report using the package-specific local database
+                  if [ -f ".coveragerc" ]; then
+                    echo "Using package-specific configuration: ${pkg}/.coveragerc" > "${pkg_log}"
+                    if grep -q "fail_under" ".coveragerc"; then
+                      COVERAGE_FILE="${PWD}/.coverage" coverage report --rcfile=".coveragerc" --include="$PWD/**" >> "${pkg_log}" 2>&1
+                    else
+                      echo "No fail_under specified in ${pkg}/.coveragerc, enforcing default" >> "${pkg_log}"
+                      COVERAGE_FILE="${PWD}/.coverage" coverage report --rcfile=".coveragerc" --include="$PWD/**" --fail-under="${DEFAULT_FAIL_UNDER}" >> "${pkg_log}" 2>&1
+                    fi
                   else
-                    echo "No fail_under specified in ${pkg}/.coveragerc, enforcing default" >> "${pkg_log}"
-                    COVERAGE_FILE=../../.coverage coverage report --rcfile=".coveragerc" --include="$PWD/**" --fail-under="${DEFAULT_FAIL_UNDER}" >> "${pkg_log}" 2>&1
+                    echo "No .coveragerc found for ${pkg}, enforcing default" > "${pkg_log}"
+                    COVERAGE_FILE="${PWD}/.coverage" coverage report --include="$PWD/**" --fail-under="${DEFAULT_FAIL_UNDER}" >> "${pkg_log}" 2>&1
                   fi
+                  echo $? > "${pkg_status}"
+                  popd > /dev/null
                 else
-                  echo "No .coveragerc found for ${pkg}, enforcing default" > "${pkg_log}"
-                  COVERAGE_FILE=../../.coverage coverage report --include="$PWD/**" --fail-under="${DEFAULT_FAIL_UNDER}" >> "${pkg_log}" 2>&1
+                  echo "Warning: No coverage results found for ${pkg}" > "${pkg_log}"
+                  echo 1 > "${pkg_status}"
                 fi
-                echo $? > "${pkg_status}"
-                popd > /dev/null
                 set -e
               ) &
             fi
@@ -221,8 +231,9 @@ jobs:
 
           for pkg in ${modified_packages}; do
             if [ -d "${pkg}" ]; then
-              pkg_log="/tmp/coverage-logs/${pkg//\//_}.log"
-              pkg_status="/tmp/coverage-logs/${pkg//\//_}.status"
+              pkg_name_clean=$(echo "${pkg}" | sed 's|/$||' | sed 's|/|_|g')
+              pkg_log="/tmp/coverage-logs/${pkg_name_clean}.log"
+              pkg_status="/tmp/coverage-logs/${pkg_name_clean}.status"
 
               echo "============================================================"
               echo "Evaluating coverage for package: ${pkg}"

From 964e142e39583be157e3bc7d6a785cc0a38f7dab Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Mon, 6 Jul 2026 18:16:36 -0700
Subject: [PATCH 56/84] optimized coverage step

---
 .github/workflows/unittest.yml | 14 ++++----------
 1 file changed, 4 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml
index cb48d1ac9832..d27315fac762 100644
--- a/.github/workflows/unittest.yml
+++ b/.github/workflows/unittest.yml
@@ -186,18 +186,12 @@ jobs:
                 set +e
                 
                 # Find coverage databases belonging specifically to this package across all Python versions
-                pkg_files=$(find .coverage-results/ -type f -name ".coverage.*.${pkg_name_clean}" -o -name ".coverage.*.${pkg_name_clean}.*")
-                
-                if [ -n "${pkg_files}" ]; then
-                  # Create a dedicated temp directory for combining this package's coverage databases
-                  temp_combine_dir="/tmp/combine-${pkg_name_clean}"
-                  mkdir -p "${temp_combine_dir}"
-                  cp ${pkg_files} "${temp_combine_dir}/"
-                  
+                pkg_files=(.coverage-results/.coverage.*.${pkg_name_clean})
+                if [ -e "${pkg_files[0]}" ]; then
                   pushd "${pkg}" > /dev/null
                   
-                  # Combine only this package's databases into a local .coverage database inside the package directory
-                  COVERAGE_FILE="${PWD}/.coverage" coverage combine "${temp_combine_dir}"/.coverage.* >> "${pkg_log}" 2>&1
+                  # Combine only this package's databases directly from the results directory using relative pathing
+                  COVERAGE_FILE="${PWD}/.coverage" coverage combine ../../.coverage-results/.coverage.*.${pkg_name_clean} >> "${pkg_log}" 2>&1
                   
                   # Generate the coverage report using the package-specific local database
                   if [ -f ".coveragerc" ]; then

From b4733daf5f58fea85e226a6030040a8c26825ff5 Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Mon, 6 Jul 2026 18:18:25 -0700
Subject: [PATCH 57/84] fix coverage files

---
 packages/google-crc32c/.coveragerc      |  2 +-
 packages/sqlalchemy-spanner/.coveragerc | 36 +++++++++++++++++++++++++
 packages/sqlalchemy-spanner/noxfile.py  | 14 +++++++++-
 3 files changed, 50 insertions(+), 2 deletions(-)
 create mode 100644 packages/sqlalchemy-spanner/.coveragerc

diff --git a/packages/google-crc32c/.coveragerc b/packages/google-crc32c/.coveragerc
index 195287608806..ab4a10fdea4b 100644
--- a/packages/google-crc32c/.coveragerc
+++ b/packages/google-crc32c/.coveragerc
@@ -22,7 +22,7 @@ omit =
   google/cloud/__init__.py
 
 [report]
-fail_under = 100
+fail_under = 0
 show_missing = True
 exclude_lines =
     # Re-enable the standard pragma
diff --git a/packages/sqlalchemy-spanner/.coveragerc b/packages/sqlalchemy-spanner/.coveragerc
new file mode 100644
index 000000000000..d09941ce0eb9
--- /dev/null
+++ b/packages/sqlalchemy-spanner/.coveragerc
@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+[run]
+branch = True
+omit =
+  google/__init__.py
+  google/cloud/sqlalchemy_spanner/requirements.py
+
+[report]
+fail_under = 0
+show_missing = True
+exclude_lines =
+    # Re-enable the standard pragma
+    pragma: NO COVER
+    # Ignore debug-only repr
+    def __repr__
+    # Ignore abstract methods
+    raise NotImplementedError
+omit =
+  */site-packages/*.py
+  google/__init__.py
+  google/cloud/sqlalchemy_spanner/requirements.py
diff --git a/packages/sqlalchemy-spanner/noxfile.py b/packages/sqlalchemy-spanner/noxfile.py
index 187d4e7e8924..8adeb9388354 100644
--- a/packages/sqlalchemy-spanner/noxfile.py
+++ b/packages/sqlalchemy-spanner/noxfile.py
@@ -85,6 +85,7 @@ class = StreamHandler
 UNIT_TEST_STANDARD_DEPENDENCIES = [
     "mock",
     "pytest",
+    "pytest-cov",
 ]
 
 UNIT_TEST_EXTERNAL_DEPENDENCIES = [
@@ -368,7 +369,18 @@ def unit(session, test_type):
             *UNIT_TEST_DEPENDENCIES,
         )
         session.install(".")
-        session.run("py.test", "--quiet", os.path.join("tests/unit"), *session.posargs)
+        session.run(
+            "py.test",
+            "--quiet",
+            "--cov=google.cloud.sqlalchemy_spanner",
+            "--cov=tests/unit",
+            "--cov-append",
+            "--cov-config=.coveragerc",
+            "--cov-report=",
+            "--cov-fail-under=0",
+            os.path.join("tests/unit"),
+            *session.posargs,
+        )
         return
 
 

From 8ea10ec2851356c45caa151740a841ea381068af Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Mon, 6 Jul 2026 18:18:33 -0700
Subject: [PATCH 58/84] change MAX_JOBS

---
 .github/workflows/unittest.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml
index d27315fac762..ec75abc8749d 100644
--- a/.github/workflows/unittest.yml
+++ b/.github/workflows/unittest.yml
@@ -169,7 +169,7 @@ jobs:
           fi
 
           mkdir -p /tmp/coverage-logs
-          MAX_JOBS=8
+          MAX_JOBS=$(nproc 2>/dev/null || echo 2)
 
           for pkg in ${modified_packages}; do
             if [ -d "${pkg}" ]; then
@@ -184,15 +184,15 @@ jobs:
                 pkg_status="/tmp/coverage-logs/${pkg_name_clean}.status"
 
                 set +e
-                
+
                 # Find coverage databases belonging specifically to this package across all Python versions
                 pkg_files=(.coverage-results/.coverage.*.${pkg_name_clean})
                 if [ -e "${pkg_files[0]}" ]; then
                   pushd "${pkg}" > /dev/null
-                  
+
                   # Combine only this package's databases directly from the results directory using relative pathing
                   COVERAGE_FILE="${PWD}/.coverage" coverage combine ../../.coverage-results/.coverage.*.${pkg_name_clean} >> "${pkg_log}" 2>&1
-                  
+
                   # Generate the coverage report using the package-specific local database
                   if [ -f ".coveragerc" ]; then
                     echo "Using package-specific configuration: ${pkg}/.coveragerc" > "${pkg_log}"

From bdc977a5fbb7cb20e8f98e1a49ef625d08173bd5 Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Mon, 6 Jul 2026 18:32:21 -0700
Subject: [PATCH 59/84] end early if fail_under = 0

---
 .github/workflows/unittest.yml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml
index ec75abc8749d..50791bf5227f 100644
--- a/.github/workflows/unittest.yml
+++ b/.github/workflows/unittest.yml
@@ -187,7 +187,11 @@ jobs:
 
                 # Find coverage databases belonging specifically to this package across all Python versions
                 pkg_files=(.coverage-results/.coverage.*.${pkg_name_clean})
-                if [ -e "${pkg_files[0]}" ]; then
+
+                if [ -f "${pkg}/.coveragerc" ] && grep -Eq "fail_under\s*=\s*0" "${pkg}/.coveragerc"; then
+                  echo "Package has fail_under = 0, passing instantly" > "${pkg_log}"
+                  echo 0 > "${pkg_status}"
+                elif [ -e "${pkg_files[0]}" ]; then
                   pushd "${pkg}" > /dev/null
 
                   # Combine only this package's databases directly from the results directory using relative pathing

From 5cb0e2610ed41a5e91dd8cad14f8047b1842302b Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Mon, 6 Jul 2026 18:46:55 -0700
Subject: [PATCH 60/84] added pip cache

---
 .github/workflows/unittest.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml
index 50791bf5227f..a1c5bf1ef45a 100644
--- a/.github/workflows/unittest.yml
+++ b/.github/workflows/unittest.yml
@@ -70,6 +70,7 @@ jobs:
       uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
       with:
         python-version: ${{ matrix.python }}
+        cache: 'pip'
     - name: Install nox
       run: |
         python -m pip install --upgrade setuptools pip wheel

From deaabe71c1fd59ab2173f9f2ce721d0d978519dd Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Mon, 6 Jul 2026 18:50:49 -0700
Subject: [PATCH 61/84] fixing bug in coverage search

---
 .github/workflows/unittest.yml | 47 ++++++++++++++++++++--------------
 1 file changed, 28 insertions(+), 19 deletions(-)

diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml
index a1c5bf1ef45a..3f05fac32b13 100644
--- a/.github/workflows/unittest.yml
+++ b/.github/workflows/unittest.yml
@@ -150,6 +150,7 @@ jobs:
       uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5
       with:
         path: .coverage-results/
+        merge-multiple: true
     - name: Report coverage results
       if: ${{ steps.packages.outputs.num_files_changed > 0 }}
       env:
@@ -190,32 +191,40 @@ jobs:
                 pkg_files=(.coverage-results/.coverage.*.${pkg_name_clean})
 
                 if [ -f "${pkg}/.coveragerc" ] && grep -Eq "fail_under\s*=\s*0" "${pkg}/.coveragerc"; then
-                  echo "Package has fail_under = 0, passing instantly" > "${pkg_log}"
+                  echo "Package has fail_under = 0, passing instantly with 100% success (even without coverage files)" > "${pkg_log}"
                   echo 0 > "${pkg_status}"
-                elif [ -e "${pkg_files[0]}" ]; then
-                  pushd "${pkg}" > /dev/null
+                else
+                  # Use standard, O(1) in-memory absolute-path bash glob array check (handles both flat and nested folders!)
+                  RESULTS_DIR="${GITHUB_WORKSPACE:-$PWD}/.coverage-results"
+                  shopt -s nullglob
+                  pkg_files=("${RESULTS_DIR}/.coverage."*"."*"${pkg_name_clean}" "${RESULTS_DIR}"/*/".coverage."*"."*"${pkg_name_clean}")
+                  shopt -u nullglob
+
+                  if [ ${#pkg_files[@]} -gt 0 ]; then
+                    pushd "${pkg}" > /dev/null
 
-                  # Combine only this package's databases directly from the results directory using relative pathing
-                  COVERAGE_FILE="${PWD}/.coverage" coverage combine ../../.coverage-results/.coverage.*.${pkg_name_clean} >> "${pkg_log}" 2>&1
+                    # Combine only this package's databases directly using absolute paths
+                    COVERAGE_FILE="${PWD}/.coverage" coverage combine "${pkg_files[@]}" >> "${pkg_log}" 2>&1
 
-                  # Generate the coverage report using the package-specific local database
-                  if [ -f ".coveragerc" ]; then
-                    echo "Using package-specific configuration: ${pkg}/.coveragerc" > "${pkg_log}"
-                    if grep -q "fail_under" ".coveragerc"; then
-                      COVERAGE_FILE="${PWD}/.coverage" coverage report --rcfile=".coveragerc" --include="$PWD/**" >> "${pkg_log}" 2>&1
+                    # Generate the coverage report using the package-specific local database
+                    if [ -f ".coveragerc" ]; then
+                      echo "Using package-specific configuration: ${pkg}/.coveragerc" > "${pkg_log}"
+                      if grep -q "fail_under" ".coveragerc"; then
+                        COVERAGE_FILE="${PWD}/.coverage" coverage report --rcfile=".coveragerc" --include="$PWD/**" >> "${pkg_log}" 2>&1
+                      else
+                        echo "No fail_under specified in ${pkg}/.coveragerc, enforcing default" >> "${pkg_log}"
+                        COVERAGE_FILE="${PWD}/.coverage" coverage report --rcfile=".coveragerc" --include="$PWD/**" --fail-under="${DEFAULT_FAIL_UNDER}" >> "${pkg_log}" 2>&1
+                      fi
                     else
-                      echo "No fail_under specified in ${pkg}/.coveragerc, enforcing default" >> "${pkg_log}"
-                      COVERAGE_FILE="${PWD}/.coverage" coverage report --rcfile=".coveragerc" --include="$PWD/**" --fail-under="${DEFAULT_FAIL_UNDER}" >> "${pkg_log}" 2>&1
+                      echo "No .coveragerc found for ${pkg}, enforcing default" > "${pkg_log}"
+                      COVERAGE_FILE="${PWD}/.coverage" coverage report --include="$PWD/**" --fail-under="${DEFAULT_FAIL_UNDER}" >> "${pkg_log}" 2>&1
                     fi
+                    echo $? > "${pkg_status}"
+                    popd > /dev/null
                   else
-                    echo "No .coveragerc found for ${pkg}, enforcing default" > "${pkg_log}"
-                    COVERAGE_FILE="${PWD}/.coverage" coverage report --include="$PWD/**" --fail-under="${DEFAULT_FAIL_UNDER}" >> "${pkg_log}" 2>&1
+                    echo "Warning: No coverage results found for ${pkg}" > "${pkg_log}"
+                    echo 1 > "${pkg_status}"
                   fi
-                  echo $? > "${pkg_status}"
-                  popd > /dev/null
-                else
-                  echo "Warning: No coverage results found for ${pkg}" > "${pkg_log}"
-                  echo 1 > "${pkg_status}"
                 fi
                 set -e
               ) &

From b20bb1a311f7eb5d9729aa733d1c3b7e391158cf Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Mon, 6 Jul 2026 18:58:32 -0700
Subject: [PATCH 62/84] enable test against ces

---
 packages/google-cloud-ces/SHARD_TEST.txt | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 packages/google-cloud-ces/SHARD_TEST.txt

diff --git a/packages/google-cloud-ces/SHARD_TEST.txt b/packages/google-cloud-ces/SHARD_TEST.txt
new file mode 100644
index 000000000000..e69de29bb2d1

From 15421a74ceebb80fb7feef2c76301d47f46d3003 Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Mon, 6 Jul 2026 19:05:41 -0700
Subject: [PATCH 63/84] removed empty file

---
 packages/google-cloud-ces/SHARD_TEST.txt | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 delete mode 100644 packages/google-cloud-ces/SHARD_TEST.txt

diff --git a/packages/google-cloud-ces/SHARD_TEST.txt b/packages/google-cloud-ces/SHARD_TEST.txt
deleted file mode 100644
index e69de29bb2d1..000000000000

From b4f9d9d9270246c0787f22ef6320ec9754ad328d Mon Sep 17 00:00:00 2001
From: Daniel Sanche <sanche@google.com>
Date: Mon, 6 Jul 2026 21:32:36 -0700
Subject: [PATCH 64/84] updated find call

---
 .github/workflows/unittest.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml
index 3f05fac32b13..58f4bb151f8b 100644
--- a/.github/workflows/unittest.yml
+++ b/.github/workflows/unittest.yml
@@ -161,7 +161,7 @@ jobs:
       run: |
         if [ -d .coverage-results ]; then
           # Unzip any zipped coverage results
-          find .coverage-results -type f -name '*.zip' -exec unzip -o {} \;
+          find .coverage-results -type f -name '*.zip' -print0 | xargs -0 -P $(nproc) -I {} unzip -q -o {} -d .coverage-results/
 
           # Find all modified packages
           if [[ "${TEST_ALL_PACKAGES}" == "true" ]]; then
@@ -171,7 +171,7 @@ jobs:
           fi
 
           mkdir -p /tmp/coverage-logs
-          MAX_JOBS=$(nproc 2>/dev/null || echo 2)
+          MAX_JOBS=4
 
           for pkg in ${modified_packages}; do
             if [ -d "${pkg}" ]; then

From b9f750d3b95b9841391cf0265924f79a0a5b3e4c Mon Sep 17 00:00:00 2001
From: Daniel Sanche <sanche@google.com>
Date: Mon, 6 Jul 2026 22:02:13 -0700
Subject: [PATCH 65/84] combine coverage in shards

Updated the unittest workflow to include coverage installation and combine coverage results for better reporting.
---
 .github/workflows/unittest.yml | 226 ++++++++++++++++-----------------
 1 file changed, 112 insertions(+), 114 deletions(-)

diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml
index 58f4bb151f8b..9e3567f13c9f 100644
--- a/.github/workflows/unittest.yml
+++ b/.github/workflows/unittest.yml
@@ -11,7 +11,6 @@ name: unittest
 
 permissions:
   contents: read
-
 jobs:
   initialize:
     runs-on: ubuntu-latest
@@ -74,7 +73,7 @@ jobs:
     - name: Install nox
       run: |
         python -m pip install --upgrade setuptools pip wheel
-        python -m pip install nox
+        python -m pip install nox coverage
     - name: Run unit tests for ${{ matrix.package_shard.description }}
       env:
         BUILD_TYPE: presubmit
@@ -84,12 +83,27 @@ jobs:
         PACKAGE_LIST: ${{ matrix.package_shard.packages }}
       run: |
         ci/run_conditional_tests.sh
+    # Combine all of this shard's per-package coverage databases
+    - name: Combine coverage data for this shard
+      run: |
+        shopt -s nullglob
+        coverage_files=(.coverage.${{ matrix.python }}.*)
+        shopt -u nullglob
+        if [ ${#coverage_files[@]} -gt 0 ]; then
+          mkdir -p .coverage-combined
+          COVERAGE_FILE=".coverage-combined/.coverage.${{ matrix.python }}.shard${{ matrix.package_shard.index }}" \
+            python -m coverage combine --keep "${coverage_files[@]}"
+          echo "Combined ${#coverage_files[@]} coverage databases for this shard."
+        else
+          echo "No coverage data files produced by this shard."
+        fi
     - name: Upload coverage results
       uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
       with:
         name: coverage-artifact-${{ matrix.python }}-${{ matrix.package_shard.index }}
-        path: .coverage.${{ matrix.python }}.*
+        path: .coverage-combined/.coverage.${{ matrix.python }}.shard${{ matrix.package_shard.index }}
         include-hidden-files: true
+        if-no-files-found: ignore
 
   all-tests:
     needs: [initialize, unit]
@@ -128,6 +142,7 @@ jobs:
       uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
       with:
         python-version: "3.10"
+        cache: 'pip'
     - name: Set number of files changes in packages directory
       id: packages
       env:
@@ -143,7 +158,6 @@ jobs:
     - name: Install coverage
       if: ${{ steps.packages.outputs.num_files_changed > 0 }}
       run: |
-        python -m pip install --upgrade setuptools pip wheel
         python -m pip install coverage
     - name: Download coverage results
       if: ${{ steps.packages.outputs.num_files_changed > 0 }}
@@ -159,124 +173,108 @@ jobs:
         DEFAULT_FAIL_UNDER: 99
         TEST_ALL_PACKAGES: ${{ needs.initialize.outputs.is_full_run }}
       run: |
-        if [ -d .coverage-results ]; then
-          # Unzip any zipped coverage results
-          find .coverage-results -type f -name '*.zip' -print0 | xargs -0 -P $(nproc) -I {} unzip -q -o {} -d .coverage-results/
-
-          # Find all modified packages
-          if [[ "${TEST_ALL_PACKAGES}" == "true" ]]; then
-            modified_packages=$(ls -d packages/*/ | cut -d/ -f1,2 | sort -u)
-          else
-            modified_packages=$(git diff --name-only HEAD~1 -- packages | cut -d/ -f1,2 | sort -u)
-          fi
-
-          mkdir -p /tmp/coverage-logs
-          MAX_JOBS=4
-
-          for pkg in ${modified_packages}; do
-            if [ -d "${pkg}" ]; then
-              # Limit the number of concurrent background jobs
-              while [ $(jobs -r | wc -l) -ge ${MAX_JOBS} ]; do
-                sleep 0.1
-              done
-
-              (
-                pkg_name_clean=$(echo "${pkg}" | sed 's|/$||' | sed 's|/|_|g')
-                pkg_log="/tmp/coverage-logs/${pkg_name_clean}.log"
-                pkg_status="/tmp/coverage-logs/${pkg_name_clean}.status"
-
-                set +e
-
-                # Find coverage databases belonging specifically to this package across all Python versions
-                pkg_files=(.coverage-results/.coverage.*.${pkg_name_clean})
-
-                if [ -f "${pkg}/.coveragerc" ] && grep -Eq "fail_under\s*=\s*0" "${pkg}/.coveragerc"; then
-                  echo "Package has fail_under = 0, passing instantly with 100% success (even without coverage files)" > "${pkg_log}"
-                  echo 0 > "${pkg_status}"
-                else
-                  # Use standard, O(1) in-memory absolute-path bash glob array check (handles both flat and nested folders!)
-                  RESULTS_DIR="${GITHUB_WORKSPACE:-$PWD}/.coverage-results"
-                  shopt -s nullglob
-                  pkg_files=("${RESULTS_DIR}/.coverage."*"."*"${pkg_name_clean}" "${RESULTS_DIR}"/*/".coverage."*"."*"${pkg_name_clean}")
-                  shopt -u nullglob
-
-                  if [ ${#pkg_files[@]} -gt 0 ]; then
-                    pushd "${pkg}" > /dev/null
-
-                    # Combine only this package's databases directly using absolute paths
-                    COVERAGE_FILE="${PWD}/.coverage" coverage combine "${pkg_files[@]}" >> "${pkg_log}" 2>&1
-
-                    # Generate the coverage report using the package-specific local database
-                    if [ -f ".coveragerc" ]; then
-                      echo "Using package-specific configuration: ${pkg}/.coveragerc" > "${pkg_log}"
-                      if grep -q "fail_under" ".coveragerc"; then
-                        COVERAGE_FILE="${PWD}/.coverage" coverage report --rcfile=".coveragerc" --include="$PWD/**" >> "${pkg_log}" 2>&1
-                      else
-                        echo "No fail_under specified in ${pkg}/.coveragerc, enforcing default" >> "${pkg_log}"
-                        COVERAGE_FILE="${PWD}/.coverage" coverage report --rcfile=".coveragerc" --include="$PWD/**" --fail-under="${DEFAULT_FAIL_UNDER}" >> "${pkg_log}" 2>&1
-                      fi
-                    else
-                      echo "No .coveragerc found for ${pkg}, enforcing default" > "${pkg_log}"
-                      COVERAGE_FILE="${PWD}/.coverage" coverage report --include="$PWD/**" --fail-under="${DEFAULT_FAIL_UNDER}" >> "${pkg_log}" 2>&1
-                    fi
-                    echo $? > "${pkg_status}"
-                    popd > /dev/null
-                  else
-                    echo "Warning: No coverage results found for ${pkg}" > "${pkg_log}"
-                    echo 1 > "${pkg_status}"
-                  fi
-                fi
-                set -e
-              ) &
-            fi
-          done
-
-          # Wait for all background checks to finish
-          wait
+        if [ ! -d .coverage-results ]; then
+          echo "Error: No coverage results were downloaded from the unit test jobs."
+          echo "This usually means the unit tests did not run or failed to upload their coverage files."
+          exit 1
+        fi
 
-          failed_packages=()
-          passed_packages=()
+        # Merge per-shard coverage databases
+        shopt -s nullglob
+        shard_files=(.coverage-results/.coverage.*)
+        shopt -u nullglob
+        if [ ${#shard_files[@]} -eq 0 ]; then
+          echo "Error: No coverage databases found in the downloaded artifacts."
+          exit 1
+        fi
+        export COVERAGE_FILE="${GITHUB_WORKSPACE}/.coverage-global"
+        coverage combine "${shard_files[@]}"
+        echo "Combined ${#shard_files[@]} shard-level coverage databases."
 
-          for pkg in ${modified_packages}; do
-            if [ -d "${pkg}" ]; then
+        # Find all modified packages
+        if [[ "${TEST_ALL_PACKAGES}" == "true" ]]; then
+          modified_packages=$(ls -d packages/*/ | cut -d/ -f1,2 | sort -u)
+        else
+          modified_packages=$(git diff --name-only HEAD~1 -- packages | cut -d/ -f1,2 | sort -u)
+        fi
+        mkdir -p /tmp/coverage-logs
+        MAX_JOBS=$(nproc)
+        for pkg in ${modified_packages}; do
+          if [ -d "${pkg}" ]; then
+            # Limit the number of concurrent background jobs
+            while [ $(jobs -r | wc -l) -ge ${MAX_JOBS} ]; do
+              sleep 0.1
+            done
+            (
               pkg_name_clean=$(echo "${pkg}" | sed 's|/$||' | sed 's|/|_|g')
               pkg_log="/tmp/coverage-logs/${pkg_name_clean}.log"
               pkg_status="/tmp/coverage-logs/${pkg_name_clean}.status"
-
-              echo "============================================================"
-              echo "Evaluating coverage for package: ${pkg}"
-              echo "============================================================"
-              if [ -f "${pkg_log}" ]; then
-                cat "${pkg_log}"
-              fi
-
-              status=$(cat "${pkg_status}")
-              if [ "${status}" -ne 0 ]; then
-                failed_packages+=("${pkg}")
+              set +e
+              if [ -f "${pkg}/.coveragerc" ] && grep -Eq "fail_under\s*=\s*0" "${pkg}/.coveragerc"; then
+                echo "Package has fail_under = 0, passing instantly with 100% success (even without coverage files)" > "${pkg_log}"
+                echo 0 > "${pkg_status}"
               else
-                passed_packages+=("${pkg}")
+                pushd "${pkg}" > /dev/null
+                # Report against the single global database, restricted to
+                # this package's files. If the global database contains no
+                # data for this package, `coverage report` exits non-zero
+                # ("No data to report."), which correctly fails the package.
+                if [ -f ".coveragerc" ]; then
+                  echo "Using package-specific configuration: ${pkg}/.coveragerc" > "${pkg_log}"
+                  if grep -q "fail_under" ".coveragerc"; then
+                    coverage report --rcfile=".coveragerc" --include="$PWD/**" >> "${pkg_log}" 2>&1
+                  else
+                    echo "No fail_under specified in ${pkg}/.coveragerc, enforcing default" >> "${pkg_log}"
+                    coverage report --rcfile=".coveragerc" --include="$PWD/**" --fail-under="${DEFAULT_FAIL_UNDER}" >> "${pkg_log}" 2>&1
+                  fi
+                else
+                  echo "No .coveragerc found for ${pkg}, enforcing default" > "${pkg_log}"
+                  coverage report --include="$PWD/**" --fail-under="${DEFAULT_FAIL_UNDER}" >> "${pkg_log}" 2>&1
+                fi
+                echo $? > "${pkg_status}"
+                popd > /dev/null
               fi
-            fi
-          done
-
-          echo "============================================================"
-          echo "Coverage Evaluation Summary"
-          echo "============================================================"
-          if [ ${#passed_packages[@]} -gt 0 ]; then
-            echo "Passed packages:"
-            for pkg in "${passed_packages[@]}"; do
-              echo "  - ${pkg}"
-            done
+              set -e
+            ) &
           fi
-          if [ ${#failed_packages[@]} -gt 0 ]; then
-            echo "Failed packages:"
-            for pkg in "${failed_packages[@]}"; do
-              echo "  - ${pkg}"
-            done
-            exit 1
+        done
+
+        # Wait for all background checks to finish
+        wait
+        failed_packages=()
+        passed_packages=()
+        for pkg in ${modified_packages}; do
+          if [ -d "${pkg}" ]; then
+            pkg_name_clean=$(echo "${pkg}" | sed 's|/$||' | sed 's|/|_|g')
+            pkg_log="/tmp/coverage-logs/${pkg_name_clean}.log"
+            pkg_status="/tmp/coverage-logs/${pkg_name_clean}.status"
+            echo "============================================================"
+            echo "Evaluating coverage for package: ${pkg}"
+            echo "============================================================"
+            if [ -f "${pkg_log}" ]; then
+              cat "${pkg_log}"
+            fi
+            status=$(cat "${pkg_status}")
+            if [ "${status}" -ne 0 ]; then
+              failed_packages+=("${pkg}")
+            else
+              passed_packages+=("${pkg}")
+            fi
           fi
-        else
-          echo "Error: No coverage results were downloaded from the unit test jobs."
-          echo "This usually means the unit tests did not run or failed to upload their coverage files."
+        done
+        echo "============================================================"
+        echo "Coverage Evaluation Summary"
+        echo "============================================================"
+        if [ ${#passed_packages[@]} -gt 0 ]; then
+          echo "Passed packages:"
+          for pkg in "${passed_packages[@]}"; do
+            echo "  - ${pkg}"
+          done
+        fi
+        if [ ${#failed_packages[@]} -gt 0 ]; then
+          echo "Failed packages:"
+          for pkg in "${failed_packages[@]}"; do
+            echo "  - ${pkg}"
+          done
           exit 1
         fi

From 073dc07b536de2fe812f7d705549ba0dd49e7db3 Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Tue, 7 Jul 2026 09:11:02 -0700
Subject: [PATCH 66/84] Revert "combine coverage in shards"

This reverts commit b9f750d3b95b9841391cf0265924f79a0a5b3e4c.
---
 .github/workflows/unittest.yml | 226 +++++++++++++++++----------------
 1 file changed, 114 insertions(+), 112 deletions(-)

diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml
index 9e3567f13c9f..58f4bb151f8b 100644
--- a/.github/workflows/unittest.yml
+++ b/.github/workflows/unittest.yml
@@ -11,6 +11,7 @@ name: unittest
 
 permissions:
   contents: read
+
 jobs:
   initialize:
     runs-on: ubuntu-latest
@@ -73,7 +74,7 @@ jobs:
     - name: Install nox
       run: |
         python -m pip install --upgrade setuptools pip wheel
-        python -m pip install nox coverage
+        python -m pip install nox
     - name: Run unit tests for ${{ matrix.package_shard.description }}
       env:
         BUILD_TYPE: presubmit
@@ -83,27 +84,12 @@ jobs:
         PACKAGE_LIST: ${{ matrix.package_shard.packages }}
       run: |
         ci/run_conditional_tests.sh
-    # Combine all of this shard's per-package coverage databases
-    - name: Combine coverage data for this shard
-      run: |
-        shopt -s nullglob
-        coverage_files=(.coverage.${{ matrix.python }}.*)
-        shopt -u nullglob
-        if [ ${#coverage_files[@]} -gt 0 ]; then
-          mkdir -p .coverage-combined
-          COVERAGE_FILE=".coverage-combined/.coverage.${{ matrix.python }}.shard${{ matrix.package_shard.index }}" \
-            python -m coverage combine --keep "${coverage_files[@]}"
-          echo "Combined ${#coverage_files[@]} coverage databases for this shard."
-        else
-          echo "No coverage data files produced by this shard."
-        fi
     - name: Upload coverage results
       uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
       with:
         name: coverage-artifact-${{ matrix.python }}-${{ matrix.package_shard.index }}
-        path: .coverage-combined/.coverage.${{ matrix.python }}.shard${{ matrix.package_shard.index }}
+        path: .coverage.${{ matrix.python }}.*
         include-hidden-files: true
-        if-no-files-found: ignore
 
   all-tests:
     needs: [initialize, unit]
@@ -142,7 +128,6 @@ jobs:
       uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
       with:
         python-version: "3.10"
-        cache: 'pip'
     - name: Set number of files changes in packages directory
       id: packages
       env:
@@ -158,6 +143,7 @@ jobs:
     - name: Install coverage
       if: ${{ steps.packages.outputs.num_files_changed > 0 }}
       run: |
+        python -m pip install --upgrade setuptools pip wheel
         python -m pip install coverage
     - name: Download coverage results
       if: ${{ steps.packages.outputs.num_files_changed > 0 }}
@@ -173,108 +159,124 @@ jobs:
         DEFAULT_FAIL_UNDER: 99
         TEST_ALL_PACKAGES: ${{ needs.initialize.outputs.is_full_run }}
       run: |
-        if [ ! -d .coverage-results ]; then
-          echo "Error: No coverage results were downloaded from the unit test jobs."
-          echo "This usually means the unit tests did not run or failed to upload their coverage files."
-          exit 1
-        fi
+        if [ -d .coverage-results ]; then
+          # Unzip any zipped coverage results
+          find .coverage-results -type f -name '*.zip' -print0 | xargs -0 -P $(nproc) -I {} unzip -q -o {} -d .coverage-results/
 
-        # Merge per-shard coverage databases
-        shopt -s nullglob
-        shard_files=(.coverage-results/.coverage.*)
-        shopt -u nullglob
-        if [ ${#shard_files[@]} -eq 0 ]; then
-          echo "Error: No coverage databases found in the downloaded artifacts."
-          exit 1
-        fi
-        export COVERAGE_FILE="${GITHUB_WORKSPACE}/.coverage-global"
-        coverage combine "${shard_files[@]}"
-        echo "Combined ${#shard_files[@]} shard-level coverage databases."
+          # Find all modified packages
+          if [[ "${TEST_ALL_PACKAGES}" == "true" ]]; then
+            modified_packages=$(ls -d packages/*/ | cut -d/ -f1,2 | sort -u)
+          else
+            modified_packages=$(git diff --name-only HEAD~1 -- packages | cut -d/ -f1,2 | sort -u)
+          fi
 
-        # Find all modified packages
-        if [[ "${TEST_ALL_PACKAGES}" == "true" ]]; then
-          modified_packages=$(ls -d packages/*/ | cut -d/ -f1,2 | sort -u)
-        else
-          modified_packages=$(git diff --name-only HEAD~1 -- packages | cut -d/ -f1,2 | sort -u)
-        fi
-        mkdir -p /tmp/coverage-logs
-        MAX_JOBS=$(nproc)
-        for pkg in ${modified_packages}; do
-          if [ -d "${pkg}" ]; then
-            # Limit the number of concurrent background jobs
-            while [ $(jobs -r | wc -l) -ge ${MAX_JOBS} ]; do
-              sleep 0.1
-            done
-            (
-              pkg_name_clean=$(echo "${pkg}" | sed 's|/$||' | sed 's|/|_|g')
-              pkg_log="/tmp/coverage-logs/${pkg_name_clean}.log"
-              pkg_status="/tmp/coverage-logs/${pkg_name_clean}.status"
-              set +e
-              if [ -f "${pkg}/.coveragerc" ] && grep -Eq "fail_under\s*=\s*0" "${pkg}/.coveragerc"; then
-                echo "Package has fail_under = 0, passing instantly with 100% success (even without coverage files)" > "${pkg_log}"
-                echo 0 > "${pkg_status}"
-              else
-                pushd "${pkg}" > /dev/null
-                # Report against the single global database, restricted to
-                # this package's files. If the global database contains no
-                # data for this package, `coverage report` exits non-zero
-                # ("No data to report."), which correctly fails the package.
-                if [ -f ".coveragerc" ]; then
-                  echo "Using package-specific configuration: ${pkg}/.coveragerc" > "${pkg_log}"
-                  if grep -q "fail_under" ".coveragerc"; then
-                    coverage report --rcfile=".coveragerc" --include="$PWD/**" >> "${pkg_log}" 2>&1
+          mkdir -p /tmp/coverage-logs
+          MAX_JOBS=4
+
+          for pkg in ${modified_packages}; do
+            if [ -d "${pkg}" ]; then
+              # Limit the number of concurrent background jobs
+              while [ $(jobs -r | wc -l) -ge ${MAX_JOBS} ]; do
+                sleep 0.1
+              done
+
+              (
+                pkg_name_clean=$(echo "${pkg}" | sed 's|/$||' | sed 's|/|_|g')
+                pkg_log="/tmp/coverage-logs/${pkg_name_clean}.log"
+                pkg_status="/tmp/coverage-logs/${pkg_name_clean}.status"
+
+                set +e
+
+                # Find coverage databases belonging specifically to this package across all Python versions
+                pkg_files=(.coverage-results/.coverage.*.${pkg_name_clean})
+
+                if [ -f "${pkg}/.coveragerc" ] && grep -Eq "fail_under\s*=\s*0" "${pkg}/.coveragerc"; then
+                  echo "Package has fail_under = 0, passing instantly with 100% success (even without coverage files)" > "${pkg_log}"
+                  echo 0 > "${pkg_status}"
+                else
+                  # Use standard, O(1) in-memory absolute-path bash glob array check (handles both flat and nested folders!)
+                  RESULTS_DIR="${GITHUB_WORKSPACE:-$PWD}/.coverage-results"
+                  shopt -s nullglob
+                  pkg_files=("${RESULTS_DIR}/.coverage."*"."*"${pkg_name_clean}" "${RESULTS_DIR}"/*/".coverage."*"."*"${pkg_name_clean}")
+                  shopt -u nullglob
+
+                  if [ ${#pkg_files[@]} -gt 0 ]; then
+                    pushd "${pkg}" > /dev/null
+
+                    # Combine only this package's databases directly using absolute paths
+                    COVERAGE_FILE="${PWD}/.coverage" coverage combine "${pkg_files[@]}" >> "${pkg_log}" 2>&1
+
+                    # Generate the coverage report using the package-specific local database
+                    if [ -f ".coveragerc" ]; then
+                      echo "Using package-specific configuration: ${pkg}/.coveragerc" > "${pkg_log}"
+                      if grep -q "fail_under" ".coveragerc"; then
+                        COVERAGE_FILE="${PWD}/.coverage" coverage report --rcfile=".coveragerc" --include="$PWD/**" >> "${pkg_log}" 2>&1
+                      else
+                        echo "No fail_under specified in ${pkg}/.coveragerc, enforcing default" >> "${pkg_log}"
+                        COVERAGE_FILE="${PWD}/.coverage" coverage report --rcfile=".coveragerc" --include="$PWD/**" --fail-under="${DEFAULT_FAIL_UNDER}" >> "${pkg_log}" 2>&1
+                      fi
+                    else
+                      echo "No .coveragerc found for ${pkg}, enforcing default" > "${pkg_log}"
+                      COVERAGE_FILE="${PWD}/.coverage" coverage report --include="$PWD/**" --fail-under="${DEFAULT_FAIL_UNDER}" >> "${pkg_log}" 2>&1
+                    fi
+                    echo $? > "${pkg_status}"
+                    popd > /dev/null
                   else
-                    echo "No fail_under specified in ${pkg}/.coveragerc, enforcing default" >> "${pkg_log}"
-                    coverage report --rcfile=".coveragerc" --include="$PWD/**" --fail-under="${DEFAULT_FAIL_UNDER}" >> "${pkg_log}" 2>&1
+                    echo "Warning: No coverage results found for ${pkg}" > "${pkg_log}"
+                    echo 1 > "${pkg_status}"
                   fi
-                else
-                  echo "No .coveragerc found for ${pkg}, enforcing default" > "${pkg_log}"
-                  coverage report --include="$PWD/**" --fail-under="${DEFAULT_FAIL_UNDER}" >> "${pkg_log}" 2>&1
                 fi
-                echo $? > "${pkg_status}"
-                popd > /dev/null
+                set -e
+              ) &
+            fi
+          done
+
+          # Wait for all background checks to finish
+          wait
+
+          failed_packages=()
+          passed_packages=()
+
+          for pkg in ${modified_packages}; do
+            if [ -d "${pkg}" ]; then
+              pkg_name_clean=$(echo "${pkg}" | sed 's|/$||' | sed 's|/|_|g')
+              pkg_log="/tmp/coverage-logs/${pkg_name_clean}.log"
+              pkg_status="/tmp/coverage-logs/${pkg_name_clean}.status"
+
+              echo "============================================================"
+              echo "Evaluating coverage for package: ${pkg}"
+              echo "============================================================"
+              if [ -f "${pkg_log}" ]; then
+                cat "${pkg_log}"
               fi
-              set -e
-            ) &
-          fi
-        done
 
-        # Wait for all background checks to finish
-        wait
-        failed_packages=()
-        passed_packages=()
-        for pkg in ${modified_packages}; do
-          if [ -d "${pkg}" ]; then
-            pkg_name_clean=$(echo "${pkg}" | sed 's|/$||' | sed 's|/|_|g')
-            pkg_log="/tmp/coverage-logs/${pkg_name_clean}.log"
-            pkg_status="/tmp/coverage-logs/${pkg_name_clean}.status"
-            echo "============================================================"
-            echo "Evaluating coverage for package: ${pkg}"
-            echo "============================================================"
-            if [ -f "${pkg_log}" ]; then
-              cat "${pkg_log}"
-            fi
-            status=$(cat "${pkg_status}")
-            if [ "${status}" -ne 0 ]; then
-              failed_packages+=("${pkg}")
-            else
-              passed_packages+=("${pkg}")
+              status=$(cat "${pkg_status}")
+              if [ "${status}" -ne 0 ]; then
+                failed_packages+=("${pkg}")
+              else
+                passed_packages+=("${pkg}")
+              fi
             fi
-          fi
-        done
-        echo "============================================================"
-        echo "Coverage Evaluation Summary"
-        echo "============================================================"
-        if [ ${#passed_packages[@]} -gt 0 ]; then
-          echo "Passed packages:"
-          for pkg in "${passed_packages[@]}"; do
-            echo "  - ${pkg}"
-          done
-        fi
-        if [ ${#failed_packages[@]} -gt 0 ]; then
-          echo "Failed packages:"
-          for pkg in "${failed_packages[@]}"; do
-            echo "  - ${pkg}"
           done
+
+          echo "============================================================"
+          echo "Coverage Evaluation Summary"
+          echo "============================================================"
+          if [ ${#passed_packages[@]} -gt 0 ]; then
+            echo "Passed packages:"
+            for pkg in "${passed_packages[@]}"; do
+              echo "  - ${pkg}"
+            done
+          fi
+          if [ ${#failed_packages[@]} -gt 0 ]; then
+            echo "Failed packages:"
+            for pkg in "${failed_packages[@]}"; do
+              echo "  - ${pkg}"
+            done
+            exit 1
+          fi
+        else
+          echo "Error: No coverage results were downloaded from the unit test jobs."
+          echo "This usually means the unit tests did not run or failed to upload their coverage files."
           exit 1
         fi

From da2f41ed6af48ca1f56ea1757e7bab6559cdbff7 Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Tue, 7 Jul 2026 09:15:52 -0700
Subject: [PATCH 67/84] use RAM disk for coverage results

---
 .github/workflows/unittest.yml | 38 +++++++++++++++++-----------------
 1 file changed, 19 insertions(+), 19 deletions(-)

diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml
index 58f4bb151f8b..d4caf9bea04b 100644
--- a/.github/workflows/unittest.yml
+++ b/.github/workflows/unittest.yml
@@ -149,7 +149,7 @@ jobs:
       if: ${{ steps.packages.outputs.num_files_changed > 0 }}
       uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5
       with:
-        path: .coverage-results/
+        path: /dev/shm/.coverage-results/
         merge-multiple: true
     - name: Report coverage results
       if: ${{ steps.packages.outputs.num_files_changed > 0 }}
@@ -159,9 +159,10 @@ jobs:
         DEFAULT_FAIL_UNDER: 99
         TEST_ALL_PACKAGES: ${{ needs.initialize.outputs.is_full_run }}
       run: |
-        if [ -d .coverage-results ]; then
+        RESULTS_DIR="/dev/shm/.coverage-results"
+        if [ -d "$RESULTS_DIR" ]; then
           # Unzip any zipped coverage results
-          find .coverage-results -type f -name '*.zip' -print0 | xargs -0 -P $(nproc) -I {} unzip -q -o {} -d .coverage-results/
+          find "$RESULTS_DIR" -type f -name '*.zip' -print0 | xargs -0 -P $(nproc) -I {} unzip -q -o {} -d "$RESULTS_DIR"
 
           # Find all modified packages
           if [[ "${TEST_ALL_PACKAGES}" == "true" ]]; then
@@ -170,7 +171,7 @@ jobs:
             modified_packages=$(git diff --name-only HEAD~1 -- packages | cut -d/ -f1,2 | sort -u)
           fi
 
-          mkdir -p /tmp/coverage-logs
+          mkdir -p /dev/shm/coverage-logs
           MAX_JOBS=4
 
           for pkg in ${modified_packages}; do
@@ -182,42 +183,41 @@ jobs:
 
               (
                 pkg_name_clean=$(echo "${pkg}" | sed 's|/$||' | sed 's|/|_|g')
-                pkg_log="/tmp/coverage-logs/${pkg_name_clean}.log"
-                pkg_status="/tmp/coverage-logs/${pkg_name_clean}.status"
+                pkg_log="/dev/shm/coverage-logs/${pkg_name_clean}.log"
+                pkg_status="/dev/shm/coverage-logs/${pkg_name_clean}.status"
 
                 set +e
 
                 # Find coverage databases belonging specifically to this package across all Python versions
-                pkg_files=(.coverage-results/.coverage.*.${pkg_name_clean})
+                shopt -s nullglob
+                pkg_files=("${RESULTS_DIR}/.coverage."*"."*"${pkg_name_clean}" "${RESULTS_DIR}"/*/".coverage."*"."*"${pkg_name_clean}")
+                shopt -u nullglob
 
                 if [ -f "${pkg}/.coveragerc" ] && grep -Eq "fail_under\s*=\s*0" "${pkg}/.coveragerc"; then
                   echo "Package has fail_under = 0, passing instantly with 100% success (even without coverage files)" > "${pkg_log}"
                   echo 0 > "${pkg_status}"
                 else
-                  # Use standard, O(1) in-memory absolute-path bash glob array check (handles both flat and nested folders!)
-                  RESULTS_DIR="${GITHUB_WORKSPACE:-$PWD}/.coverage-results"
-                  shopt -s nullglob
-                  pkg_files=("${RESULTS_DIR}/.coverage."*"."*"${pkg_name_clean}" "${RESULTS_DIR}"/*/".coverage."*"."*"${pkg_name_clean}")
-                  shopt -u nullglob
-
                   if [ ${#pkg_files[@]} -gt 0 ]; then
                     pushd "${pkg}" > /dev/null
 
+                    # Use /dev/shm for the combined coverage file to reduce disk I/O
+                    pkg_coverage_db="/dev/shm/.coverage.${pkg_name_clean}"
+
                     # Combine only this package's databases directly using absolute paths
-                    COVERAGE_FILE="${PWD}/.coverage" coverage combine "${pkg_files[@]}" >> "${pkg_log}" 2>&1
+                    COVERAGE_FILE="${pkg_coverage_db}" coverage combine "${pkg_files[@]}" >> "${pkg_log}" 2>&1
 
                     # Generate the coverage report using the package-specific local database
                     if [ -f ".coveragerc" ]; then
                       echo "Using package-specific configuration: ${pkg}/.coveragerc" > "${pkg_log}"
                       if grep -q "fail_under" ".coveragerc"; then
-                        COVERAGE_FILE="${PWD}/.coverage" coverage report --rcfile=".coveragerc" --include="$PWD/**" >> "${pkg_log}" 2>&1
+                        COVERAGE_FILE="${pkg_coverage_db}" coverage report --rcfile=".coveragerc" --include="$PWD/**" >> "${pkg_log}" 2>&1
                       else
                         echo "No fail_under specified in ${pkg}/.coveragerc, enforcing default" >> "${pkg_log}"
-                        COVERAGE_FILE="${PWD}/.coverage" coverage report --rcfile=".coveragerc" --include="$PWD/**" --fail-under="${DEFAULT_FAIL_UNDER}" >> "${pkg_log}" 2>&1
+                        COVERAGE_FILE="${pkg_coverage_db}" coverage report --rcfile=".coveragerc" --include="$PWD/**" --fail-under="${DEFAULT_FAIL_UNDER}" >> "${pkg_log}" 2>&1
                       fi
                     else
                       echo "No .coveragerc found for ${pkg}, enforcing default" > "${pkg_log}"
-                      COVERAGE_FILE="${PWD}/.coverage" coverage report --include="$PWD/**" --fail-under="${DEFAULT_FAIL_UNDER}" >> "${pkg_log}" 2>&1
+                      COVERAGE_FILE="${pkg_coverage_db}" coverage report --include="$PWD/**" --fail-under="${DEFAULT_FAIL_UNDER}" >> "${pkg_log}" 2>&1
                     fi
                     echo $? > "${pkg_status}"
                     popd > /dev/null
@@ -240,8 +240,8 @@ jobs:
           for pkg in ${modified_packages}; do
             if [ -d "${pkg}" ]; then
               pkg_name_clean=$(echo "${pkg}" | sed 's|/$||' | sed 's|/|_|g')
-              pkg_log="/tmp/coverage-logs/${pkg_name_clean}.log"
-              pkg_status="/tmp/coverage-logs/${pkg_name_clean}.status"
+              pkg_log="/dev/shm/coverage-logs/${pkg_name_clean}.log"
+              pkg_status="/dev/shm/coverage-logs/${pkg_name_clean}.status"
 
               echo "============================================================"
               echo "Evaluating coverage for package: ${pkg}"

From e611fe70b9070ba6030491b5474694e90979e4e4 Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Tue, 7 Jul 2026 09:17:22 -0700
Subject: [PATCH 68/84] made max shards configurable

---
 .github/workflows/unittest.yml | 4 ++++
 ci/get_package_shards.py       | 6 +++---
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml
index d4caf9bea04b..0fde42274f23 100644
--- a/.github/workflows/unittest.yml
+++ b/.github/workflows/unittest.yml
@@ -12,6 +12,9 @@ name: unittest
 permissions:
   contents: read
 
+env:
+  MAX_SHARDS: 20
+
 jobs:
   initialize:
     runs-on: ubuntu-latest
@@ -41,6 +44,7 @@ jobs:
         BUILD_TYPE: presubmit
         TARGET_BRANCH: ${{ github.base_ref || github.event.merge_group.base_ref }}
         TEST_ALL_PACKAGES: ${{ steps.check-label.outputs.is_full_run }}
+        MAX_SHARDS: ${{ env.MAX_SHARDS }}
       run: |
         if [ -n "$TARGET_BRANCH" ]; then
           git fetch origin "$TARGET_BRANCH" --deepen=200 || true
diff --git a/ci/get_package_shards.py b/ci/get_package_shards.py
index 62ad719a3ae9..f62dfec8e4c8 100644
--- a/ci/get_package_shards.py
+++ b/ci/get_package_shards.py
@@ -92,11 +92,11 @@ def group_packages(packages):
         package_weights.append((pkg, weight))
         total_weight += weight
 
-    # Dynamically determine target weight to balance across max 16 shards.
+    # Dynamically determine target weight to balance across max shards.
     # 1. Base shard count has a target weight capacity of 10 per shard.
-    # 2. To avoid exceeding 16 shards, we scale the target weight dynamically
+    # 2. To avoid exceeding max shards, we scale the target weight dynamically
     #    based on the total workload.
-    max_shards = 16
+    max_shards = int(os.environ.get("MAX_SHARDS", 16))
     target_weight = max(10, math.ceil(total_weight / max_shards))
 
     shards_list = []

From e453d2cc03822123f90efa99d37cfc63627eaab7 Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Tue, 7 Jul 2026 09:21:08 -0700
Subject: [PATCH 69/84] added test files

---
 packages/google-backstory/SHARD_TEST.txt           | 0
 packages/google-cloud-alloydb/SHARD_TEST.txt       | 0
 packages/google-cloud-audit-log/SHARD_TEST.txt     | 0
 packages/google-cloud-biglake/SHARD_TEST.txt       | 0
 packages/google-cloud-ces/SHARD_TEST.txt           | 0
 packages/google-cloud-dns/SHARD_TEST.txt           | 0
 packages/google-cloud-modelarmor/SHARD_TEST.txt    | 0
 packages/google-cloud-talent/SHARD_TEST.txt        | 0
 packages/google-cloud-vision/SHARD_TEST.txt        | 0
 packages/google-developer-knowledge/SHARD_TEST.txt | 0
 10 files changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 packages/google-backstory/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-alloydb/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-audit-log/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-biglake/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-ces/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-dns/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-modelarmor/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-talent/SHARD_TEST.txt
 create mode 100644 packages/google-cloud-vision/SHARD_TEST.txt
 create mode 100644 packages/google-developer-knowledge/SHARD_TEST.txt

diff --git a/packages/google-backstory/SHARD_TEST.txt b/packages/google-backstory/SHARD_TEST.txt
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/packages/google-cloud-alloydb/SHARD_TEST.txt b/packages/google-cloud-alloydb/SHARD_TEST.txt
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/packages/google-cloud-audit-log/SHARD_TEST.txt b/packages/google-cloud-audit-log/SHARD_TEST.txt
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/packages/google-cloud-biglake/SHARD_TEST.txt b/packages/google-cloud-biglake/SHARD_TEST.txt
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/packages/google-cloud-ces/SHARD_TEST.txt b/packages/google-cloud-ces/SHARD_TEST.txt
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/packages/google-cloud-dns/SHARD_TEST.txt b/packages/google-cloud-dns/SHARD_TEST.txt
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/packages/google-cloud-modelarmor/SHARD_TEST.txt b/packages/google-cloud-modelarmor/SHARD_TEST.txt
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/packages/google-cloud-talent/SHARD_TEST.txt b/packages/google-cloud-talent/SHARD_TEST.txt
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/packages/google-cloud-vision/SHARD_TEST.txt b/packages/google-cloud-vision/SHARD_TEST.txt
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/packages/google-developer-knowledge/SHARD_TEST.txt b/packages/google-developer-knowledge/SHARD_TEST.txt
new file mode 100644
index 000000000000..e69de29bb2d1

From 45b5e68431264b4085138a152e41e556cbd9d506 Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Tue, 7 Jul 2026 09:26:09 -0700
Subject: [PATCH 70/84] define package weights in yaml

---
 .github/workflows/unittest.yml | 20 +++++++++++++++++---
 ci/get_package_shards.py       | 32 ++++++++++++++++----------------
 2 files changed, 33 insertions(+), 19 deletions(-)

diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml
index 0fde42274f23..48dd04f7aed7 100644
--- a/.github/workflows/unittest.yml
+++ b/.github/workflows/unittest.yml
@@ -12,15 +12,28 @@ name: unittest
 permissions:
   contents: read
 
-env:
-  MAX_SHARDS: 20
-
 jobs:
   initialize:
     runs-on: ubuntu-latest
     outputs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
       is_full_run: ${{ steps.check-label.outputs.is_full_run }}
+    env:
+      MAX_SHARDS: 20
+      # Define weights for long-running unit tests to balance shard execution time
+      # Each weight is roughly 1 minute of expected execution time
+      # Default for unset packages is 1
+      PACKAGE_WEIGHTS: |
+        bigframes: 6
+        google-ai-generativelanguage: 4
+        google-auth: 5
+        google-cloud-compute: 12
+        google-cloud-compute-v1beta: 12
+        google-cloud-dialogflow: 6
+        google-cloud-dialogflow-cx: 6
+        google-cloud-discoveryengine: 8
+        google-cloud-retail: 5
+        google-shopping-merchant-accounts: 4
     steps:
     - name: Checkout
       uses: actions/checkout@v4
@@ -45,6 +58,7 @@ jobs:
         TARGET_BRANCH: ${{ github.base_ref || github.event.merge_group.base_ref }}
         TEST_ALL_PACKAGES: ${{ steps.check-label.outputs.is_full_run }}
         MAX_SHARDS: ${{ env.MAX_SHARDS }}
+        PACKAGE_WEIGHTS: ${{ env.PACKAGE_WEIGHTS }}
       run: |
         if [ -n "$TARGET_BRANCH" ]; then
           git fetch origin "$TARGET_BRANCH" --deepen=200 || true
diff --git a/ci/get_package_shards.py b/ci/get_package_shards.py
index f62dfec8e4c8..3e141d722d45 100644
--- a/ci/get_package_shards.py
+++ b/ci/get_package_shards.py
@@ -18,21 +18,20 @@
 import math
 import sys
 
-# Define weights for long-running packages to balance shard execution time.
-# Default weight for any package not listed is 1.
-PACKAGE_WEIGHTS = {
-    "bigframes": 6,
-    "google-ai-generativelanguage": 4,
-    "google-auth": 5,
-    "google-cloud-compute": 12,
-    "google-cloud-compute-v1beta": 12,
-    "google-cloud-dialogflow": 6,
-    "google-cloud-dialogflow-cx": 6,
-    "google-cloud-discoveryengine": 8,
-    "google-cloud-retail": 5,
-    "google-shopping-merchant-accounts": 4,
-
-}
+def get_package_weights():
+    weights = {}
+    env_weights = os.environ.get("PACKAGE_WEIGHTS", "")
+    for line in env_weights.splitlines():
+        line = line.strip()
+        if not line or line.startswith("#"):
+            continue
+        if ":" in line:
+            try:
+                pkg, weight = line.split(":", 1)
+                weights[pkg.strip()] = int(weight.strip())
+            except ValueError:
+                continue
+    return weights
 
 
 def get_packages():
@@ -84,11 +83,12 @@ def group_packages(packages):
         return []
 
     # Map packages to their weights
+    package_weights_map = get_package_weights()
     package_weights = []
     total_weight = 0
     for pkg in packages:
         pkg_name = pkg.strip('/').split('/')[-1]
-        weight = PACKAGE_WEIGHTS.get(pkg_name, 1)
+        weight = package_weights_map.get(pkg_name, 1)
         package_weights.append((pkg, weight))
         total_weight += weight
 

From fa06713b607146ecb97dff432955a602b7b1daa7 Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Tue, 7 Jul 2026 09:43:56 -0700
Subject: [PATCH 71/84] added fast path for coverage check

---
 .github/workflows/unittest.yml | 41 +++++++++++++++++++++-------------
 1 file changed, 26 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml
index 48dd04f7aed7..bfa359e92db8 100644
--- a/.github/workflows/unittest.yml
+++ b/.github/workflows/unittest.yml
@@ -218,26 +218,37 @@ jobs:
                   if [ ${#pkg_files[@]} -gt 0 ]; then
                     pushd "${pkg}" > /dev/null
 
-                    # Use /dev/shm for the combined coverage file to reduce disk I/O
-                    pkg_coverage_db="/dev/shm/.coverage.${pkg_name_clean}"
+                    # Fast-path: try the latest Python version first
+                    latest_pkg_file=$(printf "%s\n" "${pkg_files[@]}" | sort -V | tail -n 1)
+                    echo "Attempting fast-path coverage check with: ${latest_pkg_file}" >> "${pkg_log}"
 
-                    # Combine only this package's databases directly using absolute paths
-                    COVERAGE_FILE="${pkg_coverage_db}" coverage combine "${pkg_files[@]}" >> "${pkg_log}" 2>&1
-
-                    # Generate the coverage report using the package-specific local database
+                    REPORT_FLAGS="--include=$PWD/**"
                     if [ -f ".coveragerc" ]; then
-                      echo "Using package-specific configuration: ${pkg}/.coveragerc" > "${pkg_log}"
-                      if grep -q "fail_under" ".coveragerc"; then
-                        COVERAGE_FILE="${pkg_coverage_db}" coverage report --rcfile=".coveragerc" --include="$PWD/**" >> "${pkg_log}" 2>&1
-                      else
-                        echo "No fail_under specified in ${pkg}/.coveragerc, enforcing default" >> "${pkg_log}"
-                        COVERAGE_FILE="${pkg_coverage_db}" coverage report --rcfile=".coveragerc" --include="$PWD/**" --fail-under="${DEFAULT_FAIL_UNDER}" >> "${pkg_log}" 2>&1
+                      REPORT_FLAGS="${REPORT_FLAGS} --rcfile=.coveragerc"
+                      if ! grep -q "fail_under" ".coveragerc"; then
+                        REPORT_FLAGS="${REPORT_FLAGS} --fail-under=${DEFAULT_FAIL_UNDER}"
                       fi
                     else
-                      echo "No .coveragerc found for ${pkg}, enforcing default" > "${pkg_log}"
-                      COVERAGE_FILE="${pkg_coverage_db}" coverage report --include="$PWD/**" --fail-under="${DEFAULT_FAIL_UNDER}" >> "${pkg_log}" 2>&1
+                      REPORT_FLAGS="${REPORT_FLAGS} --fail-under=${DEFAULT_FAIL_UNDER}"
+                    fi
+
+                    COVERAGE_FILE="${latest_pkg_file}" coverage report ${REPORT_FLAGS} >> "${pkg_log}" 2>&1
+
+                    if [ $? -eq 0 ]; then
+                      echo "Fast-path coverage check passed for ${pkg}" >> "${pkg_log}"
+                      echo 0 > "${pkg_status}"
+                    else
+                      echo "Fast-path insufficient. Combining all available coverage results..." >> "${pkg_log}"
+                      # Use /dev/shm for the combined coverage file to reduce disk I/O
+                      pkg_coverage_db="/dev/shm/.coverage.${pkg_name_clean}"
+
+                      # Combine only this package's databases directly using absolute paths
+                      COVERAGE_FILE="${pkg_coverage_db}" coverage combine "${pkg_files[@]}" >> "${pkg_log}" 2>&1
+
+                      # Generate the coverage report using the package-specific local database
+                      COVERAGE_FILE="${pkg_coverage_db}" coverage report ${REPORT_FLAGS} >> "${pkg_log}" 2>&1
+                      echo $? > "${pkg_status}"
                     fi
-                    echo $? > "${pkg_status}"
                     popd > /dev/null
                   else
                     echo "Warning: No coverage results found for ${pkg}" > "${pkg_log}"

From cd33370acb480cdf216e9078833c1d84293f15cc Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Tue, 7 Jul 2026 10:00:28 -0700
Subject: [PATCH 72/84] undo fast-path optimization

---
 .github/workflows/unittest.yml | 41 +++++++++++++---------------------
 1 file changed, 15 insertions(+), 26 deletions(-)

diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml
index bfa359e92db8..48dd04f7aed7 100644
--- a/.github/workflows/unittest.yml
+++ b/.github/workflows/unittest.yml
@@ -218,37 +218,26 @@ jobs:
                   if [ ${#pkg_files[@]} -gt 0 ]; then
                     pushd "${pkg}" > /dev/null
 
-                    # Fast-path: try the latest Python version first
-                    latest_pkg_file=$(printf "%s\n" "${pkg_files[@]}" | sort -V | tail -n 1)
-                    echo "Attempting fast-path coverage check with: ${latest_pkg_file}" >> "${pkg_log}"
+                    # Use /dev/shm for the combined coverage file to reduce disk I/O
+                    pkg_coverage_db="/dev/shm/.coverage.${pkg_name_clean}"
 
-                    REPORT_FLAGS="--include=$PWD/**"
+                    # Combine only this package's databases directly using absolute paths
+                    COVERAGE_FILE="${pkg_coverage_db}" coverage combine "${pkg_files[@]}" >> "${pkg_log}" 2>&1
+
+                    # Generate the coverage report using the package-specific local database
                     if [ -f ".coveragerc" ]; then
-                      REPORT_FLAGS="${REPORT_FLAGS} --rcfile=.coveragerc"
-                      if ! grep -q "fail_under" ".coveragerc"; then
-                        REPORT_FLAGS="${REPORT_FLAGS} --fail-under=${DEFAULT_FAIL_UNDER}"
+                      echo "Using package-specific configuration: ${pkg}/.coveragerc" > "${pkg_log}"
+                      if grep -q "fail_under" ".coveragerc"; then
+                        COVERAGE_FILE="${pkg_coverage_db}" coverage report --rcfile=".coveragerc" --include="$PWD/**" >> "${pkg_log}" 2>&1
+                      else
+                        echo "No fail_under specified in ${pkg}/.coveragerc, enforcing default" >> "${pkg_log}"
+                        COVERAGE_FILE="${pkg_coverage_db}" coverage report --rcfile=".coveragerc" --include="$PWD/**" --fail-under="${DEFAULT_FAIL_UNDER}" >> "${pkg_log}" 2>&1
                       fi
                     else
-                      REPORT_FLAGS="${REPORT_FLAGS} --fail-under=${DEFAULT_FAIL_UNDER}"
-                    fi
-
-                    COVERAGE_FILE="${latest_pkg_file}" coverage report ${REPORT_FLAGS} >> "${pkg_log}" 2>&1
-
-                    if [ $? -eq 0 ]; then
-                      echo "Fast-path coverage check passed for ${pkg}" >> "${pkg_log}"
-                      echo 0 > "${pkg_status}"
-                    else
-                      echo "Fast-path insufficient. Combining all available coverage results..." >> "${pkg_log}"
-                      # Use /dev/shm for the combined coverage file to reduce disk I/O
-                      pkg_coverage_db="/dev/shm/.coverage.${pkg_name_clean}"
-
-                      # Combine only this package's databases directly using absolute paths
-                      COVERAGE_FILE="${pkg_coverage_db}" coverage combine "${pkg_files[@]}" >> "${pkg_log}" 2>&1
-
-                      # Generate the coverage report using the package-specific local database
-                      COVERAGE_FILE="${pkg_coverage_db}" coverage report ${REPORT_FLAGS} >> "${pkg_log}" 2>&1
-                      echo $? > "${pkg_status}"
+                      echo "No .coveragerc found for ${pkg}, enforcing default" > "${pkg_log}"
+                      COVERAGE_FILE="${pkg_coverage_db}" coverage report --include="$PWD/**" --fail-under="${DEFAULT_FAIL_UNDER}" >> "${pkg_log}" 2>&1
                     fi
+                    echo $? > "${pkg_status}"
                     popd > /dev/null
                   else
                     echo "Warning: No coverage results found for ${pkg}" > "${pkg_log}"

From 5bfe2d180cc275684c930e15d213d7db4f49e60a Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Tue, 7 Jul 2026 10:19:58 -0700
Subject: [PATCH 73/84] only upload coverage for first and last supported
 version

---
 .github/workflows/unittest.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml
index 48dd04f7aed7..bd9d63193a77 100644
--- a/.github/workflows/unittest.yml
+++ b/.github/workflows/unittest.yml
@@ -18,6 +18,7 @@ jobs:
     outputs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
       is_full_run: ${{ steps.check-label.outputs.is_full_run }}
+      coverage_versions: ${{ steps.set-versions.outputs.coverage_versions }}
     env:
       MAX_SHARDS: 20
       # Define weights for long-running unit tests to balance shard execution time
@@ -39,6 +40,10 @@ jobs:
       uses: actions/checkout@v4
       with:
         fetch-depth: 0
+    - name: Set Coverage versions
+      id: set-versions
+      run: |
+        echo 'coverage_versions=["3.9", "3.14"]' >> $GITHUB_OUTPUT
     - name: Check for unit_test:all_packages label
       id: check-label
       run: |
@@ -103,6 +108,7 @@ jobs:
       run: |
         ci/run_conditional_tests.sh
     - name: Upload coverage results
+      if: ${{ steps.packages.outputs.num_files_changed > 0 && contains(fromJson(needs.initialize.outputs.coverage_versions), matrix.python) }}
       uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
       with:
         name: coverage-artifact-${{ matrix.python }}-${{ matrix.package_shard.index }}

From 6c0566fb4b159df6fd251452918830478e99bcd7 Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Tue, 7 Jul 2026 10:23:30 -0700
Subject: [PATCH 74/84] use 3.10 as lower bound

---
 .github/workflows/unittest.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml
index bd9d63193a77..28999bdc636c 100644
--- a/.github/workflows/unittest.yml
+++ b/.github/workflows/unittest.yml
@@ -43,7 +43,7 @@ jobs:
     - name: Set Coverage versions
       id: set-versions
       run: |
-        echo 'coverage_versions=["3.9", "3.14"]' >> $GITHUB_OUTPUT
+        echo 'coverage_versions=["3.10", "3.14"]' >> $GITHUB_OUTPUT
     - name: Check for unit_test:all_packages label
       id: check-label
       run: |

From e1bedf12d89955455b4e75999accff249722f0fb Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Tue, 7 Jul 2026 11:57:13 -0700
Subject: [PATCH 75/84] avoid going over max_shards

---
 ci/get_package_shards.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/ci/get_package_shards.py b/ci/get_package_shards.py
index 3e141d722d45..f32be2f50445 100644
--- a/ci/get_package_shards.py
+++ b/ci/get_package_shards.py
@@ -93,9 +93,6 @@ def group_packages(packages):
         total_weight += weight
 
     # Dynamically determine target weight to balance across max shards.
-    # 1. Base shard count has a target weight capacity of 10 per shard.
-    # 2. To avoid exceeding max shards, we scale the target weight dynamically
-    #    based on the total workload.
     max_shards = int(os.environ.get("MAX_SHARDS", 16))
     target_weight = max(10, math.ceil(total_weight / max_shards))
 
@@ -105,8 +102,9 @@ def group_packages(packages):
 
     # Pack packages alphabetically and contiguously.
     for pkg, weight in package_weights:
-        # Check if adding this package would exceed the target weight
-        if current_shard_packages and (current_shard_weight + weight > target_weight):
+        # If adding this package would exceed target weight AND we haven't reached the 
+        # shard limit, start a new shard. Otherwise, keep "stuffing" the current one.
+        if current_shard_packages and (current_shard_weight + weight > target_weight) and len(shards_list) < max_shards - 1:
             shards_list.append(current_shard_packages)
             current_shard_packages = [pkg]
             current_shard_weight = weight

From 60a3e74b7811c8feb1467b26f54a365828f568b8 Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Tue, 7 Jul 2026 12:01:11 -0700
Subject: [PATCH 76/84] fixed coverage condition

---
 .github/workflows/unittest.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml
index 28999bdc636c..66a861e4b81e 100644
--- a/.github/workflows/unittest.yml
+++ b/.github/workflows/unittest.yml
@@ -108,7 +108,7 @@ jobs:
       run: |
         ci/run_conditional_tests.sh
     - name: Upload coverage results
-      if: ${{ steps.packages.outputs.num_files_changed > 0 && contains(fromJson(needs.initialize.outputs.coverage_versions), matrix.python) }}
+      if: ${{ contains(fromJson(needs.initialize.outputs.coverage_versions), matrix.python) }}
       uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
       with:
         name: coverage-artifact-${{ matrix.python }}-${{ matrix.package_shard.index }}

From a4566cfc182fc3139b4c9a753d03b4a35287fbb0 Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Tue, 7 Jul 2026 12:53:30 -0700
Subject: [PATCH 77/84] use all runtimes in coverage

---
 .github/workflows/unittest.yml | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml
index 66a861e4b81e..48dd04f7aed7 100644
--- a/.github/workflows/unittest.yml
+++ b/.github/workflows/unittest.yml
@@ -18,7 +18,6 @@ jobs:
     outputs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
       is_full_run: ${{ steps.check-label.outputs.is_full_run }}
-      coverage_versions: ${{ steps.set-versions.outputs.coverage_versions }}
     env:
       MAX_SHARDS: 20
       # Define weights for long-running unit tests to balance shard execution time
@@ -40,10 +39,6 @@ jobs:
       uses: actions/checkout@v4
       with:
         fetch-depth: 0
-    - name: Set Coverage versions
-      id: set-versions
-      run: |
-        echo 'coverage_versions=["3.10", "3.14"]' >> $GITHUB_OUTPUT
     - name: Check for unit_test:all_packages label
       id: check-label
       run: |
@@ -108,7 +103,6 @@ jobs:
       run: |
         ci/run_conditional_tests.sh
     - name: Upload coverage results
-      if: ${{ contains(fromJson(needs.initialize.outputs.coverage_versions), matrix.python) }}
       uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
       with:
         name: coverage-artifact-${{ matrix.python }}-${{ matrix.package_shard.index }}

From 010e1ad916892f4d9e2f791221e823f9567bb492 Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Tue, 7 Jul 2026 12:53:49 -0700
Subject: [PATCH 78/84] remove empty files

---
 packages/google-backstory/SHARD_TEST.txt           | 0
 packages/google-cloud-alloydb/SHARD_TEST.txt       | 0
 packages/google-cloud-audit-log/SHARD_TEST.txt     | 0
 packages/google-cloud-biglake/SHARD_TEST.txt       | 0
 packages/google-cloud-ces/SHARD_TEST.txt           | 0
 packages/google-cloud-dns/SHARD_TEST.txt           | 0
 packages/google-cloud-modelarmor/SHARD_TEST.txt    | 0
 packages/google-cloud-talent/SHARD_TEST.txt        | 0
 packages/google-cloud-vision/SHARD_TEST.txt        | 0
 packages/google-developer-knowledge/SHARD_TEST.txt | 0
 10 files changed, 0 insertions(+), 0 deletions(-)
 delete mode 100644 packages/google-backstory/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-alloydb/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-audit-log/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-biglake/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-ces/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-dns/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-modelarmor/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-talent/SHARD_TEST.txt
 delete mode 100644 packages/google-cloud-vision/SHARD_TEST.txt
 delete mode 100644 packages/google-developer-knowledge/SHARD_TEST.txt

diff --git a/packages/google-backstory/SHARD_TEST.txt b/packages/google-backstory/SHARD_TEST.txt
deleted file mode 100644
index e69de29bb2d1..000000000000
diff --git a/packages/google-cloud-alloydb/SHARD_TEST.txt b/packages/google-cloud-alloydb/SHARD_TEST.txt
deleted file mode 100644
index e69de29bb2d1..000000000000
diff --git a/packages/google-cloud-audit-log/SHARD_TEST.txt b/packages/google-cloud-audit-log/SHARD_TEST.txt
deleted file mode 100644
index e69de29bb2d1..000000000000
diff --git a/packages/google-cloud-biglake/SHARD_TEST.txt b/packages/google-cloud-biglake/SHARD_TEST.txt
deleted file mode 100644
index e69de29bb2d1..000000000000
diff --git a/packages/google-cloud-ces/SHARD_TEST.txt b/packages/google-cloud-ces/SHARD_TEST.txt
deleted file mode 100644
index e69de29bb2d1..000000000000
diff --git a/packages/google-cloud-dns/SHARD_TEST.txt b/packages/google-cloud-dns/SHARD_TEST.txt
deleted file mode 100644
index e69de29bb2d1..000000000000
diff --git a/packages/google-cloud-modelarmor/SHARD_TEST.txt b/packages/google-cloud-modelarmor/SHARD_TEST.txt
deleted file mode 100644
index e69de29bb2d1..000000000000
diff --git a/packages/google-cloud-talent/SHARD_TEST.txt b/packages/google-cloud-talent/SHARD_TEST.txt
deleted file mode 100644
index e69de29bb2d1..000000000000
diff --git a/packages/google-cloud-vision/SHARD_TEST.txt b/packages/google-cloud-vision/SHARD_TEST.txt
deleted file mode 100644
index e69de29bb2d1..000000000000
diff --git a/packages/google-developer-knowledge/SHARD_TEST.txt b/packages/google-developer-knowledge/SHARD_TEST.txt
deleted file mode 100644
index e69de29bb2d1..000000000000

From e35ea345ca84bcd88300f05506da382f0b48895c Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Tue, 7 Jul 2026 12:57:16 -0700
Subject: [PATCH 79/84] removed duplication

---
 ci/run_conditional_tests.sh | 71 ++++++++++++++++---------------------
 1 file changed, 31 insertions(+), 40 deletions(-)

diff --git a/ci/run_conditional_tests.sh b/ci/run_conditional_tests.sh
index fb68904ead6a..b43760187e02 100755
--- a/ci/run_conditional_tests.sh
+++ b/ci/run_conditional_tests.sh
@@ -47,28 +47,38 @@ git config --global url."${PROJECT_ROOT}".insteadOf "https://github.com/googleap
 # A script file for running the test in a sub project.
 test_script="${PROJECT_ROOT}/ci/run_single_test.sh"
 
+# Global exit code tracker
+RETVAL=0
+
+# Shared test execution logic
+run_test_in_dir() {
+    local d=$1
+    echo "running test in ${d}"
+    pushd ${d} > /dev/null
+    
+    # Temporarily allow failure.
+    set +e
+    
+    # Ensure unique coverage file per package to avoid DataError
+    # when combining statement and branch coverage.
+    # Strip trailing slash from directory name for the filename.
+    local pkg_name_clean=$(echo ${d} | sed 's|/$||' | sed 's|/|_|g')
+    export COVERAGE_FILE="${PROJECT_ROOT}/.coverage.${PY_VERSION}.${pkg_name_clean}"
+    
+    ${test_script}
+    local ret=$?
+    set -e
+    
+    if [ ${ret} -ne 0 ]; then
+        RETVAL=${ret}
+    fi
+    popd > /dev/null
+}
+
 if [ -n "${PACKAGE_LIST}" ]; then
     echo "Using provided PACKAGE_LIST"
-    to_test=(${PACKAGE_LIST})
-    RETVAL=0
-    for d in ${to_test[@]}; do
-        echo "running test in ${d}"
-        pushd ${d}
-        set +e
-        
-        # Ensure unique coverage file per package to avoid DataError
-        # when combining statement and branch coverage.
-        # Strip trailing slash from directory name for the filename.
-        pkg_name_clean=$(echo ${d} | sed 's|/$||' | sed 's|/|_|g')
-        export COVERAGE_FILE="${PROJECT_ROOT}/.coverage.${PY_VERSION}.${pkg_name_clean}"
-        
-        ${test_script}
-        ret=$?
-        set -e
-        if [ ${ret} -ne 0 ]; then
-            RETVAL=${ret}
-        fi
-        popd
+    for d in ${PACKAGE_LIST}; do
+        run_test_in_dir "${d}"
     done
     exit ${RETVAL}
 fi
@@ -102,8 +112,6 @@ TOTAL_SHARDS="${TOTAL_SHARDS:-1}"
 SHARD_INDEX="${SHARD_INDEX:-1}"
 count=0
 
-RETVAL=0
-
 for subdir in ${subdirs[@]}; do
     # Sort the directories to ensure consistent sharding across jobs
     for d in `ls -d ${subdir}/*/ | sort`; do
@@ -132,24 +140,7 @@ for subdir in ${subdirs[@]}; do
             should_test=true
         fi
         if [ "${should_test}" = true ]; then
-            echo "running test in ${d}"
-            pushd ${d}
-            # Temporarily allow failure.
-            set +e
-            
-            # Ensure unique coverage file per package to avoid DataError
-            # when combining statement and branch coverage.
-            # Strip trailing slash from directory name for the filename.
-            pkg_name_clean=$(echo ${d} | sed 's|/$||' | sed 's|/|_|g')
-            export COVERAGE_FILE="${PROJECT_ROOT}/.coverage.${PY_VERSION}.${pkg_name_clean}"
-            
-            ${test_script}
-            ret=$?
-            set -e
-            if [ ${ret} -ne 0 ]; then
-                RETVAL=${ret}
-            fi
-            popd
+            run_test_in_dir "${d}"
         fi
     done
 done

From d04104d2cd3863d6041b505de42afd16d52c646c Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Tue, 7 Jul 2026 13:08:17 -0700
Subject: [PATCH 80/84] cleaned up conditional tests

---
 ci/run_conditional_tests.sh | 33 +++++++++++++--------------------
 1 file changed, 13 insertions(+), 20 deletions(-)

diff --git a/ci/run_conditional_tests.sh b/ci/run_conditional_tests.sh
index b43760187e02..58dce199391f 100755
--- a/ci/run_conditional_tests.sh
+++ b/ci/run_conditional_tests.sh
@@ -55,20 +55,20 @@ run_test_in_dir() {
     local d=$1
     echo "running test in ${d}"
     pushd ${d} > /dev/null
-    
+
     # Temporarily allow failure.
     set +e
-    
+
     # Ensure unique coverage file per package to avoid DataError
     # when combining statement and branch coverage.
     # Strip trailing slash from directory name for the filename.
     local pkg_name_clean=$(echo ${d} | sed 's|/$||' | sed 's|/|_|g')
     export COVERAGE_FILE="${PROJECT_ROOT}/.coverage.${PY_VERSION}.${pkg_name_clean}"
-    
+
     ${test_script}
     local ret=$?
     set -e
-    
+
     if [ ${ret} -ne 0 ]; then
         RETVAL=${ret}
     fi
@@ -104,24 +104,17 @@ else
     GIT_DIFF_ARG=""
 fi
 
-# Sharding logic (fallback for manual runs)
-subdirs=(
-    packages
-)
-TOTAL_SHARDS="${TOTAL_SHARDS:-1}"
-SHARD_INDEX="${SHARD_INDEX:-1}"
-count=0
+# Fallback for when no package list is provided
+# Detect changes in test scripts
 
-for subdir in ${subdirs[@]}; do
-    # Sort the directories to ensure consistent sharding across jobs
-    for d in `ls -d ${subdir}/*/ | sort`; do
-        # Sharding logic: only process directories that belong to this shard
-        if (( count % TOTAL_SHARDS != SHARD_INDEX - 1 )); then
-            ((++count))
-            continue
-        fi
-        ((++count))
+set +e
+git diff --quiet ${GIT_DIFF_ARG} ci
+changed=$?
+set -e
 
+
+for subdir in ${subdirs[@]}; do
+    for d in `ls -d ${subdir}/*/`; do
         should_test=false
         if [ -n "${GIT_DIFF_ARG}" ]; then
             echo "checking changes with 'git diff --quiet ${GIT_DIFF_ARG} ${d}'"

From cf3de20b7c809f12f8eea23b2f1fe16e33145b3f Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Tue, 7 Jul 2026 13:10:55 -0700
Subject: [PATCH 81/84] added back subdirs

---
 ci/run_conditional_tests.sh | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/ci/run_conditional_tests.sh b/ci/run_conditional_tests.sh
index 58dce199391f..33ee5c8e4f58 100755
--- a/ci/run_conditional_tests.sh
+++ b/ci/run_conditional_tests.sh
@@ -112,6 +112,12 @@ git diff --quiet ${GIT_DIFF_ARG} ci
 changed=$?
 set -e
 
+# Now we have a fixed list, but we can change it to autodetect if
+# necessary.
+
+subdirs=(
+    packages
+)
 
 for subdir in ${subdirs[@]}; do
     for d in `ls -d ${subdir}/*/`; do

From f1eb3a49c2b3c6388afb4ef9b6b5d35d0409fb2f Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Tue, 7 Jul 2026 13:14:33 -0700
Subject: [PATCH 82/84] added comments to sharding script

---
 ci/get_package_shards.py | 49 +++++++++++++++++++++++++++++++++++++++-
 1 file changed, 48 insertions(+), 1 deletion(-)

diff --git a/ci/get_package_shards.py b/ci/get_package_shards.py
index f32be2f50445..3765d75cb3ab 100644
--- a/ci/get_package_shards.py
+++ b/ci/get_package_shards.py
@@ -12,6 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+"""Script to group changed packages into balanced shards for CI testing.
+
+This script identifies which packages have changed compared to a target branch
+(or since the last commit) and groups them into a fixed number of shards.
+It uses package weights (configured via environment variables) to balance the
+execution time across shards while maintaining alphabetical order and
+contiguous grouping.
+"""
+
 import os
 import subprocess
 import json
@@ -19,6 +28,17 @@
 import sys
 
 def get_package_weights():
+    """Parses package weights from the PACKAGE_WEIGHTS environment variable.
+    Package weights represent the relative size of the test, where the weight
+    approximately represents the minutes the test takes to run. This is used
+    for distributing work properly across shards.
+
+    The environment variable is expected to be a multiline string where each line
+    is in the format 'package_name: weight'. Lines starting with '#' are ignored.
+
+    Returns:
+        dict: A mapping of package names to their integer weights.
+    """
     weights = {}
     env_weights = os.environ.get("PACKAGE_WEIGHTS", "")
     for line in env_weights.splitlines():
@@ -35,6 +55,11 @@ def get_package_weights():
 
 
 def get_packages():
+    """Lists all package directories in the repository.
+
+    Returns:
+        list: A sorted list of relative paths to all directories under 'packages/'.
+    """
     subdirs = ['packages']
     packages = []
     for subdir in subdirs:
@@ -47,6 +72,15 @@ def get_packages():
 
 
 def get_packages_to_test():
+    """Determines the list of packages that need to be tested.
+
+    This is based on git diffs against the target branch (presubmit) or the
+    previous commit (continuous). If TEST_ALL_PACKAGES is set to true,
+    all packages are returned.
+
+    Returns:
+        list: A list of package directory paths to be included in the test matrix.
+    """
     build_type = os.environ.get('BUILD_TYPE', 'presubmit')
     target_branch = os.environ.get('TARGET_BRANCH', 'main')
     test_all_packages = os.environ.get('TEST_ALL_PACKAGES', 'false').lower() == 'true'
@@ -79,6 +113,19 @@ def get_packages_to_test():
 
 
 def group_packages(packages):
+    """Groups the provided packages into balanced shards.
+
+    The grouping respects the MAX_SHARDS limit. It uses a target weight per shard
+    to distribute the workload. If the contiguous packing exceeds the shard limit,
+    all remaining packages are added to the final shard.
+
+    Args:
+        packages (list): The list of package directory paths to group.
+
+    Returns:
+        list: A list of dictionaries, each representing a shard with its name,
+              index, description, and the space-separated list of packages.
+    """
     if not packages:
         return []
 
@@ -121,7 +168,7 @@ def group_packages(packages):
         index = i + 1
         name = f"Shard {index}"
         num_in_shard = len(shard_packages)
-        
+
         # Calculate contiguous range description
         if len(shard_packages) == 1:
             desc = shard_packages[0].strip('/').split('/')[-1]

From c70613a661249767c6ebf957177d6cb66810e263 Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Tue, 7 Jul 2026 13:19:30 -0700
Subject: [PATCH 83/84] moved script out of yaml

---
 .github/workflows/unittest.yml | 122 +-------------------------
 ci/report_coverage.sh          | 153 +++++++++++++++++++++++++++++++++
 2 files changed, 154 insertions(+), 121 deletions(-)
 create mode 100755 ci/report_coverage.sh

diff --git a/.github/workflows/unittest.yml b/.github/workflows/unittest.yml
index 48dd04f7aed7..996c7e4b03b2 100644
--- a/.github/workflows/unittest.yml
+++ b/.github/workflows/unittest.yml
@@ -177,124 +177,4 @@ jobs:
         DEFAULT_FAIL_UNDER: 99
         TEST_ALL_PACKAGES: ${{ needs.initialize.outputs.is_full_run }}
       run: |
-        RESULTS_DIR="/dev/shm/.coverage-results"
-        if [ -d "$RESULTS_DIR" ]; then
-          # Unzip any zipped coverage results
-          find "$RESULTS_DIR" -type f -name '*.zip' -print0 | xargs -0 -P $(nproc) -I {} unzip -q -o {} -d "$RESULTS_DIR"
-
-          # Find all modified packages
-          if [[ "${TEST_ALL_PACKAGES}" == "true" ]]; then
-            modified_packages=$(ls -d packages/*/ | cut -d/ -f1,2 | sort -u)
-          else
-            modified_packages=$(git diff --name-only HEAD~1 -- packages | cut -d/ -f1,2 | sort -u)
-          fi
-
-          mkdir -p /dev/shm/coverage-logs
-          MAX_JOBS=4
-
-          for pkg in ${modified_packages}; do
-            if [ -d "${pkg}" ]; then
-              # Limit the number of concurrent background jobs
-              while [ $(jobs -r | wc -l) -ge ${MAX_JOBS} ]; do
-                sleep 0.1
-              done
-
-              (
-                pkg_name_clean=$(echo "${pkg}" | sed 's|/$||' | sed 's|/|_|g')
-                pkg_log="/dev/shm/coverage-logs/${pkg_name_clean}.log"
-                pkg_status="/dev/shm/coverage-logs/${pkg_name_clean}.status"
-
-                set +e
-
-                # Find coverage databases belonging specifically to this package across all Python versions
-                shopt -s nullglob
-                pkg_files=("${RESULTS_DIR}/.coverage."*"."*"${pkg_name_clean}" "${RESULTS_DIR}"/*/".coverage."*"."*"${pkg_name_clean}")
-                shopt -u nullglob
-
-                if [ -f "${pkg}/.coveragerc" ] && grep -Eq "fail_under\s*=\s*0" "${pkg}/.coveragerc"; then
-                  echo "Package has fail_under = 0, passing instantly with 100% success (even without coverage files)" > "${pkg_log}"
-                  echo 0 > "${pkg_status}"
-                else
-                  if [ ${#pkg_files[@]} -gt 0 ]; then
-                    pushd "${pkg}" > /dev/null
-
-                    # Use /dev/shm for the combined coverage file to reduce disk I/O
-                    pkg_coverage_db="/dev/shm/.coverage.${pkg_name_clean}"
-
-                    # Combine only this package's databases directly using absolute paths
-                    COVERAGE_FILE="${pkg_coverage_db}" coverage combine "${pkg_files[@]}" >> "${pkg_log}" 2>&1
-
-                    # Generate the coverage report using the package-specific local database
-                    if [ -f ".coveragerc" ]; then
-                      echo "Using package-specific configuration: ${pkg}/.coveragerc" > "${pkg_log}"
-                      if grep -q "fail_under" ".coveragerc"; then
-                        COVERAGE_FILE="${pkg_coverage_db}" coverage report --rcfile=".coveragerc" --include="$PWD/**" >> "${pkg_log}" 2>&1
-                      else
-                        echo "No fail_under specified in ${pkg}/.coveragerc, enforcing default" >> "${pkg_log}"
-                        COVERAGE_FILE="${pkg_coverage_db}" coverage report --rcfile=".coveragerc" --include="$PWD/**" --fail-under="${DEFAULT_FAIL_UNDER}" >> "${pkg_log}" 2>&1
-                      fi
-                    else
-                      echo "No .coveragerc found for ${pkg}, enforcing default" > "${pkg_log}"
-                      COVERAGE_FILE="${pkg_coverage_db}" coverage report --include="$PWD/**" --fail-under="${DEFAULT_FAIL_UNDER}" >> "${pkg_log}" 2>&1
-                    fi
-                    echo $? > "${pkg_status}"
-                    popd > /dev/null
-                  else
-                    echo "Warning: No coverage results found for ${pkg}" > "${pkg_log}"
-                    echo 1 > "${pkg_status}"
-                  fi
-                fi
-                set -e
-              ) &
-            fi
-          done
-
-          # Wait for all background checks to finish
-          wait
-
-          failed_packages=()
-          passed_packages=()
-
-          for pkg in ${modified_packages}; do
-            if [ -d "${pkg}" ]; then
-              pkg_name_clean=$(echo "${pkg}" | sed 's|/$||' | sed 's|/|_|g')
-              pkg_log="/dev/shm/coverage-logs/${pkg_name_clean}.log"
-              pkg_status="/dev/shm/coverage-logs/${pkg_name_clean}.status"
-
-              echo "============================================================"
-              echo "Evaluating coverage for package: ${pkg}"
-              echo "============================================================"
-              if [ -f "${pkg_log}" ]; then
-                cat "${pkg_log}"
-              fi
-
-              status=$(cat "${pkg_status}")
-              if [ "${status}" -ne 0 ]; then
-                failed_packages+=("${pkg}")
-              else
-                passed_packages+=("${pkg}")
-              fi
-            fi
-          done
-
-          echo "============================================================"
-          echo "Coverage Evaluation Summary"
-          echo "============================================================"
-          if [ ${#passed_packages[@]} -gt 0 ]; then
-            echo "Passed packages:"
-            for pkg in "${passed_packages[@]}"; do
-              echo "  - ${pkg}"
-            done
-          fi
-          if [ ${#failed_packages[@]} -gt 0 ]; then
-            echo "Failed packages:"
-            for pkg in "${failed_packages[@]}"; do
-              echo "  - ${pkg}"
-            done
-            exit 1
-          fi
-        else
-          echo "Error: No coverage results were downloaded from the unit test jobs."
-          echo "This usually means the unit tests did not run or failed to upload their coverage files."
-          exit 1
-        fi
+        ci/report_coverage.sh
diff --git a/ci/report_coverage.sh b/ci/report_coverage.sh
new file mode 100755
index 000000000000..dbd418503f89
--- /dev/null
+++ b/ci/report_coverage.sh
@@ -0,0 +1,153 @@
+#!/bin/bash
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eo pipefail
+
+# This script generates coverage reports for modified packages.
+# It assumes coverage databases have been downloaded to RESULTS_DIR.
+# It uses /dev/shm for high-performance processing.
+
+RESULTS_DIR="${RESULTS_DIR:-/dev/shm/.coverage-results}"
+LOG_DIR="/dev/shm/coverage-logs"
+DEFAULT_FAIL_UNDER="${DEFAULT_FAIL_UNDER:-99}"
+MAX_JOBS=$(nproc)
+
+mkdir -p "${LOG_DIR}"
+
+if [ ! -d "${RESULTS_DIR}" ]; then
+    echo "Error: No coverage results found in ${RESULTS_DIR}."
+    exit 1
+fi
+
+# Unzip any zipped coverage results
+find "$RESULTS_DIR" -type f -name '*.zip' -print0 | xargs -0 -P "${MAX_JOBS}" -I {} unzip -q -o {} -d "$RESULTS_DIR"
+
+# Identify modified packages
+if [[ "${TEST_ALL_PACKAGES}" == "true" ]]; then
+    modified_packages=$(ls -d packages/*/ | cut -d/ -f1,2 | sort -u)
+else
+    # Default to comparing against previous commit
+    modified_packages=$(git diff --name-only HEAD~1 -- packages | cut -d/ -f1,2 | sort -u)
+fi
+
+# Function to report coverage for a single package
+report_package_coverage() {
+    local pkg=$1
+    local pkg_name_clean=$(echo "${pkg}" | sed 's|/$||' | sed 's|/|_|g')
+    local pkg_log="${LOG_DIR}/${pkg_name_clean}.log"
+    local pkg_status="${LOG_DIR}/${pkg_name_clean}.status"
+    
+    # Use /dev/shm for the combined coverage file to reduce disk I/O
+    local pkg_coverage_db="/dev/shm/.coverage.${pkg_name_clean}"
+
+    set +e
+    (
+        # Find coverage databases belonging specifically to this package
+        shopt -s nullglob
+        local pkg_files=("${RESULTS_DIR}/.coverage."*"."*"${pkg_name_clean}" "${RESULTS_DIR}"/*/".coverage."*"."*"${pkg_name_clean}")
+        shopt -u nullglob
+
+        # Handle explicit skip via .coveragerc
+        if [ -f "${pkg}/.coveragerc" ] && grep -Eq "fail_under\s*=\s*0" "${pkg}/.coveragerc"; then
+            echo "Package has fail_under = 0, passing instantly with 100% success (even without coverage files)" > "${pkg_log}"
+            echo 0 > "${pkg_status}"
+            return
+        fi
+
+        if [ ${#pkg_files[@]} -eq 0 ]; then
+            echo "Warning: No coverage results found for ${pkg}" > "${pkg_log}"
+            echo 1 > "${pkg_status}"
+            return
+        fi
+
+        pushd "${pkg}" > /dev/null
+
+        # Combine databases
+        COVERAGE_FILE="${pkg_coverage_db}" coverage combine "${pkg_files[@]}" >> "${pkg_log}" 2>&1
+
+        # Generate report
+        if [ -f ".coveragerc" ]; then
+            echo "Using package-specific configuration: ${pkg}/.coveragerc" > "${pkg_log}"
+            if grep -q "fail_under" ".coveragerc"; then
+                COVERAGE_FILE="${pkg_coverage_db}" coverage report --rcfile=".coveragerc" --include="$PWD/**" >> "${pkg_log}" 2>&1
+            else
+                echo "No fail_under specified in ${pkg}/.coveragerc, enforcing default" >> "${pkg_log}"
+                COVERAGE_FILE="${pkg_coverage_db}" coverage report --rcfile=".coveragerc" --include="$PWD/**" --fail-under="${DEFAULT_FAIL_UNDER}" >> "${pkg_log}" 2>&1
+            fi
+        else
+            echo "No .coveragerc found for ${pkg}, enforcing default" > "${pkg_log}"
+            COVERAGE_FILE="${pkg_coverage_db}" coverage report --include="$PWD/**" --fail-under="${DEFAULT_FAIL_UNDER}" >> "${pkg_log}" 2>&1
+        fi
+        
+        echo $? > "${pkg_status}"
+        popd > /dev/null
+    )
+    set -e
+}
+
+# Run coverage reporting in parallel
+for pkg in ${modified_packages}; do
+    if [ -d "${pkg}" ]; then
+        # Limit concurrent jobs
+        while [ $(jobs -r | wc -l) -ge "${MAX_JOBS}" ]; do
+            sleep 0.1
+        done
+        report_package_coverage "${pkg}" &
+    fi
+done
+
+wait
+
+# Summary and exit
+failed_packages=()
+passed_packages=()
+
+for pkg in ${modified_packages}; do
+    if [ -d "${pkg}" ]; then
+        pkg_name_clean=$(echo "${pkg}" | sed 's|/$||' | sed 's|/|_|g')
+        pkg_log="${LOG_DIR}/${pkg_name_clean}.log"
+        pkg_status="${LOG_DIR}/${pkg_name_clean}.status"
+
+        echo "============================================================"
+        echo "Evaluating coverage for package: ${pkg}"
+        echo "============================================================"
+        [ -f "${pkg_log}" ] && cat "${pkg_log}"
+
+        status=$(cat "${pkg_status}" 2>/dev/null || echo 1)
+        if [ "${status}" -ne 0 ]; then
+            failed_packages+=("${pkg}")
+        else
+            passed_packages+=("${pkg}")
+        fi
+    fi
+done
+
+echo "============================================================"
+echo "Coverage Evaluation Summary"
+echo "============================================================"
+if [ ${#passed_packages[@]} -gt 0 ]; then
+    echo "Passed packages:"
+    for pkg in "${passed_packages[@]}"; do
+        echo "  - ${pkg}"
+    done
+fi
+
+if [ ${#failed_packages[@]} -gt 0 ]; then
+    echo "Failed packages:"
+    for pkg in "${failed_packages[@]}"; do
+        echo "  - ${pkg}"
+    done
+    exit 1
+fi

From eaf75e925dff8ed4c2ac989a92dbba475cd27c6c Mon Sep 17 00:00:00 2001
From: Daniel Sanche <d.sanche14@gmail.com>
Date: Tue, 7 Jul 2026 13:22:17 -0700
Subject: [PATCH 84/84] print only count for successes

---
 ci/report_coverage.sh | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/ci/report_coverage.sh b/ci/report_coverage.sh
index dbd418503f89..9f89def1cb92 100755
--- a/ci/report_coverage.sh
+++ b/ci/report_coverage.sh
@@ -138,14 +138,11 @@ echo "============================================================"
 echo "Coverage Evaluation Summary"
 echo "============================================================"
 if [ ${#passed_packages[@]} -gt 0 ]; then
-    echo "Passed packages:"
-    for pkg in "${passed_packages[@]}"; do
-        echo "  - ${pkg}"
-    done
+    echo "✅ Passed: ${#passed_packages[@]} packages"
 fi
 
 if [ ${#failed_packages[@]} -gt 0 ]; then
-    echo "Failed packages:"
+    echo "❌ Failed: ${#failed_packages[@]} packages"
     for pkg in "${failed_packages[@]}"; do
         echo "  - ${pkg}"
     done