fix(windows): support artifact uploads, PyPI installation, and VS Code IDE detection on Windows

.github/workflows/integration-tests.yaml

@@ -18,7 +18,7 @@
 #   - GCP_SA_KEY: Service account JSON key (project_id and client_email extracted automatically)
 #
 # Optional GitHub Repository Variables (vars.*):
-#   - GCP_REGION: Google Cloud Region or comma-separated fallback list (defaults to us-central1)
+#   - GCP_REGION: Google Cloud Region or comma-separated fallback list (defaults to us-central1,us-east1,us-west1)
 #   - GCP_SUBNET: Managed Spark subnet name or URI (defaults to "default")
 #
 # Fork PRs can be tested by a repository maintainer commenting `/gcbrun` on the PR.
@@ -38,11 +38,22 @@ permissions:
   pull-requests: read
   statuses: write
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.event.issue.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' || github.event_name == 'issue_comment' }}
+
 jobs:
   integration-test:
-    name: Run integration tests
-    runs-on: ubuntu-latest
+    name: Run integration tests (${{ matrix.os }})
+    runs-on: ${{ matrix.os }}
     timeout-minutes: 25
+    defaults:
+      run:
+        shell: bash
+    strategy:
+      fail-fast: false
+      matrix:
+        os: ["ubuntu-latest", "macos-latest", "windows-latest"]
 
     # Run on push to main, manual dispatch, non-Dependabot same-repo pull requests,
     # or maintainer /gcbrun comments on PRs.
@@ -52,10 +63,6 @@ jobs:
       (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]') ||
       (github.event_name == 'issue_comment' && github.event.issue.pull_request != null && contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR"]'), github.event.comment.author_association) && (github.event.comment.body == '/gcbrun' || startsWith(github.event.comment.body, '/gcbrun ') || startsWith(github.event.comment.body, '/gcbrun\r') || startsWith(github.event.comment.body, '/gcbrun\n')))
 
-    concurrency:
-      group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.event.issue.number || github.ref }}
-      cancel-in-progress: ${{ github.event_name == 'pull_request' || github.event_name == 'issue_comment' }}
-
     steps:
     - name: Authorize /gcbrun and resolve PR head SHA
       if: github.event_name == 'issue_comment'
@@ -66,6 +73,7 @@ jobs:
         COMMENT_AT: ${{ github.event.comment.created_at }}
         PR_NUMBER: ${{ github.event.issue.number }}
         RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        STATUS_CONTEXT: Integration Tests / Run integration tests (${{ matrix.os }})
       run: |
         PERMISSION=$(gh api "repos/${GITHUB_REPOSITORY}/collaborators/${COMMENT_AUTHOR}/permission" --jq '.permission')
         if [[ "$PERMISSION" != "admin" && "$PERMISSION" != "maintain" && "$PERMISSION" != "write" ]]; then
@@ -85,12 +93,12 @@ jobs:
           exit 1
         fi
         COMMIT_AT=$(gh api "repos/${GITHUB_REPOSITORY}/commits/${PR_HEAD_SHA}" --jq '.commit.committer.date')
-        if [[ $(date -d "$COMMIT_AT" +%s) -gt $(date -d "$COMMENT_AT" +%s) ]]; then
+        if jq -en --arg a "$COMMIT_AT" --arg b "$COMMENT_AT" '($a | fromdateiso8601) > ($b | fromdateiso8601)' >/dev/null; then
           echo "::error::PR head commit ${PR_HEAD_SHA} (${COMMIT_AT}) is newer than /gcbrun comment (${COMMENT_AT})."
           exit 1
         fi
         PUSHED_AT=$(gh api "repos/${GITHUB_REPOSITORY}/commits/${PR_HEAD_SHA}/check-suites" --jq '[.check_suites[].created_at] | min // empty' || true)
-        if [[ -n "$PUSHED_AT" && $(date -d "$PUSHED_AT" +%s) -gt $(date -d "$COMMENT_AT" +%s) ]]; then
+        if [[ -n "$PUSHED_AT" ]] && jq -en --arg a "$PUSHED_AT" --arg b "$COMMENT_AT" '($a | fromdateiso8601) > ($b | fromdateiso8601)' >/dev/null; then
           echo "::error::PR head commit ${PR_HEAD_SHA} was pushed at ${PUSHED_AT}, after /gcbrun comment at ${COMMENT_AT}."
           exit 1
         fi
@@ -99,7 +107,7 @@ jobs:
           -f state="pending" \
           -f target_url="${RUN_URL}" \
           -f description="Integration tests running via /gcbrun" \
-          -f context="Integration Tests / Run integration tests"
+          -f context="${STATUS_CONTEXT}"
 
     - name: Checkout code
       uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -130,7 +138,10 @@ jobs:
     - name: Cache uv dependencies
       uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
       with:
-        path: ~/.cache/uv
+        path: |
+          ~/.cache/uv
+          ~/Library/Caches/uv
+          ~/AppData/Local/uv/cache
         key: ${{ runner.os }}-uv-integration-${{ hashFiles('setup.py', 'requirements-test.txt') }}
         restore-keys: |
           ${{ runner.os }}-uv-integration-
@@ -165,7 +176,7 @@ jobs:
         GOOGLE_CLOUD_PROJECT: ${{ env.PROJECT_ID }}
         MANAGED_SPARK_CONNECT_SERVICE_ACCOUNT: ${{ env.SA_EMAIL }}
         # Non-secret infrastructure variables (falling back to secrets during transition)
-        GOOGLE_CLOUD_REGION: ${{ vars.GCP_REGION || secrets.GCP_REGION || 'us-central1' }}
+        GOOGLE_CLOUD_REGION: ${{ vars.GCP_REGION || secrets.GCP_REGION || 'us-central1,us-east1,us-west1' }}
         MANAGED_SPARK_CONNECT_SUBNET: ${{ vars.GCP_SUBNET || secrets.GCP_SUBNET }}
         MANAGED_SPARK_CONNECT_AUTH_TYPE: "SERVICE_ACCOUNT"
       run: |
@@ -177,6 +188,7 @@ jobs:
         GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         JOB_STATUS: ${{ job.status }}
         RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+        STATUS_CONTEXT: Integration Tests / Run integration tests (${{ matrix.os }})
       run: |
         if [[ "$JOB_STATUS" == "success" ]]; then
           STATE="success"
@@ -189,4 +201,4 @@ jobs:
           -f state="${STATE}" \
           -f target_url="${RUN_URL}" \
           -f description="${DESC}" \
-          -f context="Integration Tests / Run integration tests"
+          -f context="${STATUS_CONTEXT}"

.github/workflows/tests.yaml

@@ -28,12 +28,16 @@ concurrency:
 
 jobs:
   test:
-    name: Run unit tests ([${{ matrix.spark-extra }}])
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
+    name: Run unit tests (${{ matrix.os }}, [${{ matrix.spark-extra }}])
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 15
+    defaults:
+      run:
+        shell: bash
     strategy:
       fail-fast: false
       matrix:
+        os: ["ubuntu-latest", "macos-latest", "windows-latest"]
         # Mutually exclusive: [client] installs pyspark-client, [spark] full
         # pyspark[connect].
         spark-extra: ["client", "spark"]
@@ -55,7 +59,10 @@ jobs:
     - name: Cache uv dependencies
       uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
       with:
-        path: ~/.cache/uv
+        path: |
+          ~/.cache/uv
+          ~/Library/Caches/uv
+          ~/AppData/Local/uv/cache
         key: ${{ runner.os }}-uv-${{ matrix.spark-extra }}-${{ hashFiles('setup.py', 'requirements-test.txt') }}
         restore-keys: |
           ${{ runner.os }}-uv-${{ matrix.spark-extra }}-
@@ -81,6 +88,6 @@ jobs:
     # the JVM that only full pyspark ships.
     - name: Run local Spark tests
       if: matrix.spark-extra == 'spark'
-      run: python -m pytest tests/local/ -v --tb=short
-      env:
-        JAVA_HOME: ${{ env.JAVA_HOME_17_X64 }}
+      run: |
+        export JAVA_HOME="${JAVA_HOME_17_X64:-${JAVA_HOME_17_ARM64:-${JAVA_HOME_21_X64:-${JAVA_HOME_21_ARM64:-$JAVA_HOME}}}}"
+        python -m pytest tests/local/ -v --tb=short

google/cloud/managed_spark_connect/ide.py

@@ -312,6 +312,12 @@ def _read_product_json(path: str) -> Optional[dict]:
     return product
 
 
+def _is_hex_commit_prefix(name: str) -> bool:
+    return len(name) == 10 and all(
+        c in "0123456789abcdef" for c in name.lower()
+    )
+
+
 def _find_product_json(exe: str) -> Optional[Tuple[str, dict, bool]]:
     """Returns (path, product, is_server) for the editor owning ``exe``."""
     # A relative path (e.g. macOS `ps` reports some commands by bare name)
@@ -340,6 +346,21 @@ def _find_product_json(exe: str) -> Optional[Tuple[str, dict, bool]]:
             if not parent or parent == d:
                 break
             d = parent
+    # Windows InnoSetup installs (VS Code 1.96+) place Code.exe at
+    # <install>\Code.exe and the versioned payload under
+    # <install>\<commit[:10]>\resources\app\product.json.
+    try:
+        entries = sorted(os.scandir(exe_dir), key=lambda e: e.name)
+    except OSError:
+        entries = []
+    for entry in entries:
+        if _is_hex_commit_prefix(entry.name):
+            path = os.path.join(entry.path, "resources", "app", "product.json")
+            product = _read_product_json(path)
+            if product is not None:
+                commit = _str_or_none(product.get("commit"))
+                if commit and commit.lower().startswith(entry.name.lower()):
+                    return path, product, False
     return None
 
 

google/cloud/managed_spark_connect/pypi_artifacts.py

@@ -2,6 +2,7 @@
 import logging
 import os
 import tempfile
+import uuid
 
 from packaging.requirements import Requirement
 
@@ -38,7 +39,7 @@ def write_packages_config(self, s8s_session_uuid: str) -> str:
         file_path = os.path.join(
             tempfile.gettempdir(),
             s8s_session_uuid,
-            "add-artifacts-1729-" + self.__str__() + ".json",
+            f"add-artifacts-1729-{uuid.uuid4()}.json",
         )
 
         os.makedirs(os.path.dirname(file_path), exist_ok=True)

google/cloud/managed_spark_connect/session.py

@@ -20,6 +20,7 @@
 import json
 import logging
 import os
+import pathlib
 import random
 import re
 import string
@@ -231,6 +232,25 @@ def _is_valid_label_value(value: str) -> bool:
     return bool(re.match(pattern, value))
 
 
+def _normalize_artifact_path(path_or_uri: str) -> str:
+    """Convert Windows drive-letter paths (e.g. C:\\...) to file:// URIs.
+
+    PySpark's ArtifactManager._parse_artifacts uses urllib.parse.urlparse(),
+    which misinterprets a Windows drive letter like 'C:\\...' as URL scheme 'c'
+    and raises '[UNSUPPORTED_OPERATION] c scheme is not supported'.
+    """
+    if (
+        len(path_or_uri) >= 3
+        and path_or_uri[0].isalpha()
+        and path_or_uri[1] == ":"
+        and path_or_uri[2] in ("\\", "/")
+    ):
+        path_part, sep, fragment = path_or_uri.partition("#")
+        uri = pathlib.PureWindowsPath(path_part).as_uri()
+        return f"{uri}#{fragment}" if sep else uri
+    return path_or_uri
+
+
 def _is_valid_session_id(session_id: str) -> bool:
     """
     Validates if a string complies with Google Cloud session ID format.
@@ -1517,14 +1537,23 @@ def addArtifacts(
             artifacts = PyPiArtifacts(set(artifact))
             logger.debug("Making addArtifact call to install packages")
             self.addArtifact(
-                artifacts.write_packages_config(self._active_s8s_session_uuid),
+                _normalize_artifact_path(
+                    artifacts.write_packages_config(
+                        self._active_s8s_session_uuid
+                    )
+                ),
                 file=True,
             )
         else:
             super().addArtifacts(
-                *artifact, pyfile=pyfile, archive=archive, file=file
+                *(_normalize_artifact_path(p) for p in artifact),
+                pyfile=pyfile,
+                archive=archive,
+                file=file,
             )
 
+    addArtifact = addArtifacts
+
     @staticmethod
     def _get_active_session_file_path():
         return os.getenv("MANAGED_SPARK_CONNECT_ACTIVE_SESSION_FILE_PATH")