diff --git a/.github/workflows/pr_build_linux.yml b/.github/workflows/pr_build_linux.yml index a8d925b1c2..297412ef9f 100644 --- a/.github/workflows/pr_build_linux.yml +++ b/.github/workflows/pr_build_linux.yml @@ -81,7 +81,7 @@ jobs: container: image: amd64/rust env: - JAVA_TOOL_OPTIONS: ${{ matrix.profile.java_version == '17' && '--add-exports=java.base/sun.nio.ch=ALL-UNNAMED --add-exports=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED' || '' }} + JAVA_TOOL_OPTIONS: ${{ (matrix.profile.java_version == '17' || matrix.profile.java_version == '21') && '--add-exports=java.base/sun.nio.ch=ALL-UNNAMED --add-exports=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED' || '' }} strategy: matrix: profile: @@ -94,6 +94,9 @@ jobs: - name: "Spark 4.0, JDK 17" java_version: "17" maven_opts: "-Pspark-4.0" + - name: "Spark 4.0, JDK 21" + java_version: "21" + maven_opts: "-Pspark-4.0" # Spark 4.1 is intentionally absent: the lint job invokes -Psemanticdb, # but semanticdb-scalac_2.13.17 is not yet published, so we cannot # currently run scalafix against the spark-4.1 profile. @@ -293,6 +296,11 @@ jobs: maven_opts: "-Pspark-4.0" scan_impl: "auto" + - name: "Spark 4.0, JDK 21" + java_version: "21" + maven_opts: "-Pspark-4.0" + scan_impl: "auto" + - name: "Spark 4.1, JDK 17" java_version: "17" maven_opts: "-Pspark-4.1" @@ -388,7 +396,7 @@ jobs: container: image: amd64/rust env: - JAVA_TOOL_OPTIONS: ${{ matrix.profile.java_version == '17' && '--add-exports=java.base/sun.nio.ch=ALL-UNNAMED --add-exports=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED' || '' }} + JAVA_TOOL_OPTIONS: ${{ (matrix.profile.java_version == '17' || matrix.profile.java_version == '21') && '--add-exports=java.base/sun.nio.ch=ALL-UNNAMED --add-exports=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED' || '' }} steps: - uses: runs-on/action@742bf56072eb4845a0f94b3394673e4903c90ff0 # v2.1.0 diff --git a/.github/workflows/spark_sql_test.yml b/.github/workflows/spark_sql_test.yml index 7f3a1e5922..569a4a3ab9 100644 --- a/.github/workflows/spark_sql_test.yml +++ b/.github/workflows/spark_sql_test.yml @@ -141,6 +141,7 @@ jobs: - {spark-short: '3.4', spark-full: '3.4.3', java: 11, scan-impl: 'auto'} - {spark-short: '3.5', spark-full: '3.5.8', java: 11, scan-impl: 'auto'} - {spark-short: '4.0', spark-full: '4.0.2', java: 17, scan-impl: 'auto'} + - {spark-short: '4.0', spark-full: '4.0.2', java: 21, scan-impl: 'auto'} fail-fast: false name: spark-sql-${{ matrix.config.scan-impl }}-${{ matrix.module.name }}/spark-${{ matrix.config.spark-full }} runs-on: ${{ matrix.os }} @@ -175,6 +176,14 @@ jobs: fi env: LC_ALL: "C.UTF-8" + # Mirror Spark's own JDK 21 / 25 CI workaround. apache/spark's + # build_java21.yml and build_java25.yml set this same env var to + # process-isolate the V1/V2 Parquet and Orc source suites because + # they exhibit cross-suite resource interactions (file-stream and + # thread leaks) under the newer JDKs. project/SparkBuild.scala + # reads DEDICATED_JVM_SBT_TESTS and forks a separate JVM per + # listed suite. Empty value is a safe no-op. + DEDICATED_JVM_SBT_TESTS: ${{ matrix.config.spark-short == '4.0' && 'org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormatV1Suite,org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormatV2Suite,org.apache.spark.sql.execution.datasources.orc.OrcSourceV1Suite,org.apache.spark.sql.execution.datasources.orc.OrcSourceV2Suite' || '' }} - name: Upload fallback log if: ${{ github.event.inputs.collect-fallback-logs == 'true' }} uses: actions/upload-artifact@v7 diff --git a/docs/source/user-guide/latest/installation.md b/docs/source/user-guide/latest/installation.md index 05e95f1069..dce2a4e4fc 100644 --- a/docs/source/user-guide/latest/installation.md +++ b/docs/source/user-guide/latest/installation.md @@ -50,7 +50,7 @@ use only and should not be used in production yet. | Spark Version | Java Version | Scala Version | Comet Tests in CI | Spark SQL Tests in CI | | ------------- | ------------ | ------------- | ----------------- | --------------------- | -| 4.0.2 | 17 | 2.13 | Yes | Yes | +| 4.0.2 | 17/21 | 2.13 | Yes | Yes | Note that Comet may not fully work with proprietary forks of Apache Spark such as the Spark versions offered by Cloud Service Providers.