diff --git a/eng/pipelines/scripts/generate-project-list.ps1 b/eng/pipelines/scripts/generate-project-list.ps1 index a4432a8b3977..2208fd131056 100644 --- a/eng/pipelines/scripts/generate-project-list.ps1 +++ b/eng/pipelines/scripts/generate-project-list.ps1 @@ -4,6 +4,29 @@ $additionalModulesList = @() . "${PSScriptRoot}/../../common/scripts/common.ps1" +# If ProjectListOverride is set (e.g., from matrix variables), use it directly +# to avoid building unnecessary modules in jobs that only test a subset. +# Do not honor ProjectListOverride for FromSource runs — FromSource builds use +# ClientFromSourcePom.xml which builds all libraries and the project list must +# be computed from the artifacts, not overridden. +if ($env:PROJECTLISTOVERRIDE -and $env:PROJECTLISTOVERRIDE -notlike '*ProjectListOverride*') { + if ($env:TESTFROMSOURCE -eq 'true') { + Write-Host "Ignoring ProjectListOverride for FromSource run (TestFromSource=true)" + } else { + $projects = $env:PROJECTLISTOVERRIDE + Write-Host "Using ProjectListOverride = $projects" + Write-Host "##vso[task.setvariable variable=ProjectList;]$projects" + Write-Host "##vso[task.setvariable variable=ArtifactsList;]$projects" + Write-Host "##vso[task.setvariable variable=AdditionalModulesList;]" + + $sha256 = new-object -TypeName System.Security.Cryptography.SHA256Managed + $utf8 = new-object -TypeName System.Text.UTF8Encoding + $projectListSha256 = [Convert]::ToBase64String($sha256.ComputeHash($utf8.GetBytes($projects))) + Write-Host "##vso[task.setvariable variable=ProjectListSha256;]$projectListSha256" + return + } +} + if ($env:ARTIFACTSJSON -and $env:ARTIFACTSJSON -notlike '*ArtifactsJson*') { $artifacts = $env:ARTIFACTSJSON | ConvertFrom-Json foreach ($artifact in $artifacts) { diff --git a/eng/pipelines/templates/jobs/ci.tests.yml b/eng/pipelines/templates/jobs/ci.tests.yml index 82d12ff45fdf..cdfac6273e60 100644 --- a/eng/pipelines/templates/jobs/ci.tests.yml +++ b/eng/pipelines/templates/jobs/ci.tests.yml @@ -40,6 +40,9 @@ parameters: - name: BuildParallelization type: string default: '2C' + - name: BuildOptions + type: string + default: '' - name: AdditionalLintingOptions type: string default: '' @@ -145,6 +148,7 @@ jobs: TestOptions: ${{ parameters.TestOptions }} TestParallelization: ${{ parameters.TestParallelization }} BuildParallelization: ${{ parameters.BuildParallelization }} + BuildOptions: ${{ parameters.BuildOptions }} TestEnvVars: ${{ parameters.EnvVars }} VersionOverride: ${{ parameters.VersionOverride }} diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index 703c81308c01..95233db735c1 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -40,6 +40,9 @@ parameters: - name: BuildParallelization type: string default: '2C' + - name: BuildOptions + type: string + default: '' - name: TestGoals type: string default: $(TestGoals) @@ -503,6 +506,7 @@ jobs: TestOptions: ${{ parameters.TestOptions }} TestParallelization: ${{ parameters.TestParallelization }} BuildParallelization: ${{ parameters.BuildParallelization }} + BuildOptions: ${{ parameters.BuildOptions }} AdditionalLintingOptions: ${{ parameters.AdditionalLintingOptions }} TimeoutInMinutes: ${{ parameters.TimeoutInMinutes }} EnvVars: ${{ parameters.EnvVars }} diff --git a/eng/pipelines/templates/stages/cosmos-emulator-matrix-pr.json b/eng/pipelines/templates/stages/cosmos-emulator-matrix-pr.json new file mode 100644 index 000000000000..94625e96ffdb --- /dev/null +++ b/eng/pipelines/templates/stages/cosmos-emulator-matrix-pr.json @@ -0,0 +1,115 @@ +{ + "displayNames": { + "clean verify": "" + }, + "matrix": { + "Agent": { + "windows-2022": { + "OSVmImage": "env:WINDOWSVMIMAGE", + "Pool": "env:WINDOWSPOOL" + } + }, + "TestGoals": "clean verify", + "EmulatorConfig": { + "Emulator Only Integration Tests Tcp - Java 8": { + "ProfileFlag": "-Pemulator", + "PROTOCOLS": "[\"Tcp\"]", + "DESIRED_CONSISTENCIES": "[\"Session\"]", + "JavaTestVersion": "1.8", + "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true -Dshade.skip=true -Dmaven.antrun.skip=true", + "BuildOptions": "-Dshade.skip=true -Dmaven.antrun.skip=true", + "ProjectListOverride": "com.azure:azure-cosmos,com.azure:azure-cosmos-test,com.azure:azure-cosmos-tests" + }, + "Emulator Only Integration Tests Tcp - Java 17": { + "ProfileFlag": "-Pemulator", + "PROTOCOLS": "[\"Tcp\"]", + "DESIRED_CONSISTENCIES": "[\"Strong\"]", + "JavaTestVersion": "1.17", + "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true -Dshade.skip=true -Dmaven.antrun.skip=true", + "BuildOptions": "-Dshade.skip=true -Dmaven.antrun.skip=true", + "ProjectListOverride": "com.azure:azure-cosmos,com.azure:azure-cosmos-test,com.azure:azure-cosmos-tests" + }, + "Long Emulator Only Integration Tests Tcp - Java 8": { + "ProfileFlag": "-Plong-emulator", + "PROTOCOLS": "[\"Tcp\"]", + "DESIRED_CONSISTENCIES": "[\"Session\"]", + "JavaTestVersion": "1.8", + "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true -Dshade.skip=true -Dmaven.antrun.skip=true", + "BuildOptions": "-Dshade.skip=true -Dmaven.antrun.skip=true", + "ProjectListOverride": "com.azure:azure-cosmos,com.azure:azure-cosmos-test,com.azure:azure-cosmos-tests" + }, + "Long Emulator Only Integration Tests Tcp - Java 17": { + "ProfileFlag": "-Plong-emulator", + "PROTOCOLS": "[\"Tcp\"]", + "DESIRED_CONSISTENCIES": "[\"Strong\"]", + "JavaTestVersion": "1.17", + "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true -Dshade.skip=true -Dmaven.antrun.skip=true", + "BuildOptions": "-Dshade.skip=true -Dmaven.antrun.skip=true", + "ProjectListOverride": "com.azure:azure-cosmos,com.azure:azure-cosmos-test,com.azure:azure-cosmos-tests" + }, + "Encryption Emulator Only Integration Tests": { + "ProfileFlag": "-Pencryption-integration", + "PROTOCOLS": "[\"Tcp\"]", + "DESIRED_CONSISTENCIES": "[\"Session\"]", + "JavaTestVersion": "1.8", + "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true -Dshade.skip=true -Dmaven.antrun.skip=true", + "BuildOptions": "-Dshade.skip=true -Dmaven.antrun.skip=true", + "ProjectListOverride": "com.azure:azure-cosmos,com.azure:azure-cosmos-encryption,com.azure:azure-cosmos-test,com.azure:azure-cosmos-tests" + }, + "Spark 3.3 Integration Tests targeting Cosmos Emulator - Java 8'": { + "ProfileFlag": "-Dspark-e2e_3-3", + "PROTOCOLS": "[\"Tcp\"]", + "DESIRED_CONSISTENCIES": "[\"Session\"]", + "JavaTestVersion": "1.8", + "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -Dhadoop.home.dir=D:/Hadoop -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true", + "BuildOptions": "", + "ProjectListOverride": "com.azure:azure-cosmos,com.azure:azure-cosmos-test,com.azure:azure-cosmos-tests,com.azure.cosmos.spark:azure-cosmos-spark_3,com.azure.cosmos.spark:azure-cosmos-spark_3-3_2-12" + }, + "Spark 3.4 Integration Tests targeting Cosmos Emulator - Java 11'": { + "ProfileFlag": "-Dspark-e2e_3-4", + "PROTOCOLS": "[\"Tcp\"]", + "DESIRED_CONSISTENCIES": "[\"Session\"]", + "JavaTestVersion": "1.11", + "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -Dhadoop.home.dir=D:/Hadoop -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true", + "BuildOptions": "", + "ProjectListOverride": "com.azure:azure-cosmos,com.azure:azure-cosmos-test,com.azure:azure-cosmos-tests,com.azure.cosmos.spark:azure-cosmos-spark_3,com.azure.cosmos.spark:azure-cosmos-spark_3-4_2-12" + }, + "Spark 3.5, Scala 2.12 Integration Tests targeting Cosmos Emulator - Java 17'": { + "ProfileFlag": "-Dspark-e2e_3-5_2-12", + "PROTOCOLS": "[\"Tcp\"]", + "DESIRED_CONSISTENCIES": "[\"Session\"]", + "JavaTestVersion": "1.17", + "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -Dhadoop.home.dir=D:/Hadoop -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true", + "BuildOptions": "", + "ProjectListOverride": "com.azure:azure-cosmos,com.azure:azure-cosmos-test,com.azure:azure-cosmos-tests,com.azure.cosmos.spark:azure-cosmos-spark_3,com.azure.cosmos.spark:azure-cosmos-spark_3-5,com.azure.cosmos.spark:azure-cosmos-spark_3-5_2-12" + }, + "Spark 3.5, Scala 2.13 Integration Tests targeting Cosmos Emulator - Java 17'": { + "ProfileFlag": "-Dspark-e2e_3-5_2-13", + "PROTOCOLS": "[\"Tcp\"]", + "DESIRED_CONSISTENCIES": "[\"Session\"]", + "JavaTestVersion": "1.17", + "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -Dhadoop.home.dir=D:/Hadoop -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true", + "BuildOptions": "", + "ProjectListOverride": "com.azure:azure-cosmos,com.azure:azure-cosmos-test,com.azure:azure-cosmos-tests,com.azure.cosmos.spark:azure-cosmos-spark_3,com.azure.cosmos.spark:azure-cosmos-spark_3-5,com.azure.cosmos.spark:azure-cosmos-spark_3-5_2-13" + }, + "Spark 4.0, Scala 2.13 Integration Tests targeting Cosmos Emulator - Java 21'": { + "ProfileFlag": "-Dspark-e2e_4-0_2-13", + "PROTOCOLS": "[\"Tcp\"]", + "DESIRED_CONSISTENCIES": "[\"Session\"]", + "JavaTestVersion": "1.21", + "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -Dhadoop.home.dir=D:/Hadoop -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true", + "BuildOptions": "", + "ProjectListOverride": "com.azure:azure-cosmos,com.azure:azure-cosmos-test,com.azure:azure-cosmos-tests,com.azure.cosmos.spark:azure-cosmos-spark_3,com.azure.cosmos.spark:azure-cosmos-spark_4-0_2-13" + }, + "Kafka Integration Tests targeting Cosmos Emulator - Java 17": { + "ProfileFlag": "-Pkafka-emulator", + "PROTOCOLS": "[\"Tcp\"]", + "DESIRED_CONSISTENCIES": "[\"Session\"]", + "JavaTestVersion": "1.17", + "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true", + "BuildOptions": "", + "ProjectListOverride": "com.azure:azure-cosmos,com.azure:azure-cosmos-test,com.azure:azure-cosmos-tests,com.azure.cosmos.kafka:azure-cosmos-kafka-connect" + } + } + } +} diff --git a/eng/pipelines/templates/stages/cosmos-emulator-matrix.json b/eng/pipelines/templates/stages/cosmos-emulator-matrix.json index 2823975d5ea8..3347a308cc8f 100644 --- a/eng/pipelines/templates/stages/cosmos-emulator-matrix.json +++ b/eng/pipelines/templates/stages/cosmos-emulator-matrix.json @@ -4,121 +4,156 @@ }, "matrix": { "Agent": { - "windows-2022": { "OSVmImage": "env:WINDOWSVMIMAGE", "Pool": "env:WINDOWSPOOL" } + "windows-2022": { + "OSVmImage": "env:WINDOWSVMIMAGE", + "Pool": "env:WINDOWSPOOL" + } }, "TestGoals": "clean verify", "EmulatorConfig": { "Emulator Only Integration Tests Tcp - Java 8": { - "ProfileFlag": "-Pemulator", - "PROTOCOLS": "[\"Tcp\"]", - "DESIRED_CONSISTENCIES": "[\"Session\"]", - "JavaTestVersion": "1.8", - "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true" + "ProfileFlag": "-Pemulator", + "PROTOCOLS": "[\"Tcp\"]", + "DESIRED_CONSISTENCIES": "[\"Session\"]", + "JavaTestVersion": "1.8", + "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true -Dshade.skip=true -Dmaven.antrun.skip=true", + "BuildOptions": "-Dshade.skip=true -Dmaven.antrun.skip=true", + "ProjectListOverride": "com.azure:azure-cosmos,com.azure:azure-cosmos-test,com.azure:azure-cosmos-tests" }, "Emulator Only Integration Tests Tcp - Java 17": { "ProfileFlag": "-Pemulator", "PROTOCOLS": "[\"Tcp\"]", "DESIRED_CONSISTENCIES": "[\"Strong\"]", "JavaTestVersion": "1.17", - "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true" + "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true -Dshade.skip=true -Dmaven.antrun.skip=true", + "BuildOptions": "-Dshade.skip=true -Dmaven.antrun.skip=true", + "ProjectListOverride": "com.azure:azure-cosmos,com.azure:azure-cosmos-test,com.azure:azure-cosmos-tests" }, "Long Emulator Only Integration Tests Tcp - Java 8": { - "ProfileFlag": "-Plong-emulator", - "PROTOCOLS": "[\"Tcp\"]", - "DESIRED_CONSISTENCIES": "[\"Session\"]", - "JavaTestVersion": "1.8", - "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true" + "ProfileFlag": "-Plong-emulator", + "PROTOCOLS": "[\"Tcp\"]", + "DESIRED_CONSISTENCIES": "[\"Session\"]", + "JavaTestVersion": "1.8", + "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true -Dshade.skip=true -Dmaven.antrun.skip=true", + "BuildOptions": "-Dshade.skip=true -Dmaven.antrun.skip=true", + "ProjectListOverride": "com.azure:azure-cosmos,com.azure:azure-cosmos-test,com.azure:azure-cosmos-tests" }, "Long Emulator Only Integration Tests Tcp - Java 17": { "ProfileFlag": "-Plong-emulator", "PROTOCOLS": "[\"Tcp\"]", "DESIRED_CONSISTENCIES": "[\"Strong\"]", "JavaTestVersion": "1.17", - "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true" + "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true -Dshade.skip=true -Dmaven.antrun.skip=true", + "BuildOptions": "-Dshade.skip=true -Dmaven.antrun.skip=true", + "ProjectListOverride": "com.azure:azure-cosmos,com.azure:azure-cosmos-test,com.azure:azure-cosmos-tests" }, "Encryption Emulator Only Integration Tests": { - "ProfileFlag": "-Pencryption-integration", - "PROTOCOLS": "[\"Tcp\"]", - "DESIRED_CONSISTENCIES": "[\"Session\"]", - "JavaTestVersion": "1.8", - "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true" + "ProfileFlag": "-Pencryption-integration", + "PROTOCOLS": "[\"Tcp\"]", + "DESIRED_CONSISTENCIES": "[\"Session\"]", + "JavaTestVersion": "1.8", + "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true -Dshade.skip=true -Dmaven.antrun.skip=true", + "BuildOptions": "-Dshade.skip=true -Dmaven.antrun.skip=true", + "ProjectListOverride": "com.azure:azure-cosmos,com.azure:azure-cosmos-encryption,com.azure:azure-cosmos-test,com.azure:azure-cosmos-tests" }, "Spark 3.3 Integration Tests targeting Cosmos Emulator - Java 8'": { "ProfileFlag": "-Dspark-e2e_3-3", "PROTOCOLS": "[\"Tcp\"]", "DESIRED_CONSISTENCIES": "[\"Session\"]", "JavaTestVersion": "1.8", - "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -Dhadoop.home.dir=D:/Hadoop -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true" + "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -Dhadoop.home.dir=D:/Hadoop -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true", + "BuildOptions": "", + "ProjectListOverride": "com.azure:azure-cosmos,com.azure:azure-cosmos-test,com.azure:azure-cosmos-tests,com.azure.cosmos.spark:azure-cosmos-spark_3,com.azure.cosmos.spark:azure-cosmos-spark_3-3_2-12" }, "Spark 3.3 Integration Tests targeting Cosmos Emulator - Java 11'": { "ProfileFlag": "-Dspark-e2e_3-3", "PROTOCOLS": "[\"Tcp\"]", "DESIRED_CONSISTENCIES": "[\"Session\"]", "JavaTestVersion": "1.11", - "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -Dhadoop.home.dir=D:/Hadoop -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true" + "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -Dhadoop.home.dir=D:/Hadoop -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true", + "BuildOptions": "", + "ProjectListOverride": "com.azure:azure-cosmos,com.azure:azure-cosmos-test,com.azure:azure-cosmos-tests,com.azure.cosmos.spark:azure-cosmos-spark_3,com.azure.cosmos.spark:azure-cosmos-spark_3-3_2-12" }, "Spark 3.4 Integration Tests targeting Cosmos Emulator - Java 8'": { "ProfileFlag": "-Dspark-e2e_3-4", "PROTOCOLS": "[\"Tcp\"]", "DESIRED_CONSISTENCIES": "[\"Session\"]", "JavaTestVersion": "1.8", - "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -Dhadoop.home.dir=D:/Hadoop -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true" + "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -Dhadoop.home.dir=D:/Hadoop -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true", + "BuildOptions": "", + "ProjectListOverride": "com.azure:azure-cosmos,com.azure:azure-cosmos-test,com.azure:azure-cosmos-tests,com.azure.cosmos.spark:azure-cosmos-spark_3,com.azure.cosmos.spark:azure-cosmos-spark_3-4_2-12" }, "Spark 3.4 Integration Tests targeting Cosmos Emulator - Java 11'": { "ProfileFlag": "-Dspark-e2e_3-4", "PROTOCOLS": "[\"Tcp\"]", "DESIRED_CONSISTENCIES": "[\"Session\"]", "JavaTestVersion": "1.11", - "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -Dhadoop.home.dir=D:/Hadoop -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true" + "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -Dhadoop.home.dir=D:/Hadoop -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true", + "BuildOptions": "", + "ProjectListOverride": "com.azure:azure-cosmos,com.azure:azure-cosmos-test,com.azure:azure-cosmos-tests,com.azure.cosmos.spark:azure-cosmos-spark_3,com.azure.cosmos.spark:azure-cosmos-spark_3-4_2-12" }, "Spark 3.5, Scala 2.12 Integration Tests targeting Cosmos Emulator - Java 8'": { "ProfileFlag": "-Dspark-e2e_3-5_2-12", "PROTOCOLS": "[\"Tcp\"]", "DESIRED_CONSISTENCIES": "[\"Session\"]", "JavaTestVersion": "1.8", - "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -Dhadoop.home.dir=D:/Hadoop -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true" + "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -Dhadoop.home.dir=D:/Hadoop -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true", + "BuildOptions": "", + "ProjectListOverride": "com.azure:azure-cosmos,com.azure:azure-cosmos-test,com.azure:azure-cosmos-tests,com.azure.cosmos.spark:azure-cosmos-spark_3,com.azure.cosmos.spark:azure-cosmos-spark_3-5,com.azure.cosmos.spark:azure-cosmos-spark_3-5_2-12" }, "Spark 3.5, Scala 2.12 Integration Tests targeting Cosmos Emulator - Java 17'": { "ProfileFlag": "-Dspark-e2e_3-5_2-12", "PROTOCOLS": "[\"Tcp\"]", "DESIRED_CONSISTENCIES": "[\"Session\"]", "JavaTestVersion": "1.17", - "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -Dhadoop.home.dir=D:/Hadoop -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true" + "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -Dhadoop.home.dir=D:/Hadoop -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true", + "BuildOptions": "", + "ProjectListOverride": "com.azure:azure-cosmos,com.azure:azure-cosmos-test,com.azure:azure-cosmos-tests,com.azure.cosmos.spark:azure-cosmos-spark_3,com.azure.cosmos.spark:azure-cosmos-spark_3-5,com.azure.cosmos.spark:azure-cosmos-spark_3-5_2-12" }, "Spark 3.5, Scala 2.13 Integration Tests targeting Cosmos Emulator - Java 17'": { "ProfileFlag": "-Dspark-e2e_3-5_2-13", "PROTOCOLS": "[\"Tcp\"]", "DESIRED_CONSISTENCIES": "[\"Session\"]", "JavaTestVersion": "1.17", - "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -Dhadoop.home.dir=D:/Hadoop -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true" + "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -Dhadoop.home.dir=D:/Hadoop -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true", + "BuildOptions": "", + "ProjectListOverride": "com.azure:azure-cosmos,com.azure:azure-cosmos-test,com.azure:azure-cosmos-tests,com.azure.cosmos.spark:azure-cosmos-spark_3,com.azure.cosmos.spark:azure-cosmos-spark_3-5,com.azure.cosmos.spark:azure-cosmos-spark_3-5_2-13" }, "Spark 4.0, Scala 2.13 Integration Tests targeting Cosmos Emulator - Java 17'": { "ProfileFlag": "-Dspark-e2e_4-0_2-13", "PROTOCOLS": "[\"Tcp\"]", "DESIRED_CONSISTENCIES": "[\"Session\"]", "JavaTestVersion": "1.17", - "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -Dhadoop.home.dir=D:/Hadoop -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true" + "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -Dhadoop.home.dir=D:/Hadoop -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true", + "BuildOptions": "", + "ProjectListOverride": "com.azure:azure-cosmos,com.azure:azure-cosmos-test,com.azure:azure-cosmos-tests,com.azure.cosmos.spark:azure-cosmos-spark_3,com.azure.cosmos.spark:azure-cosmos-spark_4-0_2-13" }, "Spark 4.0, Scala 2.13 Integration Tests targeting Cosmos Emulator - Java 21'": { "ProfileFlag": "-Dspark-e2e_4-0_2-13", "PROTOCOLS": "[\"Tcp\"]", "DESIRED_CONSISTENCIES": "[\"Session\"]", "JavaTestVersion": "1.21", - "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -Dhadoop.home.dir=D:/Hadoop -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true" + "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -Dhadoop.home.dir=D:/Hadoop -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true", + "BuildOptions": "", + "ProjectListOverride": "com.azure:azure-cosmos,com.azure:azure-cosmos-test,com.azure:azure-cosmos-tests,com.azure.cosmos.spark:azure-cosmos-spark_3,com.azure.cosmos.spark:azure-cosmos-spark_4-0_2-13" }, "Kafka Integration Tests targeting Cosmos Emulator - Java 11": { "ProfileFlag": "-Pkafka-emulator", "PROTOCOLS": "[\"Tcp\"]", "DESIRED_CONSISTENCIES": "[\"Session\"]", "JavaTestVersion": "1.11", - "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true" + "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true", + "BuildOptions": "", + "ProjectListOverride": "com.azure:azure-cosmos,com.azure:azure-cosmos-test,com.azure:azure-cosmos-tests,com.azure.cosmos.kafka:azure-cosmos-kafka-connect" }, "Kafka Integration Tests targeting Cosmos Emulator - Java 17": { "ProfileFlag": "-Pkafka-emulator", "PROTOCOLS": "[\"Tcp\"]", "DESIRED_CONSISTENCIES": "[\"Session\"]", "JavaTestVersion": "1.17", - "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true" + "AdditionalArgs": "-DACCOUNT_HOST=https://localhost:8081/ -DCOSMOS.AZURE_COSMOS_DISABLE_NON_STREAMING_ORDER_BY=true", + "BuildOptions": "", + "ProjectListOverride": "com.azure:azure-cosmos,com.azure:azure-cosmos-test,com.azure:azure-cosmos-tests,com.azure.cosmos.kafka:azure-cosmos-kafka-connect" } } } diff --git a/eng/pipelines/templates/stages/cosmos-sdk-client.yml b/eng/pipelines/templates/stages/cosmos-sdk-client.yml index 9d2977f14107..ea9b86583e14 100644 --- a/eng/pipelines/templates/stages/cosmos-sdk-client.yml +++ b/eng/pipelines/templates/stages/cosmos-sdk-client.yml @@ -61,7 +61,8 @@ extends: MatrixReplace: - AZURE_TEST.*=.*/ - .*Version=1.2(1|5)/1.17 - BuildParallelization: 1 + BuildParallelization: 2 + BuildOptions: '-Dshade.skip=true -Dmaven.antrun.skip=true' TestOptions: '-Punit' - stage: TestEmulator @@ -80,18 +81,21 @@ extends: JobTemplatePath: /eng/pipelines/templates/jobs/live.tests.yml MatrixConfigs: - Name: Cosmos_emulator_integration - Path: eng/pipelines/templates/stages/cosmos-emulator-matrix.json + # Use reduced matrix for PRs (11 jobs) vs full matrix for CI/main (16 jobs) + ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + Path: eng/pipelines/templates/stages/cosmos-emulator-matrix-pr.json + ${{ else }}: + Path: eng/pipelines/templates/stages/cosmos-emulator-matrix.json Selection: all GenerateVMJobs: true MatrixFilters: - ${{ parameters.LanguageFilter }} - ${{ parameters.MatrixFilters }} AdditionalParameters: - BuildParallelization: 1 + BuildParallelization: 2 + BuildOptions: '$(BuildOptions)' DisableAzureResourceCreation: true ServiceDirectory: cosmos - # Increased timeout to 120 because of cosmos emulator taking 25-30 mins to download emulator - # Issue filed to improve download speed: https://github.com/Azure/azure-sdk-for-java/issues/12970 TimeoutInMinutes: 120 TestOptions: '$(ProfileFlag) $(AdditionalArgs)' PreSteps: @@ -179,7 +183,7 @@ extends: - ${{ parameters.LanguageFilter }} - ${{ parameters.MatrixFilters }} AdditionalParameters: - BuildParallelization: 1 + BuildParallelization: 2 DisableAzureResourceCreation: true ServiceDirectory: cosmos TimeoutInMinutes: 120 diff --git a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java index af894f913052..2e0beac7e0f0 100644 --- a/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java +++ b/sdk/cosmos/azure-cosmos-tests/src/test/java/com/azure/cosmos/rx/TestSuiteBase.java @@ -1,5 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. + package com.azure.cosmos.rx; import com.azure.core.credential.AzureKeyCredential;