ci: group behavior test setups by service

.github/actions/test_behavior_core/action.yaml

@@ -18,8 +18,8 @@
 name: Test Core
 description: 'Test Core with given setup and service'
 inputs:
-  setup:
-    description: "The setup action for test"
+  setups:
+    description: "The comma-separated setup actions for test"
   service:
     description: "The service to test"
   feature:
@@ -30,20 +30,90 @@ runs:
   steps:
     - name: Setup
       shell: bash
+      env:
+        TEST_FEATURE: ${{ inputs.feature }}
+        TEST_SERVICE: ${{ inputs.service }}
+        TEST_SETUPS: ${{ inputs.setups }}
       run: |
-        mkdir -p ./dynamic_test_core &&
-        cat <<EOF >./dynamic_test_core/action.yml
+        mkdir -p ./dynamic_test_core
+        cat <<'EOF' >./dynamic_test_core/action.yml
         runs:
           using: composite
           steps:
-          - name: Setup Test Core
-            uses: ./.github/services/${{ inputs.service }}/${{ inputs.setup }}
-          - name: Run Test Core
+        EOF
+
+        index=0
+        IFS=',' read -r -a setups <<<"$TEST_SETUPS"
+        for setup in "${setups[@]}"; do
+          setup_action="./dynamic_test_core/setup_$index"
+          mkdir -p "$setup_action"
+          cat <<EOF >"$setup_action/action.yml"
+        runs:
+          using: composite
+          steps:
+          - name: Record existing Docker resources for $setup
+            if: always()
+            shell: bash
+            run: |
+              : >"\$RUNNER_TEMP/opendal-containers-$index"
+              : >"\$RUNNER_TEMP/opendal-networks-$index"
+              : >"\$RUNNER_TEMP/opendal-volumes-$index"
+              if docker info >/dev/null 2>&1; then
+                docker ps -aq | sort >"\$RUNNER_TEMP/opendal-containers-$index"
+                docker network ls -q | sort >"\$RUNNER_TEMP/opendal-networks-$index"
+                docker volume ls -q | sort >"\$RUNNER_TEMP/opendal-volumes-$index"
+              fi
+          - name: Setup Test Core - $setup
+            id: setup_$index
+            if: always()
+            uses: ./.github/services/$TEST_SERVICE/$setup
+          - name: Run Test Core - $setup
+            if: always() && steps.setup_$index.outcome == 'success'
             shell: bash
             working-directory: core
-            run: cargo test behavior --features tests,${{ inputs.feature }}
+            run: cargo test behavior --features tests,$TEST_FEATURE
             env:
-              OPENDAL_TEST: ${{ inputs.service }}
+              OPENDAL_TEST: $TEST_SERVICE
+          - name: Cleanup Test Core - $setup
+            if: always()
+            shell: bash
+            run: |
+              if docker info >/dev/null 2>&1; then
+                current_containers=\$(docker ps -aq | sort)
+                new_containers=\$(comm -13 "\$RUNNER_TEMP/opendal-containers-$index" <(printf '%s\n' "\$current_containers"))
+                if [ -n "\$new_containers" ]; then
+                  docker rm -f \$new_containers
+                fi
+
+                current_networks=\$(docker network ls -q | sort)
+                new_networks=\$(comm -13 "\$RUNNER_TEMP/opendal-networks-$index" <(printf '%s\n' "\$current_networks"))
+                if [ -n "\$new_networks" ]; then
+                  docker network rm \$new_networks
+                fi
+
+                current_volumes=\$(docker volume ls -q | sort)
+                new_volumes=\$(comm -13 "\$RUNNER_TEMP/opendal-volumes-$index" <(printf '%s\n' "\$current_volumes"))
+                if [ -n "\$new_volumes" ]; then
+                  docker volume rm \$new_volumes
+                fi
+              fi
+
+              service_prefix=\$(printf '%s' "$TEST_SERVICE" | tr '[:lower:]-' '[:upper:]_')
+              while IFS='=' read -r name _; do
+                case "\$name" in
+                  "OPENDAL_\${service_prefix}_"* | OPENDAL_DISABLE_RANDOM_ROOT | OPENDAL_TEST_CAPABILITY_OVERRIDES)
+                    echo "\$name=__OPENDAL_TEST_UNSET__" >>"\$GITHUB_ENV"
+                    ;;
+                esac
+              done < <(env)
+        EOF
+
+          cat <<EOF >>./dynamic_test_core/action.yml
+          - name: Test Core - $setup
+            if: always()
+            uses: ./dynamic_test_core/setup_$index
         EOF
+          index=$((index + 1))
+        done
     - name: Run
       uses: ./dynamic_test_core

.github/scripts/test_behavior/plan.py

@@ -130,7 +130,10 @@ def mark_service_affected(service: str) -> None:
                 setattr(hint, f"integration_{integration}", True)
             hint.all_service = True
 
-        if p == ".github/workflows/test_behavior_core.yml":
+        if p in [
+            ".github/workflows/test_behavior_core.yml",
+            ".github/actions/test_behavior_core/action.yaml",
+        ]:
             hint.core = True
             hint.all_service = True
 
@@ -236,6 +239,21 @@ def unique_cases(cases):
     return list(ucases.values())
 
 
+def group_cases_by_service(cases: list[dict[str, str]]) -> list[dict[str, Any]]:
+    grouped_cases = {}
+    for case in cases:
+        service = case["service"]
+        if service not in grouped_cases:
+            grouped_cases[service] = {
+                "service": service,
+                "feature": case["feature"],
+                "setups": [],
+            }
+        grouped_cases[service]["setups"].append(case["setup"])
+
+    return list(grouped_cases.values())
+
+
 def generate_core_cases(
     cases: list[dict[str, str]], hint: Hint
 ) -> list[dict[str, str]]:
@@ -358,7 +376,7 @@ def plan(changed_files: list[str]) -> dict[str, Any]:
     cases = provided_cases()
     hint = calculate_hint(changed_files)
 
-    core_cases = generate_core_cases(cases, hint)
+    core_cases = group_cases_by_service(generate_core_cases(cases, hint))
 
     jobs = {
         "components": {
@@ -377,7 +395,11 @@ def plan(changed_files: list[str]) -> dict[str, Any]:
                 {
                     "os": "windows-latest",
                     "cases": [
-                        {"setup": "local_fs", "service": "fs", "feature": "services-fs"}
+                        {
+                            "setups": ["local_fs"],
+                            "service": "fs",
+                            "feature": "services-fs",
+                        }
                     ],
                 }
             )

.github/scripts/test_behavior/test_plan.py

@@ -16,9 +16,10 @@
 # under the License.
 
 import unittest
+from pathlib import Path
 from unittest.mock import patch
 
-from plan import plan
+from plan import group_cases_by_service, plan
 
 
 class BehaviorTestPlan(unittest.TestCase):
@@ -53,6 +54,77 @@ def test_core_services_hdfs_native_mapping(self):
         self.assertTrue("hdfs_native" in cases)
         self.assertFalse("fs" in cases)
 
+    def test_group_core_cases_by_service(self):
+        cases = [
+            {
+                "service": "webdav",
+                "setup": "nginx_with_empty_password",
+                "feature": "services-webdav",
+            },
+            {
+                "service": "webdav",
+                "setup": "nginx_with_password",
+                "feature": "services-webdav",
+            },
+            {
+                "service": "memory",
+                "setup": "memory",
+                "feature": "services-memory",
+            },
+        ]
+
+        self.assertEqual(
+            group_cases_by_service(cases),
+            [
+                {
+                    "service": "webdav",
+                    "feature": "services-webdav",
+                    "setups": [
+                        "nginx_with_empty_password",
+                        "nginx_with_password",
+                    ],
+                },
+                {
+                    "service": "memory",
+                    "feature": "services-memory",
+                    "setups": ["memory"],
+                },
+            ],
+        )
+
+    def test_core_groups_multiple_setups(self):
+        result = plan(["core/services/webdav/src/lib.rs"])
+        cases = result["core"][0]["cases"]
+        webdav = next(v for v in cases if v["service"] == "webdav")
+
+        self.assertIn("nginx_with_empty_password", webdav["setups"])
+        self.assertIn("nginx_with_password", webdav["setups"])
+        self.assertIn("nginx_with_redirect", webdav["setups"])
+        self.assertEqual(
+            len([v for v in cases if v["service"] == "webdav"]),
+            1,
+        )
+
+    def test_core_action(self):
+        result = plan([".github/actions/test_behavior_core/action.yaml"])
+        self.assertTrue(result["components"]["core"])
+        self.assertTrue(len(result["core"]) > 0)
+
+        for target in result["core"]:
+            for case in target["cases"]:
+                self.assertIn("setups", case)
+                self.assertNotIn("setup", case)
+
+        windows = next(v for v in result["core"] if v["os"] == "windows-latest")
+        self.assertEqual(windows["cases"][0]["setups"], ["local_fs"])
+
+    def test_service_setups_do_not_checkout(self):
+        service_dir = Path(__file__).parents[2] / "services"
+        setup_actions = service_dir.glob("*/*/action.yml")
+
+        for action in setup_actions:
+            self.assertNotIn("actions/checkout@", action.read_text(), str(action))
+
     def test_binding_java(self):
         result = plan(["bindings/java/pom.xml"])
         self.assertFalse(result["components"]["core"])

.github/services/redis/redis_tls/action.yml

@@ -21,8 +21,6 @@ description: 'Behavior test for redis tls'
 runs:
   using: "composite"
   steps:
-    - uses: actions/checkout@v6
-
     - name: Setup Redis with TLS
       shell: bash
       working-directory: fixtures/redis

.github/services/redis/redis_with_cluster_tls/action.yml

@@ -21,8 +21,6 @@ description: 'Behavior test for redis with cluster tls'
 runs:
   using: "composite"
   steps:
-    - uses: actions/checkout@v6
-
     - name: Setup Redis Cluster with TLS
       shell: bash
       working-directory: fixtures/redis

.github/workflows/test_behavior_core.yml

@@ -29,7 +29,7 @@ on:
 
 jobs:
   test:
-    name: ${{ matrix.cases.service }} / ${{ matrix.cases.setup }}
+    name: ${{ matrix.cases.service }}
     runs-on: ${{ inputs.os }}
     strategy:
       fail-fast: false
@@ -55,6 +55,6 @@ jobs:
       - name: Test Core
         uses: ./.github/actions/test_behavior_core
         with:
-          setup: ${{ matrix.cases.setup }}
+          setups: ${{ join(matrix.cases.setups, ',') }}
           service: ${{ matrix.cases.service }}
           feature: ${{ matrix.cases.feature }}

core/testkit/src/utils.rs

@@ -29,6 +29,7 @@ use sha2::Digest;
 use sha2::Sha256;
 
 const OPENDAL_TEST_CAPABILITY_OVERRIDES: &str = "OPENDAL_TEST_CAPABILITY_OVERRIDES";
+const OPENDAL_TEST_UNSET_VALUE: &str = "__OPENDAL_TEST_UNSET__";
 
 pub(crate) fn sha256_digest(data: impl AsRef<[u8]>) -> String {
     use std::fmt::Write;
@@ -49,6 +50,22 @@ pub static TEST_RUNTIME: LazyLock<tokio::runtime::Runtime> = LazyLock::new(|| {
         .unwrap()
 });
 
+fn collect_config(
+    prefix: &str,
+    vars: impl IntoIterator<Item = (String, String)>,
+) -> HashMap<String, String> {
+    vars.into_iter()
+        .filter_map(|(k, v)| {
+            if v == OPENDAL_TEST_UNSET_VALUE {
+                return None;
+            }
+            k.to_lowercase()
+                .strip_prefix(prefix)
+                .map(|k| (k.to_string(), v))
+        })
+        .collect()
+}
+
 /// Init a service with given scheme.
 ///
 /// - Load scheme from `OPENDAL_TEST`
@@ -68,13 +85,7 @@ pub fn init_test_service() -> Result<Option<Operator>> {
         format!("opendal_{scheme_key}_")
     };
 
-    let mut cfg = env::vars()
-        .filter_map(|(k, v)| {
-            k.to_lowercase()
-                .strip_prefix(&prefix)
-                .map(|k| (k.to_string(), v))
-        })
-        .collect::<HashMap<String, String>>();
+    let mut cfg = collect_config(&prefix, env::vars());
 
     // Use random root unless OPENDAL_DISABLE_RANDOM_ROOT is set to true.
     let disable_random_root = env::var("OPENDAL_DISABLE_RANDOM_ROOT").unwrap_or_default() == "true";
@@ -91,7 +102,9 @@ pub fn init_test_service() -> Result<Option<Operator>> {
     let scheme = scheme.replace('_', "-");
     let mut op = Operator::via_iter(scheme, cfg).expect("must succeed");
 
-    if let Ok(overrides) = env::var(OPENDAL_TEST_CAPABILITY_OVERRIDES) {
+    if let Ok(overrides) = env::var(OPENDAL_TEST_CAPABILITY_OVERRIDES)
+        && overrides != OPENDAL_TEST_UNSET_VALUE
+    {
         op = op.layer(CapabilityOverrideLayer::from_overrides(&overrides)?);
     }
 
@@ -102,3 +115,35 @@ pub fn init_test_service() -> Result<Option<Operator>> {
 
     Ok(Some(op))
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_collect_config_skips_unset_values() {
+        let cfg = collect_config(
+            "opendal_s3_",
+            [
+                (
+                    "OPENDAL_S3_ENDPOINT".to_string(),
+                    "http://localhost".to_string(),
+                ),
+                (
+                    "OPENDAL_S3_ALLOW_ANONYMOUS".to_string(),
+                    OPENDAL_TEST_UNSET_VALUE.to_string(),
+                ),
+                ("OPENDAL_S3_PASSWORD".to_string(), String::new()),
+                ("OPENDAL_GCS_BUCKET".to_string(), "test".to_string()),
+            ],
+        );
+
+        assert_eq!(
+            cfg.get("endpoint").map(String::as_str),
+            Some("http://localhost")
+        );
+        assert_eq!(cfg.get("password").map(String::as_str), Some(""));
+        assert!(!cfg.contains_key("allow_anonymous"));
+        assert!(!cfg.contains_key("bucket"));
+    }
+}