diff --git a/launchable/commands/subset.py b/launchable/commands/subset.py index dc4afb115..b25e2b2fb 100644 --- a/launchable/commands/subset.py +++ b/launchable/commands/subset.py @@ -248,9 +248,18 @@ class FallbackMode(str, Enum): default="run-all", help="Behavior when the subset API is unavailable or the model is untrained. " "'run-all' (default) runs all tests as usual; 'stop' exits with a non-zero status so CI halts; " - "'random-sample' picks a random subset locally based on the count derived from --target " + "'random-sample' picks a random subset locally based on the count derived from --fallback-sampling-target " "(no duration estimates are available in this path).", ) +@click.option( + "--fallback-sampling-target", + "fallback_sampling_target", + hidden=True, + type=PERCENTAGE, + default=None, + help="Sampling ratio (0%–100%) used when --fallback-mode=random-sample. " + "Required when --fallback-mode=random-sample is specified.", +) @click.pass_context def subset( context: click.core.Context, @@ -283,6 +292,7 @@ def subset( similarity: Optional[float] = None, subset_id_file: Optional[str] = None, fallback_mode: str = "run-all", + fallback_sampling_target: Optional[float] = None, ): fallback_mode_enum = FallbackMode(fallback_mode) app = context.obj @@ -316,6 +326,18 @@ def warn(msg: str): stack_trace=msg ) + if fallback_mode_enum == FallbackMode.RANDOM_SAMPLE and fallback_sampling_target is None: + print_error_and_die( + "--fallback-sampling-target is required when --fallback-mode=random-sample", + Tracking.ErrorEvent.USER_ERROR + ) + + if fallback_sampling_target is not None and fallback_mode_enum != FallbackMode.RANDOM_SAMPLE: + print_error_and_die( + "--fallback-sampling-target can only be used with --fallback-mode=random-sample", + Tracking.ErrorEvent.USER_ERROR + ) + if is_get_tests_from_guess and is_get_tests_from_previous_sessions: print_error_and_die( "--get-tests-from-guess (list up tests from git ls-files and subset from there) and --get-tests-from-previous-sessions (list up tests from the recent runs and subset from there) are mutually exclusive. Which one do you want to use?", # noqa E501 @@ -425,6 +447,7 @@ def __init__(self, app: Application): self.is_get_tests_from_guess = is_get_tests_from_guess self.subset_id_file = subset_id_file self.fallback_mode = fallback_mode_enum + self.fallback_sampling_target = fallback_sampling_target super(Optimize, self).__init__(app=app) def _default_output_handler(self, output: List[TestPath], rests: List[TestPath]): @@ -618,7 +641,8 @@ def _fallback_result(self) -> SubsetResult: ) sys.exit(1) elif self.fallback_mode == FallbackMode.RANDOM_SAMPLE: - target_fraction = float(target) if target is not None else 1.0 + assert self.fallback_sampling_target is not None + target_fraction = float(self.fallback_sampling_target) click.echo( "Warning: the service failed to subset. Falling back to local random sample at {:.0%}.".format( target_fraction), diff --git a/tests/commands/test_api_error.py b/tests/commands/test_api_error.py index c935fd62b..a3623fd98 100644 --- a/tests/commands/test_api_error.py +++ b/tests/commands/test_api_error.py @@ -513,9 +513,11 @@ def test_api_error_fallback_random_sample(self): status=500) with tempfile.NamedTemporaryFile(delete=False) as rest_file: - result = self.cli(*self._subset_args(rest_file.name, ("--fallback-mode", "random-sample")), mix_stderr=False) + extra = ("--fallback-mode", "random-sample", "--fallback-sampling-target", "100%") + result = self.cli(*self._subset_args(rest_file.name, extra), mix_stderr=False) self.assert_success(result) self.assertIn("example_test.rb", result.stdout) + self.assertIn("at 100%", result.stderr) @responses.activate @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) @@ -561,7 +563,8 @@ def test_brainless_fallback_random_sample(self): status=200) with tempfile.NamedTemporaryFile(delete=False) as rest_file: - result = self.cli(*self._subset_args(rest_file.name, ("--fallback-mode", "random-sample")), mix_stderr=False) + extra = ("--fallback-mode", "random-sample", "--fallback-sampling-target", "50%") + result = self.cli(*self._subset_args(rest_file.name, extra), mix_stderr=False) self.assert_success(result) self.assertIn("example_test.rb", result.stdout) @@ -580,3 +583,13 @@ def test_brainless_fallback_run_all_default(self): result = self.cli(*self._subset_args(rest_file.name), mix_stderr=False) self.assert_success(result) self.assertIn("example_test.rb", result.stdout) + + # --- Validation error cases --- + + @responses.activate + @mock.patch.dict(os.environ, {"LAUNCHABLE_TOKEN": CliTestCase.launchable_token}) + def test_random_sample_without_fallback_sampling_target_fails(self): + with tempfile.NamedTemporaryFile(delete=False) as rest_file: + result = self.cli(*self._subset_args(rest_file.name, ("--fallback-mode", "random-sample")), mix_stderr=False) + self.assertNotEqual(result.exit_code, 0) + self.assertIn("--fallback-sampling-target", result.stderr)