From 22056fe123ebeb8c21ba0de809bed31c306734bf Mon Sep 17 00:00:00 2001 From: Syed Jafri Date: Thu, 13 Aug 2026 15:50:21 -0700 Subject: [PATCH 1/2] fix: rever preset reward function deletion from hyperparams dict --- .../src/sagemaker/train/rlvr_trainer.py | 18 +++++++++++++++--- .../train/test_rlvr_trainer_integration.py | 2 ++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/sagemaker-train/src/sagemaker/train/rlvr_trainer.py b/sagemaker-train/src/sagemaker/train/rlvr_trainer.py index 7435f561bb..5f03cb5b8c 100644 --- a/sagemaker-train/src/sagemaker/train/rlvr_trainer.py +++ b/sagemaker-train/src/sagemaker/train/rlvr_trainer.py @@ -240,9 +240,6 @@ def _process_hyperparameters(self): if hasattr(self.hyperparameters, 'reward_lambda_arn'): delattr(self.hyperparameters, 'reward_lambda_arn') self.hyperparameters._specs.pop('reward_lambda_arn', None) - if hasattr(self.hyperparameters, 'preset_reward_function'): - delattr(self.hyperparameters, 'preset_reward_function') - self.hyperparameters._specs.pop('preset_reward_function', None) if hasattr(self.hyperparameters, 'data_path'): delattr(self.hyperparameters, 'data_path') self.hyperparameters._specs.pop('data_path', None) @@ -410,7 +407,22 @@ def train(self, training_dataset: Optional[Union[str, DataSet]] = None, Returns: TrainingJob: The SageMaker training job object, or None if dry_run=True. + + Raises: + ValueError: If neither a custom reward function nor a preset reward + function hyperparameter is configured. """ + # A reward signal is required: either a custom reward function (Lambda ARN, + # evaluator ARN, or Evaluator object) or the preset_reward_function hyperparameter. + preset_reward_function = getattr(self.hyperparameters, "preset_reward_function", None) + if not self.custom_reward_function and not preset_reward_function: + raise ValueError( + "RLVR training requires a reward signal. Provide either " + "'custom_reward_function' (a Lambda ARN, evaluator ARN, or Evaluator object) " + "when initializing RLVRTrainer, or set the 'preset_reward_function' " + "hyperparameter (e.g. trainer.hyperparameters.preset_reward_function = 'prime_code')." + ) + # Dispatch based on compute type if isinstance(self.compute, HyperPodCompute): return self._train_hyperpod( diff --git a/sagemaker-train/tests/integ/train/test_rlvr_trainer_integration.py b/sagemaker-train/tests/integ/train/test_rlvr_trainer_integration.py index a022b6846f..8676259307 100644 --- a/sagemaker-train/tests/integ/train/test_rlvr_trainer_integration.py +++ b/sagemaker-train/tests/integ/train/test_rlvr_trainer_integration.py @@ -92,6 +92,8 @@ def test_rlvr_trainer_lora_complete_workflow(sagemaker_session): accept_eula=True, base_job_name=f"rlvr-lora-integ-{unique_id}", ) + + rlvr_trainer.hyperparameters.preset_reward_function = "prime_code" # Create training job training_job = rlvr_trainer.train(wait=False) From 5a0a158e9f5379c4db2082dcd57dcb4d8d012bcf Mon Sep 17 00:00:00 2001 From: Syed Jafri Date: Fri, 14 Aug 2026 10:13:27 -0700 Subject: [PATCH 2/2] testing: add unit test to prevent future regression of preset_reward_function --- .../tests/unit/train/test_rlvr_trainer.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/sagemaker-train/tests/unit/train/test_rlvr_trainer.py b/sagemaker-train/tests/unit/train/test_rlvr_trainer.py index 3929b3cfa5..75dd95c607 100644 --- a/sagemaker-train/tests/unit/train/test_rlvr_trainer.py +++ b/sagemaker-train/tests/unit/train/test_rlvr_trainer.py @@ -200,6 +200,32 @@ def test_train_without_datasets_raises_error(self, mock_finetuning_options, mock with pytest.raises(Exception): trainer.train(wait=False) + @patch('sagemaker.train.common_utils.finetune_utils._get_beta_session') + @patch('sagemaker.train.rlvr_trainer._validate_and_resolve_model_package_group') + @patch('sagemaker.train.rlvr_trainer._get_fine_tuning_options_and_model_arn') + def test_train_raises_when_no_reward_signal(self, mock_finetuning_options, mock_validate_group, mock_get_session): + """Test train() raises ValueError when no reward signal is configured. + + Neither custom_reward_function nor the preset_reward_function hyperparameter + is set, so the guard in train() must raise. Using Mock(spec=[]) ensures + getattr(hyperparameters, "preset_reward_function", None) returns None rather + than an auto-created (truthy) Mock attribute. + """ + mock_validate_group.return_value = "test-group" + mock_get_session.return_value = Mock() + mock_hyperparams = Mock(spec=["to_dict"]) # no preset_reward_function attr + mock_hyperparams.to_dict.return_value = {} + mock_finetuning_options.return_value = (mock_hyperparams, "model-arn", False) + + trainer = RLVRTrainer( + model="test-model", + model_package_group="test-group", + training_dataset="s3://bucket/train", + ) # no custom_reward_function + + with pytest.raises(ValueError, match="requires a reward signal"): + trainer.train(wait=False) + @patch('sagemaker.train.common_utils.finetune_utils._get_beta_session') @patch('sagemaker.train.common_utils.finetune_utils._resolve_model_name') @patch('sagemaker.train.rlvr_trainer._get_fine_tuning_options_and_model_arn')