From 2203869c64e84e5404fe51f327be3698351bc8b9 Mon Sep 17 00:00:00 2001 From: Lucas Jia Date: Mon, 6 Jul 2026 18:44:12 -0700 Subject: [PATCH 1/2] test: disable HF Xet/hf_transfer in serve integ tests to fix endpoint deploy The schema_builder and serve_tei SAGEMAKER_ENDPOINT tests began failing around 2026-06-29 because HuggingFace Hub migrated model repos to the Xet storage backend. The HF DLC container downloads the model at startup with hf_transfer enabled, which now hits the Xet CDN (us.aws.cdn.hf.co) and gets 403 Forbidden. The container never passes the ping health check, the endpoint enters Failed, and the test hits its 15-minute deploy timeout. Inject HF_HUB_ENABLE_HF_TRANSFER=0 and HF_HUB_DISABLE_XET=1 into the container env so the model is pulled over plain HTTPS, restoring the pre-Xet download path. Evidence: endpoint container logs show hf_transfer 403 against the Xet bridge URL; the same tests passed in build #2712 (2026-06-16) before the rollout. --- tests/integ/sagemaker/serve/test_schema_builder.py | 14 ++++++++++++++ tests/integ/sagemaker/serve/test_serve_tei.py | 7 ++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/tests/integ/sagemaker/serve/test_schema_builder.py b/tests/integ/sagemaker/serve/test_schema_builder.py index 1a50678cd7..fa3c55f4cc 100644 --- a/tests/integ/sagemaker/serve/test_schema_builder.py +++ b/tests/integ/sagemaker/serve/test_schema_builder.py @@ -92,6 +92,13 @@ def test_model_builder_happy_path_with_task_provided_local_schema_mode( model_metadata={"HF_TASK": task_provided}, instance_type=instance_type_provided, sagemaker_session=sagemaker_session, + env_vars={ + # Force plain HTTPS model download in the container. HF Hub repos are now + # backed by Xet storage, and the DLC's hf_transfer path returns 403 against + # the Xet CDN, so the container fails the ping health check on startup. + "HF_HUB_ENABLE_HF_TRANSFER": "0", + "HF_HUB_DISABLE_XET": "1", + }, ) model = model_builder.build(sagemaker_session=sagemaker_session) @@ -169,6 +176,13 @@ def test_model_builder_happy_path_with_task_provided_remote_schema_mode( model_metadata={"HF_TASK": task_provided}, instance_type=instance_type_provided, sagemaker_session=sagemaker_session, + env_vars={ + # Force plain HTTPS model download in the container. HF Hub repos are now + # backed by Xet storage, and the DLC's hf_transfer path returns 403 against + # the Xet CDN, so the container fails the ping health check on startup. + "HF_HUB_ENABLE_HF_TRANSFER": "0", + "HF_HUB_DISABLE_XET": "1", + }, ) model = model_builder.build(sagemaker_session=sagemaker_session) diff --git a/tests/integ/sagemaker/serve/test_serve_tei.py b/tests/integ/sagemaker/serve/test_serve_tei.py index 4c824da401..af22a18531 100644 --- a/tests/integ/sagemaker/serve/test_serve_tei.py +++ b/tests/integ/sagemaker/serve/test_serve_tei.py @@ -47,7 +47,12 @@ def model_builder_model_schema_builder(sagemaker_session): schema_builder=SchemaBuilder(sample_input, loaded_response), env_vars={ # Add this to bypass JumpStart model mapping - "HF_MODEL_ID": "BAAI/bge-m3" + "HF_MODEL_ID": "BAAI/bge-m3", + # Force plain HTTPS model download in the container. HF Hub repos are now + # backed by Xet storage, and the DLC's hf_transfer path returns 403 against + # the Xet CDN, so the container fails the ping health check on startup. + "HF_HUB_ENABLE_HF_TRANSFER": "0", + "HF_HUB_DISABLE_XET": "1", }, ) From 56ed32030d1497b44f5e0bce96042d51d9ae9c2f Mon Sep 17 00:00:00 2001 From: Lucas Jia Date: Tue, 7 Jul 2026 10:15:39 -0700 Subject: [PATCH 2/2] test: Mock JumpStartModel in optimize/compile JS builder test --- tests/unit/sagemaker/serve/builder/test_js_builder.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/unit/sagemaker/serve/builder/test_js_builder.py b/tests/unit/sagemaker/serve/builder/test_js_builder.py index 0f976ba060..2cc1decb41 100644 --- a/tests/unit/sagemaker/serve/builder/test_js_builder.py +++ b/tests/unit/sagemaker/serve/builder/test_js_builder.py @@ -1272,12 +1272,14 @@ def test_jumpstart_model_provider_calls_jumpstart_speculative_decoding( mock_js_speculative_decoding.assert_called_once() + @patch("sagemaker.serve.builder.jumpstart_builder.JumpStartModel") @patch("sagemaker.serve.builder.jumpstart_builder._capture_telemetry", side_effect=None) @patch.object(ModelBuilder, "_get_serve_setting", autospec=True) def test_optimize_quantize_and_compile_for_jumpstart( self, mock_serve_settings, mock_telemetry, + mock_js_model, ): mock_sagemaker_session = Mock() mock_metadata_config = Mock() @@ -1319,6 +1321,10 @@ def test_optimize_quantize_and_compile_for_jumpstart( model_builder.pysdk_model = mock_pysdk_model + # Avoid a real JumpStartModel construction (which would hit S3 for the + # JumpStart manifest); return a deterministic env for neuron lookup. + mock_js_model.return_value.env = {"SAGEMAKER_ENV": "1"} + out_put = model_builder._optimize_for_jumpstart( accept_eula=True, quantization_config={