From 1d355c041bc0ba22a333b214c056d5e12fbf3931 Mon Sep 17 00:00:00 2001 From: sandeshk-intel Date: Wed, 20 May 2026 10:11:24 +0530 Subject: [PATCH 1/4] vLLM cpu image upgrade to v0.21.0 (#110) Signed-off-by: sandeshk --- .github/workflows/code-scans.yaml | 2 +- core/helm-charts/vllm/values.yaml | 2 +- core/helm-charts/vllm/xeon-values.yaml | 5 +---- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/code-scans.yaml b/.github/workflows/code-scans.yaml index a94b5dd0..64402fd5 100644 --- a/.github/workflows/code-scans.yaml +++ b/.github/workflows/code-scans.yaml @@ -52,7 +52,7 @@ jobs: continue-on-error: true with: scan-type: 'image' - image-ref: 'public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:v0.10.2' + image-ref: 'public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:v0.21.0' severity: 'HIGH,CRITICAL' exit-code: 0 format: 'table' diff --git a/core/helm-charts/vllm/values.yaml b/core/helm-charts/vllm/values.yaml index 0a3d5df1..7e3688ae 100644 --- a/core/helm-charts/vllm/values.yaml +++ b/core/helm-charts/vllm/values.yaml @@ -25,7 +25,7 @@ image: # Uncomment the following line to set desired image pull policy if needed, as one of Always, IfNotPresent, Never. # pullPolicy: "" # Overrides the image tag whose default is the chart appVersion. - tag: "v0.10.2" + tag: "v0.21.0" imagePullSecrets: [] nameOverride: "" diff --git a/core/helm-charts/vllm/xeon-values.yaml b/core/helm-charts/vllm/xeon-values.yaml index 77d92bf1..89eb749d 100644 --- a/core/helm-charts/vllm/xeon-values.yaml +++ b/core/helm-charts/vllm/xeon-values.yaml @@ -27,7 +27,7 @@ max_model_len: 8192 # vLLM CPU image configuration image: repository: public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo - tag: "v0.10.2" + tag: "v0.21.0" pullPolicy: IfNotPresent # Node affinity for Xeon inference nodes @@ -74,7 +74,6 @@ modelConfigs: "2048", "--max-num-seqs", "256", - "--disable-log-requests", "--enable-auto-tool-choice", "--tool-call-parser", "llama3_json", ] @@ -137,7 +136,6 @@ modelConfigs: "2048", "--max-num-seqs", "256", - "--disable-log-requests", ] tensor_parallel_size: "{{ .Values.tensor_parallel_size }}" pipeline_parallel_size: "{{ .Values.pipeline_parallel_size }}" @@ -167,7 +165,6 @@ modelConfigs: "2048", "--max-num-seqs", "256", - "--disable-log-requests", ] tensor_parallel_size: "{{ .Values.tensor_parallel_size }}" pipeline_parallel_size: "{{ .Values.pipeline_parallel_size }}" From f7a881cf18989d1e6d117c781d3a22119de2f59a Mon Sep 17 00:00:00 2001 From: sandeshk-intel Date: Mon, 25 May 2026 13:55:17 +0530 Subject: [PATCH 2/4] Add vllm user to avoid get user issue during torch compilation (#111) * Add vllm user to avoid get user issue during torch compilation Signed-off-by: sandeshk * Update xeon-values.yaml * Update vLLM cpu image Signed-off-by: sandeshk --------- Signed-off-by: sandeshk --- .github/workflows/code-scans.yaml | 2 +- core/helm-charts/vllm/values.yaml | 2 +- core/helm-charts/vllm/xeon-values.yaml | 10 +++++++++- core/scripts/vllm-quickstart/README.md | 2 +- core/scripts/vllm-quickstart/models.json | 2 +- 5 files changed, 13 insertions(+), 5 deletions(-) diff --git a/.github/workflows/code-scans.yaml b/.github/workflows/code-scans.yaml index 64402fd5..d6aaecd2 100644 --- a/.github/workflows/code-scans.yaml +++ b/.github/workflows/code-scans.yaml @@ -52,7 +52,7 @@ jobs: continue-on-error: true with: scan-type: 'image' - image-ref: 'public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:v0.21.0' + image-ref: 'vllm/vllm-openai-cpu:v0.21.0' severity: 'HIGH,CRITICAL' exit-code: 0 format: 'table' diff --git a/core/helm-charts/vllm/values.yaml b/core/helm-charts/vllm/values.yaml index 7e3688ae..cf384277 100644 --- a/core/helm-charts/vllm/values.yaml +++ b/core/helm-charts/vllm/values.yaml @@ -21,7 +21,7 @@ accelDevice: "" port: 2080 shmSize: 1Gi image: - repository: public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo + repository: vllm/vllm-openai-cpu # Uncomment the following line to set desired image pull policy if needed, as one of Always, IfNotPresent, Never. # pullPolicy: "" # Overrides the image tag whose default is the chart appVersion. diff --git a/core/helm-charts/vllm/xeon-values.yaml b/core/helm-charts/vllm/xeon-values.yaml index 89eb749d..2e7f2beb 100644 --- a/core/helm-charts/vllm/xeon-values.yaml +++ b/core/helm-charts/vllm/xeon-values.yaml @@ -26,7 +26,7 @@ max_model_len: 8192 # vLLM CPU image configuration image: - repository: public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo + repository: vllm/vllm-openai-cpu tag: "v0.21.0" pullPolicy: IfNotPresent @@ -58,6 +58,7 @@ modelConfigs: VLLM_CPU_NUM_OF_RESERVED_CPU: "0" HF_HUB_DISABLE_XET: "1" VLLM_CPU_SGL_KERNEL: "1" + USER: "vllm" extraCmdArgs: [ "--block-size", @@ -89,6 +90,7 @@ modelConfigs: VLLM_CPU_NUM_OF_RESERVED_CPU: "0" VLLM_CPU_SGL_KERNEL: "1" HF_HUB_DISABLE_XET: "1" + USER: "vllm" extraCmdArgs: [ "--block-size", @@ -120,6 +122,7 @@ modelConfigs: VLLM_CPU_NUM_OF_RESERVED_CPU: "0" VLLM_CPU_SGL_KERNEL: "1" HF_HUB_DISABLE_XET: "1" + USER: "vllm" extraCmdArgs: [ "--block-size", @@ -149,6 +152,7 @@ modelConfigs: VLLM_CPU_NUM_OF_RESERVED_CPU: "0" VLLM_CPU_SGL_KERNEL: "1" HF_HUB_DISABLE_XET: "1" + USER: "vllm" extraCmdArgs: [ "--block-size", @@ -178,6 +182,7 @@ modelConfigs: VLLM_CPU_NUM_OF_RESERVED_CPU: "0" VLLM_CPU_SGL_KERNEL: "1" HF_HUB_DISABLE_XET: "1" + USER: "vllm" extraCmdArgs: [ "--block-size", @@ -209,6 +214,7 @@ modelConfigs: VLLM_CPU_NUM_OF_RESERVED_CPU: "0" VLLM_CPU_SGL_KERNEL: "1" HF_HUB_DISABLE_XET: "1" + USER: "vllm" extraCmdArgs: [ "--block-size", @@ -242,6 +248,7 @@ modelConfigs: VLLM_CPU_SGL_KERNEL: "1" HF_HUB_DISABLE_XET: "1" LOGNAME: "vllm" + USER: "vllm" extraCmdArgs: [ "--block-size", @@ -273,6 +280,7 @@ defaultModelConfigs: VLLM_CPU_NUM_OF_RESERVED_CPU: "0" VLLM_CPU_SGL_KERNEL: "1" HF_HUB_DISABLE_XET: "1" + USER: "vllm" extraCmdArgs: [ "--block-size", diff --git a/core/scripts/vllm-quickstart/README.md b/core/scripts/vllm-quickstart/README.md index 7dba3473..5b113814 100644 --- a/core/scripts/vllm-quickstart/README.md +++ b/core/scripts/vllm-quickstart/README.md @@ -116,7 +116,7 @@ The `models.json` file contains all configuration: ```json { "docker": { - "image": "public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:v0.11.2", + "image": "vllm/vllm-openai-cpu:v0.21.0", "port": "8000:8000", "environment": { ... }, "volumes": [ ... ] diff --git a/core/scripts/vllm-quickstart/models.json b/core/scripts/vllm-quickstart/models.json index e7e24eff..fa0f3acb 100644 --- a/core/scripts/vllm-quickstart/models.json +++ b/core/scripts/vllm-quickstart/models.json @@ -1,6 +1,6 @@ { "docker": { - "image": "public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:v0.11.2", + "image": "vllm/vllm-openai-cpu:v0.21.0", "port": "8000:8000", "environment": { "VLLM_CPU_SGL_KERNEL": "1", From 4fe496d69c30f63184dc6c232358aacad839f919 Mon Sep 17 00:00:00 2001 From: sandeshk-intel Date: Wed, 8 Jul 2026 11:36:47 +0530 Subject: [PATCH 3/4] vLLM image update to v0.24.0 (#118) Signed-off-by: sandeshk --- .github/workflows/code-scans.yaml | 2 +- core/helm-charts/vllm/values.yaml | 2 +- core/helm-charts/vllm/xeon-values.yaml | 10 +++++++++- core/roles/inference-tools/tasks/main.yml | 2 +- core/scripts/vllm-quickstart/README.md | 2 +- core/scripts/vllm-quickstart/models.json | 2 +- 6 files changed, 14 insertions(+), 6 deletions(-) diff --git a/.github/workflows/code-scans.yaml b/.github/workflows/code-scans.yaml index d6aaecd2..14046c9d 100644 --- a/.github/workflows/code-scans.yaml +++ b/.github/workflows/code-scans.yaml @@ -52,7 +52,7 @@ jobs: continue-on-error: true with: scan-type: 'image' - image-ref: 'vllm/vllm-openai-cpu:v0.21.0' + image-ref: 'vllm/vllm-openai-cpu:v0.24.0' severity: 'HIGH,CRITICAL' exit-code: 0 format: 'table' diff --git a/core/helm-charts/vllm/values.yaml b/core/helm-charts/vllm/values.yaml index cf384277..264806d7 100644 --- a/core/helm-charts/vllm/values.yaml +++ b/core/helm-charts/vllm/values.yaml @@ -25,7 +25,7 @@ image: # Uncomment the following line to set desired image pull policy if needed, as one of Always, IfNotPresent, Never. # pullPolicy: "" # Overrides the image tag whose default is the chart appVersion. - tag: "v0.21.0" + tag: "v0.24.0" imagePullSecrets: [] nameOverride: "" diff --git a/core/helm-charts/vllm/xeon-values.yaml b/core/helm-charts/vllm/xeon-values.yaml index 2e7f2beb..5f5c23ca 100644 --- a/core/helm-charts/vllm/xeon-values.yaml +++ b/core/helm-charts/vllm/xeon-values.yaml @@ -27,7 +27,7 @@ max_model_len: 8192 # vLLM CPU image configuration image: repository: vllm/vllm-openai-cpu - tag: "v0.21.0" + tag: "v0.24.0" pullPolicy: IfNotPresent # Node affinity for Xeon inference nodes @@ -59,6 +59,7 @@ modelConfigs: HF_HUB_DISABLE_XET: "1" VLLM_CPU_SGL_KERNEL: "1" USER: "vllm" + TRITON_CACHE_DIR: "/tmp/.triton" extraCmdArgs: [ "--block-size", @@ -91,6 +92,7 @@ modelConfigs: VLLM_CPU_SGL_KERNEL: "1" HF_HUB_DISABLE_XET: "1" USER: "vllm" + TRITON_CACHE_DIR: "/tmp/.triton" extraCmdArgs: [ "--block-size", @@ -123,6 +125,7 @@ modelConfigs: VLLM_CPU_SGL_KERNEL: "1" HF_HUB_DISABLE_XET: "1" USER: "vllm" + TRITON_CACHE_DIR: "/tmp/.triton" extraCmdArgs: [ "--block-size", @@ -153,6 +156,7 @@ modelConfigs: VLLM_CPU_SGL_KERNEL: "1" HF_HUB_DISABLE_XET: "1" USER: "vllm" + TRITON_CACHE_DIR: "/tmp/.triton" extraCmdArgs: [ "--block-size", @@ -183,6 +187,7 @@ modelConfigs: VLLM_CPU_SGL_KERNEL: "1" HF_HUB_DISABLE_XET: "1" USER: "vllm" + TRITON_CACHE_DIR: "/tmp/.triton" extraCmdArgs: [ "--block-size", @@ -215,6 +220,7 @@ modelConfigs: VLLM_CPU_SGL_KERNEL: "1" HF_HUB_DISABLE_XET: "1" USER: "vllm" + TRITON_CACHE_DIR: "/tmp/.triton" extraCmdArgs: [ "--block-size", @@ -249,6 +255,7 @@ modelConfigs: HF_HUB_DISABLE_XET: "1" LOGNAME: "vllm" USER: "vllm" + TRITON_CACHE_DIR: "/tmp/.triton" extraCmdArgs: [ "--block-size", @@ -281,6 +288,7 @@ defaultModelConfigs: VLLM_CPU_SGL_KERNEL: "1" HF_HUB_DISABLE_XET: "1" USER: "vllm" + TRITON_CACHE_DIR: "/tmp/.triton" extraCmdArgs: [ "--block-size", diff --git a/core/roles/inference-tools/tasks/main.yml b/core/roles/inference-tools/tasks/main.yml index 12801161..51ae0eec 100644 --- a/core/roles/inference-tools/tasks/main.yml +++ b/core/roles/inference-tools/tasks/main.yml @@ -19,7 +19,7 @@ ansible.builtin.pip: name: kubernetes state: present - extra_args: "--break-system-packages" + extra_args: "--break-system-packages --ignore-installed" become: true when: pip_install_result is failed tags: always diff --git a/core/scripts/vllm-quickstart/README.md b/core/scripts/vllm-quickstart/README.md index 5b113814..3a0977c3 100644 --- a/core/scripts/vllm-quickstart/README.md +++ b/core/scripts/vllm-quickstart/README.md @@ -116,7 +116,7 @@ The `models.json` file contains all configuration: ```json { "docker": { - "image": "vllm/vllm-openai-cpu:v0.21.0", + "image": "vllm/vllm-openai-cpu:v0.24.0", "port": "8000:8000", "environment": { ... }, "volumes": [ ... ] diff --git a/core/scripts/vllm-quickstart/models.json b/core/scripts/vllm-quickstart/models.json index fa0f3acb..330bbe3f 100644 --- a/core/scripts/vllm-quickstart/models.json +++ b/core/scripts/vllm-quickstart/models.json @@ -1,6 +1,6 @@ { "docker": { - "image": "vllm/vllm-openai-cpu:v0.21.0", + "image": "vllm/vllm-openai-cpu:v0.24.0", "port": "8000:8000", "environment": { "VLLM_CPU_SGL_KERNEL": "1", From 66f457ba6e168d3846c17c4c6c0518a2c34391f6 Mon Sep 17 00:00:00 2001 From: sandeshk-intel Date: Mon, 27 Jul 2026 09:24:16 +0530 Subject: [PATCH 4/4] Remove SGL kernel flag in default config and update in docs (#120) Signed-off-by: sandeshk --- core/helm-charts/vllm/xeon-values.yaml | 1 - docs/deploy-llm-model-from-hugging-face.md | 23 ++++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/core/helm-charts/vllm/xeon-values.yaml b/core/helm-charts/vllm/xeon-values.yaml index 5f5c23ca..61944121 100644 --- a/core/helm-charts/vllm/xeon-values.yaml +++ b/core/helm-charts/vllm/xeon-values.yaml @@ -285,7 +285,6 @@ defaultModelConfigs: VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1" VLLM_ENGINE_ITERATION_TIMEOUT_S: "120" VLLM_CPU_NUM_OF_RESERVED_CPU: "0" - VLLM_CPU_SGL_KERNEL: "1" HF_HUB_DISABLE_XET: "1" USER: "vllm" TRITON_CACHE_DIR: "/tmp/.triton" diff --git a/docs/deploy-llm-model-from-hugging-face.md b/docs/deploy-llm-model-from-hugging-face.md index 7c104bdd..545f0c06 100644 --- a/docs/deploy-llm-model-from-hugging-face.md +++ b/docs/deploy-llm-model-from-hugging-face.md @@ -20,3 +20,26 @@ This option allows you to deploy any Hugging Face-hosted LLM on the Inference Cl - **Tensor Parallel Size** (based on available Intel® AI Accelerator cards) > **Note**: This deploys a model that has **not** been pre-validated. Make sure the tensor parallel size is configured correctly. An incorrect value can result in the model being stuck in a "not ready" state. + +### Customizing Environment Variables and vLLM Arguments + +Models deployed from Hugging Face use the `defaultModelConfigs` settings in `core/helm-charts/vllm/xeon-values.yaml`. You can customize environment variables and vLLM command-line arguments by editing the `defaultModelConfigs` section of that file before deploying. + +**Example: enabling the SGL kernel, configuring KV cache, and other environment variables** + +```yaml +defaultModelConfigs: + configMapValues: + VLLM_CPU_SGL_KERNEL: "1" # Enable the SGL kernel for improved CPU performance + VLLM_CPU_KVCACHE_SPACE: "40" # KV cache memory allocation in GB + # Add or override any other environment variables here + extraCmdArgs: + [ + "--block-size", "128", + "--dtype", "bfloat16", + "--max-model-len", "8192", + # Add or override any vLLM CLI arguments here + ] +``` + +You can set any vLLM-supported environment variable under `configMapValues` and any vLLM CLI flag under `extraCmdArgs`.