[Improvement] Separate S3 storage configuration for MLRun and Kubeflow Pipeline

charts/mlrun-ce/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v1
 name: mlrun-ce
-version: 0.11.0
+version: 0.12.0-rc.1
 description: MLRun Open Source Stack
 home: https://iguazio.com
 icon: https://www.iguazio.com/wp-content/uploads/2019/10/Iguazio-Logo.png

charts/mlrun-ce/templates/NOTES.txt

@@ -81,7 +81,7 @@ SeaweedFS Admin UI is available at:
 {{ .Values.global.externalHostAddress }}:{{ .Values.seaweedfs.adminService.nodePort | default 30093 }}
 {{- end }}
 {{- end }}
--  S3 credentials: {{ .Values.storage.s3.accessKey }} / {{ .Values.storage.s3.secretKey }}
+-  S3 credentials: {{ .Values.storage.local.accessKey }} / {{ .Values.storage.local.secretKey }}
 {{- end }}
 {{- end }}
 

charts/mlrun-ce/templates/_helpers.tpl

@@ -151,24 +151,107 @@ S3 Service Port - returns the port for pipeline config
 {{- end -}}
 
 {{/*
-S3 Access Key - uses top-level s3.storage.accessKey for all components (MLRun, Jupyter, Pipelines)
+S3 Access Key - for MLRun and Jupyter.
+In "local" mode uses the internal SeaweedFS credential (storage.local.accessKey).
+In "s3" mode uses the external AWS credential (storage.s3.accessKey).
 */}}
 {{- define "mlrun-ce.s3.accessKey" -}}
+{{- if eq .Values.storage.mode "local" -}}
+{{- .Values.storage.local.accessKey -}}
+{{- else -}}
 {{- .Values.storage.s3.accessKey -}}
 {{- end -}}
+{{- end -}}
 
 {{/*
-S3 Secret Key - uses top-level s3.storage.secretKey for all components (MLRun, Jupyter, Pipelines)
+S3 Secret Key - for MLRun and Jupyter.
 */}}
 {{- define "mlrun-ce.s3.secretKey" -}}
+{{- if eq .Values.storage.mode "local" -}}
+{{- .Values.storage.local.secretKey -}}
+{{- else -}}
 {{- .Values.storage.s3.secretKey -}}
 {{- end -}}
+{{- end -}}
 
 {{/*
-S3 Bucket - uses top-level s3.storage.bucket for all components
+S3 Bucket - for MLRun and Jupyter.
 */}}
 {{- define "mlrun-ce.s3.bucket" -}}
-{{- .Values.storage.s3.bucket -}}
+{{- if eq .Values.storage.mode "local" -}}
+{{- .Values.storage.local.bucket -}}
+{{- else -}}
+{{- coalesce .Values.global.infrastructure.aws.bucketName .Values.storage.s3.bucket "mlrun" -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Used by: SeaweedFS IAM config, bucket-init job, and KFP Pipelines.
+Always points at the in-cluster SeaweedFS regardless of storage.mode.
+*/}}
+{{- define "mlrun-ce.seaweedfs.s3.accessKey" -}}
+{{- .Values.storage.local.accessKey -}}
+{{- end -}}
+
+{{/*
+SeaweedFS S3 Secret Key - sourced from storage.local.secretKey.
+*/}}
+{{- define "mlrun-ce.seaweedfs.s3.secretKey" -}}
+{{- .Values.storage.local.secretKey -}}
+{{- end -}}
+
+{{/*
+SeaweedFS S3 Bucket - sourced from storage.local.bucket.
+*/}}
+{{- define "mlrun-ce.seaweedfs.s3.bucket" -}}
+{{- .Values.storage.local.bucket -}}
+{{- end -}}
+
+{{/*
+Pipelines S3 Access Key - always uses the in-cluster SeaweedFS credentials.
+KFP always uses SeaweedFS regardless of storage.mode.
+*/}}
+{{- define "mlrun-ce.pipelines.s3.accessKey" -}}
+{{- include "mlrun-ce.seaweedfs.s3.accessKey" . -}}
+{{- end -}}
+
+{{/*
+Pipelines S3 Secret Key - always uses the in-cluster SeaweedFS credentials.
+*/}}
+{{- define "mlrun-ce.pipelines.s3.secretKey" -}}
+{{- include "mlrun-ce.seaweedfs.s3.secretKey" . -}}
+{{- end -}}
+
+{{/*
+Pipelines S3 Bucket - always uses the SeaweedFS bucket.
+*/}}
+{{- define "mlrun-ce.pipelines.s3.bucket" -}}
+{{- include "mlrun-ce.seaweedfs.s3.bucket" . -}}
+{{- end -}}
+
+{{/*
+Pipelines S3 Host - always in-cluster SeaweedFS.
+*/}}
+{{- define "mlrun-ce.pipelines.s3.host" -}}
+{{- include "mlrun-ce.s3.service.host" . -}}
+{{- end -}}
+
+{{/*
+Pipelines S3 Port - always SeaweedFS port.
+*/}}
+{{- define "mlrun-ce.pipelines.s3.port" -}}
+{{- include "mlrun-ce.s3.service.port" . -}}
+{{- end -}}
+
+{{/*
+Pipelines S3 Secure / Insecure - always plain HTTP (in-cluster SeaweedFS).
+*/}}
+{{- define "mlrun-ce.pipelines.s3.secure" -}}
+false
+{{- end -}}
+
+{{- define "mlrun-ce.pipelines.s3.insecure" -}}
+true
 {{- end -}}
 
 {{/*
@@ -197,7 +280,7 @@ s3://
 {{- end -}}
 
 {{- define "mlrun-ce.artifactPath" -}}
-{{- $bucket := .Values.global.infrastructure.aws.bucketName | default "mlrun" -}}
+{{- $bucket := include "mlrun-ce.s3.bucket" . -}}
 {{- $container := .Values.storage.azure.containerName | default "" -}}
 {{- if eq .Values.storage.mode "azure-blob" -}}
 az://{{ $container }}/projects/{{ `{{run.project}}` }}/artifacts
@@ -207,7 +290,7 @@ s3://{{ $bucket }}/projects/{{ `{{run.project}}` }}/artifacts
 {{- end -}}
 
 {{- define "mlrun-ce.featureStore.dataPrefix" -}}
-{{- $bucket := .Values.global.infrastructure.aws.bucketName | default "mlrun" -}}
+{{- $bucket := include "mlrun-ce.s3.bucket" . -}}
 {{- $container := .Values.storage.azure.containerName | default "" -}}
 {{- if eq .Values.storage.mode "azure-blob" -}}
 az://{{ $container }}/projects/{project}/FeatureStore/{name}/{kind}
@@ -217,7 +300,7 @@ s3://{{ $bucket }}/projects/{project}/FeatureStore/{name}/{kind}
 {{- end -}}
 
 {{- define "mlrun-ce.model-endpoint.monitoring.userSpace" -}}
-{{- $bucket := .Values.global.infrastructure.aws.bucketName | default "mlrun" -}}
+{{- $bucket := include "mlrun-ce.s3.bucket" . -}}
 {{- $container := .Values.storage.azure.containerName | default "" -}}
 {{- if eq .Values.storage.mode "azure-blob" -}}
 az://{{ $container }}/projects/{{ `{{project}}` }}/model-endpoints/{{ `{{kind}}` }}
@@ -227,7 +310,7 @@ s3://{{ $bucket }}/projects/{{ `{{project}}` }}/model-endpoints/{{ `{{kind}}` }}
 {{- end -}}
 
 {{- define "mlrun-ce.model-endpoint.monitoring.application" -}}
-{{- $bucket := .Values.global.infrastructure.aws.bucketName | default "mlrun" -}}
+{{- $bucket := include "mlrun-ce.s3.bucket" . -}}
 {{- $container := .Values.storage.azure.containerName | default "" -}}
 {{- if eq .Values.storage.mode "azure-blob" -}}
 az://{{ $container }}/users/pipelines/{{ `{{project}}` }}/monitoring-apps/
@@ -237,7 +320,7 @@ s3://{{ $bucket }}/users/pipelines/{{ `{{project}}` }}/monitoring-apps/
 {{- end -}}
 
 {{- define "mlrun-ce.model-endpoint.monitoring.default" -}}
-{{- $bucket := .Values.global.infrastructure.aws.bucketName | default "mlrun" -}}
+{{- $bucket := include "mlrun-ce.s3.bucket" . -}}
 {{- $container := .Values.storage.azure.containerName | default "" -}}
 {{- if eq .Values.storage.mode "azure-blob" -}}
 az://{{ $container }}/projects/{{ `{{project}}` }}/model-endpoints/{{ `{{kind}}` }}

charts/mlrun-ce/templates/config/mlrun-env-configmap.yaml

@@ -10,7 +10,7 @@ data:
   MLRUN_HTTPDB__PROJECTS__LEADER: mlrun
   MLRUN_HTTPDB__PROJECTS__FOLLOWERS: nuclio
   # Storage credentials are loaded from the 'storage-credentials' Secret
-  # (AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY/AWS_ENDPOINT_URL_S3 for s3, or AZURE_STORAGE_* for azure-blob)
+  # (local: AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY/AWS_ENDPOINT_URL_S3; s3: AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY; azure-blob: AZURE_STORAGE_*)
   MLRUN_FUNCTION__SPEC__SERVICE_ACCOUNT__DEFAULT: {{ .Values.mlrun.api.functionSpecServiceAccountDefault | default "" | quote }}
   MLRUN_HTTPDB__REAL_PATH: {{ include "mlrun-ce.httpdb.realPath" . | quote }}
   MLRUN_ARTIFACT_PATH: {{ include "mlrun-ce.artifactPath" . | quote }}

charts/mlrun-ce/templates/config/storage-secret.yaml

@@ -25,8 +25,7 @@ stringData:
   {{- with .Values.storage.azure.tenantId }}
   AZURE_STORAGE_TENANT_ID: {{ . }}
   {{- end }}
-{{- else }}
-{{- if not .Values.global.infrastructure.aws.s3NonAnonymous }}
+{{- else if not .Values.global.infrastructure.aws.s3NonAnonymous }}
 apiVersion: v1
 kind: Secret
 metadata:
@@ -37,6 +36,7 @@ type: Opaque
 stringData:
   AWS_ACCESS_KEY_ID: {{ include "mlrun-ce.s3.accessKey" . }}
   AWS_SECRET_ACCESS_KEY: {{ include "mlrun-ce.s3.secretKey" . }}
+  {{- if eq .Values.storage.mode "local" }}
   AWS_ENDPOINT_URL_S3: {{ include "mlrun-ce.s3.service.url" . }}
+  {{- end }}
 {{- end }}
-{{- end }}

charts/mlrun-ce/templates/config/storage-validation.yaml

@@ -1,6 +1,15 @@
 {{- if and (eq .Values.storage.mode "s3") (not .Values.storage.s3.bucket) }}
 {{ fail "storage.mode is set to \"s3\" but storage.s3.bucket is not provided. Please set storage.s3.bucket." }}
 {{- end }}
+{{- if and (eq .Values.storage.mode "s3") (not .Values.global.infrastructure.aws.s3NonAnonymous) (not .Values.storage.s3.accessKey) }}
+{{ fail "storage.mode is set to \"s3\" but storage.s3.accessKey is empty. Set storage.s3.accessKey, or set global.infrastructure.aws.s3NonAnonymous=true to use IAM role-based access." }}
+{{- end }}
+{{- if and (eq .Values.storage.mode "s3") (not .Values.global.infrastructure.aws.s3NonAnonymous) (not .Values.storage.s3.secretKey) }}
+{{ fail "storage.mode is set to \"s3\" but storage.s3.secretKey is empty. Set storage.s3.secretKey, or set global.infrastructure.aws.s3NonAnonymous=true to use IAM role-based access." }}
+{{- end }}
+{{- if and (eq .Values.storage.mode "local") (not .Values.storage.local.bucket) }}
+{{ fail "storage.mode is set to \"local\" but storage.local.bucket is not provided. Please set storage.local.bucket." }}
+{{- end }}
 {{- if and (eq .Values.storage.mode "azure-blob") (not .Values.storage.azure.containerName) }}
 {{ fail "storage.mode is set to \"azure-blob\" but storage.azure.containerName is not provided. Please set storage.azure.containerName." }}
-{{- end }}
+{{- end }}

charts/mlrun-ce/templates/pipelines/configmaps/pipeline-install-config.yaml

@@ -1,13 +1,13 @@
 {{- if .Values.pipelines.enabled -}}
 apiVersion: v1
 data:
-  objectStoreServiceHost: {{ include "mlrun-ce.s3.service.host" . }}
-  objectStoreServicePort: {{ include "mlrun-ce.s3.service.port" . | quote }}
+  objectStoreServiceHost: {{ include "mlrun-ce.pipelines.s3.host" . }}
+  objectStoreServicePort: {{ include "mlrun-ce.pipelines.s3.port" . | quote }}
   ConMaxLifeTime: 120s
   appName: pipeline
   appVersion: {{ .Values.pipelines.images.apiServer.tag }}
   autoUpdatePipelineDefaultVersion: "false"
-  bucketName: "{{ include "mlrun-ce.s3.bucket" . }}"
+  bucketName: "{{ include "mlrun-ce.pipelines.s3.bucket" . }}"
   cacheDb: cachedb
   cacheImage: {{ .Values.pipelines.images.cacheImage.repository }}:{{ .Values.pipelines.images.cacheImage.tag }}
   cacheNodeRestrictions: "false"

charts/mlrun-ce/templates/pipelines/configmaps/workflow-controller-configmap.yaml

@@ -9,9 +9,9 @@ data:
   artifactRepository: |
     archiveLogs: {{ .Values.pipelines.archiveLogs }}
     s3:
-      endpoint: "{{ include "mlrun-ce.s3.service.host" . }}:{{ include "mlrun-ce.s3.service.port" . }}"
-      bucket: "{{ include "mlrun-ce.s3.bucket" . }}"
-      insecure: true
+      endpoint: "{{ include "mlrun-ce.pipelines.s3.host" . }}:{{ include "mlrun-ce.pipelines.s3.port" . }}"
+      bucket: "{{ include "mlrun-ce.pipelines.s3.bucket" . }}"
+      insecure: {{ include "mlrun-ce.pipelines.s3.insecure" . }}
       accessKeySecret:
         name: mlpipeline-seaweedfs-artifact
         key: accesskey

charts/mlrun-ce/templates/pipelines/deployments/ml-pipeline-ui.yaml

@@ -33,9 +33,9 @@ spec:
         - name: VIEWER_TENSORBOARD_POD_TEMPLATE_SPEC_PATH
           value: /etc/config/viewer-pod-template.json
         - name: MINIO_HOST
-          value: {{ include "mlrun-ce.s3.service.host" . }}
+          value: {{ include "mlrun-ce.pipelines.s3.host" . }}
         - name: MINIO_PORT
-          value: {{ include "mlrun-ce.s3.service.port" . | quote }}
+          value: {{ include "mlrun-ce.pipelines.s3.port" . | quote }}
         - name: MINIO_NAMESPACE
           # This is required because otherwise the namespace is appended to the MinIO hostname used to fetch artifacts, causing the fetch to fail
           value: ""

charts/mlrun-ce/templates/pipelines/deployments/ml-pipeline.yaml

@@ -75,7 +75,7 @@ spec:
                   name: pipeline-install-config
                   key: objectStoreServicePort
             - name: OBJECTSTORECONFIG_SECURE
-              value: "false"
+              value: {{ include "mlrun-ce.pipelines.s3.secure" . | quote }}
             - name: OBJECTSTORECONFIG_BUCKETNAME
               valueFrom:
                 configMapKeyRef:

charts/mlrun-ce/templates/pipelines/secrets/mlpipeline-seaweedfs-artifact.yaml

@@ -1,8 +1,8 @@
 {{- if .Values.pipelines.enabled -}}
 apiVersion: v1
 data:
-  accesskey: {{ include "mlrun-ce.s3.accessKey" . | b64enc | quote }}
-  secretkey: {{ include "mlrun-ce.s3.secretKey" . | b64enc | quote }}
+  accesskey: {{ include "mlrun-ce.pipelines.s3.accessKey" . | b64enc | quote }}
+  secretkey: {{ include "mlrun-ce.pipelines.s3.secretKey" . | b64enc | quote }}
 kind: Secret
 metadata:
   annotations:

charts/mlrun-ce/templates/seaweedfs/s3-bucket-init-job.yaml

@@ -37,13 +37,13 @@ spec:
         image: amazon/aws-cli:2.15.0
         env:
         - name: AWS_ACCESS_KEY_ID
-          value: {{ include "mlrun-ce.s3.accessKey" . | quote }}
+          value: {{ include "mlrun-ce.seaweedfs.s3.accessKey" . | quote }}
         - name: AWS_SECRET_ACCESS_KEY
-          value: {{ include "mlrun-ce.s3.secretKey" . | quote }}
+          value: {{ include "mlrun-ce.seaweedfs.s3.secretKey" . | quote }}
         - name: AWS_ENDPOINT_URL
           value: {{ include "mlrun-ce.s3.service.url" . | quote }}
         - name: BUCKET_NAME
-          value: {{ include "mlrun-ce.s3.bucket" . | quote }}
+          value: {{ include "mlrun-ce.seaweedfs.s3.bucket" . | quote }}
         command:
         - /bin/sh
         - -c

charts/mlrun-ce/templates/seaweedfs/seaweedfs-s3-config.yaml

@@ -15,8 +15,8 @@ stringData:
           "name": "admin",
           "credentials": [
             {
-              "accessKey": "{{ include "mlrun-ce.s3.accessKey" . }}",
-              "secretKey": "{{ include "mlrun-ce.s3.secretKey" . }}"
+              "accessKey": "{{ include "mlrun-ce.seaweedfs.s3.accessKey" . }}",
+              "secretKey": "{{ include "mlrun-ce.seaweedfs.s3.secretKey" . }}"
             }
           ],
           "actions": [

charts/mlrun-ce/values.yaml

@@ -21,21 +21,39 @@ global:
       domainNameCertificate: ~
 
 # =============================================================================
-# S3-compatible storage configuration
-# These credentials are used by MLRun, Jupyter, and Kubeflow Pipelines
-# to access the storage backend.
+# S3-compatible storage configuration for MLRun and Jupyter.
+# Kubeflow Pipelines always uses the bundled SeaweedFS — storage.mode only
+# affects MLRun and Jupyter.
 # =============================================================================
 
 # storage.mode selects which backend credentials are injected into the 'storage-credentials' Secret.
 # Options:
-#   s3 (default) - uses storage.s3.accessKey/secretKey/bucket with SeaweedFS endpoint
-#   azure-blob   - uses storage.azure.* fields
+#   local (default) - bundled SeaweedFS; injects AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY,
+#                     and AWS_ENDPOINT_URL_S3 pointing at the in-cluster SeaweedFS S3 service.
+#                     Credentials come from storage.local.accessKey/secretKey; artifact path from
+#                     storage.local.bucket.
+#                     SeaweedFS IAM, bucket-init job, and KFP Pipelines always use storage.local.*
+#                     regardless of storage.mode.
+#   s3              - external AWS S3; injects AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY only
+#                     into MLRun and Jupyter (no custom endpoint — SDK uses the standard AWS endpoint).
+#                     Credentials come from storage.s3.accessKey/secretKey; artifact path from
+#                     storage.s3.bucket (or global.infrastructure.aws.bucketName).
+#                     SeaweedFS and KFP Pipelines continue to use storage.local.* unchanged.
+#   azure-blob      - Azure Blob Storage; injects AZURE_STORAGE_* fields from storage.azure.*
 storage:
-  mode: s3
-  s3:
+  mode: local
+  # Single source of truth for the in-cluster SeaweedFS.
+  # Always used by: SeaweedFS IAM config, bucket-init job, and KFP Pipelines.
+  # Also used by MLRun and Jupyter when mode is "local".
+  local:
     accessKey: "seaweed"
     secretKey: "seaweed123"
     bucket: "mlrun"
+  # External AWS S3 credentials — only applied to MLRun and Jupyter when mode is "s3".
+  s3:
+    accessKey: ""
+    secretKey: ""
+    bucket: ""
   azure:
     containerName: ""
     connectionString: ""
@@ -357,6 +375,7 @@ seaweedfs:
   # S3 auth config - enableAuth gates the seaweedfs-s3-config Secret creation in
   # templates/seaweedfs/seaweedfs-s3-config.yaml even though the dedicated s3 pod
   # is disabled. The secret is consumed by allInOne.s3.existingConfigSecret below.
+  # Credentials are sourced from storage.local.* — for the in-cluster SeaweedFS.
   s3:
     port: 8333
     enableAuth: true