WIP: feat: install full obs. stack from manifests.

go.mod

@@ -65,6 +65,7 @@ require (
 	github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb // indirect
 	github.com/fsnotify/fsnotify v1.9.0 // indirect
 	github.com/fxamacker/cbor/v2 v2.9.0 // indirect
+	github.com/go-errors/errors v1.5.1 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-logr/zapr v1.3.0 // indirect
 	github.com/go-openapi/analysis v0.25.0 // indirect
@@ -110,6 +111,7 @@ require (
 	github.com/moby/spdystream v0.5.1 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
+	github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
 	github.com/muhlemmer/gu v0.3.1 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
@@ -136,6 +138,7 @@ require (
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/fasttemplate v1.2.2 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
+	github.com/xlab/treeprint v1.2.0 // indirect
 	github.com/zitadel/oidc/v3 v3.47.4 // indirect
 	github.com/zitadel/schema v1.3.2 // indirect
 	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
@@ -181,6 +184,8 @@ require (
 	k8s.io/klog/v2 v2.140.0 // indirect
 	k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a // indirect
 	sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect
+	sigs.k8s.io/kustomize/api v0.21.1 // indirect
+	sigs.k8s.io/kustomize/kyaml v0.21.1 // indirect
 	sigs.k8s.io/randfill v1.0.0 // indirect
 	sigs.k8s.io/structured-merge-diff/v6 v6.3.2 // indirect
 	sigs.k8s.io/yaml v1.6.0 // indirect

go.sum

@@ -105,6 +105,8 @@ github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S
 github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
 github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM=
 github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ=
+github.com/go-errors/errors v1.5.1 h1:ZwEMSLRCapFLflTpT7NKaAc7ukJ8ZPEjzlxt8rPN8bk=
+github.com/go-errors/errors v1.5.1/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
 github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA=
 github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
@@ -262,6 +264,8 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8=
 github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0=
+github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4=
 github.com/muhlemmer/gu v0.3.1 h1:7EAqmFrW7n3hETvuAdmFmn4hS8W+z3LgKtrnow+YzNM=
 github.com/muhlemmer/gu v0.3.1/go.mod h1:YHtHR+gxM+bKEIIs7Hmi9sPT3ZDUvTN/i88wQpZkrdM=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
@@ -384,6 +388,7 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
 github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
@@ -392,6 +397,8 @@ github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQ
 github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
 github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
 github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
+github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ=
+github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0=
 github.com/zitadel/oidc/v3 v3.47.4 h1:aRNmqY5XkVNUybkMUO+s1QXxjja0M2hoschoqMNgn6U=
 github.com/zitadel/oidc/v3 v3.47.4/go.mod h1:XxFh0666HRXycyrKmono+3gY0RACpYJLgy4r/+kliKY=
 github.com/zitadel/schema v1.3.2 h1:gfJvt7dOMfTmxzhscZ9KkapKo3Nei3B6cAxjav+lyjI=
@@ -518,6 +525,7 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q=
@@ -544,6 +552,11 @@ sigs.k8s.io/controller-runtime v0.23.3 h1:VjB/vhoPoA9l1kEKZHBMnQF33tdCLQKJtydy4i
 sigs.k8s.io/controller-runtime v0.23.3/go.mod h1:B6COOxKptp+YaUT5q4l6LqUJTRpizbgf9KSRNdQGns0=
 sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg=
 sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg=
+sigs.k8s.io/kustomize v1.0.11/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5QeXV4WCbnt/PEpU=
+sigs.k8s.io/kustomize/api v0.21.1 h1:lzqbzvz2CSvsjIUZUBNFKtIMsEw7hVLJp0JeSIVmuJs=
+sigs.k8s.io/kustomize/api v0.21.1/go.mod h1:f3wkKByTrgpgltLgySCntrYoq5d3q7aaxveSagwTlwI=
+sigs.k8s.io/kustomize/kyaml v0.21.1 h1:IVlbmhC076nf6foyL6Taw4BkrLuEsXUXNpsE+ScX7fI=
+sigs.k8s.io/kustomize/kyaml v0.21.1/go.mod h1:hmxADesM3yUN2vbA5z1/YTBnzLJ1dajdqpQonwBL1FQ=
 sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU=
 sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY=
 sigs.k8s.io/structured-merge-diff/v6 v6.3.2 h1:kwVWMx5yS1CrnFWA/2QHyRVJ8jM6dBA80uLmm0wJkk8=

pkg/controllers/observability/overlay.go

@@ -0,0 +1,168 @@
+package observability
+
+import (
+	"fmt"
+	"io/fs"
+	"path/filepath"
+
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/kustomize/api/krusty"
+	"sigs.k8s.io/kustomize/api/types"
+	"sigs.k8s.io/kustomize/kyaml/filesys"
+	sigsyaml "sigs.k8s.io/yaml"
+
+	"github.com/rhobs/observability-operator/stack"
+)
+
+const (
+	overlayDir = "/stack/generated"
+	stackRoot  = "/stack"
+)
+
+type Overlay struct {
+	namespace  string
+	components []string
+	patches    map[string][]byte
+	resources  map[string][]byte
+}
+
+func NewOverlay(namespace string) *Overlay {
+	return &Overlay{
+		namespace:  namespace,
+		components: nil,
+		patches:    make(map[string][]byte),
+		resources:  make(map[string][]byte),
+	}
+}
+
+func (o *Overlay) AddComponent(relPath string) {
+	o.components = append(o.components, relPath)
+}
+
+func (o *Overlay) AddPatch(path string, content []byte) {
+	o.patches[path] = content
+}
+
+func (o *Overlay) AddResource(path string, content []byte) {
+	o.resources[path] = content
+}
+
+func (o *Overlay) Build() ([]client.Object, error) {
+	fSys := filesys.MakeFsInMemory()
+
+	if err := copyEmbeddedFS(fSys, stack.FS, stackRoot); err != nil {
+		return nil, fmt.Errorf("loading embedded stack manifests: %w", err)
+	}
+
+	if err := o.writeOverlay(fSys); err != nil {
+		return nil, fmt.Errorf("writing overlay: %w", err)
+	}
+
+	k := krusty.MakeKustomizer(krusty.MakeDefaultOptions())
+	resMap, err := k.Run(fSys, overlayDir)
+	if err != nil {
+		return nil, fmt.Errorf("kustomize build: %w", err)
+	}
+
+	var objects []client.Object
+	for _, res := range resMap.Resources() {
+		jsonBytes, err := res.MarshalJSON()
+		if err != nil {
+			return nil, fmt.Errorf("marshaling resource %s: %w", res.CurId(), err)
+		}
+		obj := &unstructured.Unstructured{}
+		if err := obj.UnmarshalJSON(jsonBytes); err != nil {
+			return nil, fmt.Errorf("unmarshaling resource %s: %w", res.CurId(), err)
+		}
+		objects = append(objects, obj)
+	}
+	return objects, nil
+}
+
+func (o *Overlay) writeOverlay(fSys filesys.FileSystem) error {
+	if err := fSys.MkdirAll(overlayDir); err != nil {
+		return err
+	}
+
+	// Build the kustomization
+	kust := types.Kustomization{
+		TypeMeta: types.TypeMeta{
+			APIVersion: "kustomize.config.k8s.io/v1beta1",
+			Kind:       "Kustomization",
+		},
+		Components:   o.components,
+		Transformers: []string{"namespace-transformer.yaml"},
+	}
+
+	for path := range o.resources {
+		kust.Resources = append(kust.Resources, path)
+	}
+	for path := range o.patches {
+		kust.Patches = append(kust.Patches, types.Patch{Path: path})
+	}
+
+	kustYAML, err := sigsyaml.Marshal(kust)
+	if err != nil {
+		return fmt.Errorf("marshaling kustomization: %w", err)
+	}
+	if err := fSys.WriteFile(filepath.Join(overlayDir, "kustomization.yaml"), kustYAML); err != nil {
+		return err
+	}
+
+	// Write the namespace transformer
+	nsTransformer := fmt.Sprintf(`apiVersion: builtin
+kind: NamespaceTransformer
+metadata:
+  name: namespace-transformer
+namespace: %s
+unsetOnly: true
+setRoleBindingSubjects: allServiceAccounts
+`, o.namespace)
+	if err := fSys.WriteFile(filepath.Join(overlayDir, "namespace-transformer.yaml"), []byte(nsTransformer)); err != nil {
+		return err
+	}
+
+	// Write patch and resource files
+	for path, content := range o.patches {
+		fullPath := filepath.Join(overlayDir, path)
+		if err := fSys.MkdirAll(filepath.Dir(fullPath)); err != nil {
+			return err
+		}
+		if err := fSys.WriteFile(fullPath, content); err != nil {
+			return err
+		}
+	}
+	for path, content := range o.resources {
+		fullPath := filepath.Join(overlayDir, path)
+		if err := fSys.MkdirAll(filepath.Dir(fullPath)); err != nil {
+			return err
+		}
+		if err := fSys.WriteFile(fullPath, content); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func jsonToYAML(j []byte) ([]byte, error) {
+	return sigsyaml.JSONToYAML(j)
+}
+
+func copyEmbeddedFS(fSys filesys.FileSystem, embedded fs.FS, root string) error {
+	return fs.WalkDir(embedded, ".", func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		target := filepath.Join(root, path)
+		if d.IsDir() {
+			return fSys.MkdirAll(target)
+		}
+		data, err := fs.ReadFile(embedded, path)
+		if err != nil {
+			return err
+		}
+		return fSys.WriteFile(target, data)
+	})
+}

pkg/controllers/observability/reconcilers.go

@@ -59,7 +59,7 @@ func getReconcilers(ctx context.Context, k8sClient client.Client, k8sReader clie
 	otelSubs := subscription(opts.OpenTelemetryOperator)
 	tempoSubs := subscription(opts.TempoOperator)
 
-	// instance objects
+	// OTEL collector and its RBAC (not yet converted to overlay)
 	otelCol, err := otelCollector(instance)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create OpenTelemetryCollector: %w", err)
@@ -68,26 +68,25 @@ func getReconcilers(ctx context.Context, k8sClient client.Client, k8sReader clie
 	otelcolRBAC, otelcolRBACBinding := otelCollectorComponentsRBAC(instance)
 	instanceObjects = append(instanceObjects, otelcolRBAC)
 	instanceObjects = append(instanceObjects, otelcolRBACBinding)
-	instanceObjects = append(instanceObjects, tempoStack(instance))
+	otelcolTempoRBAC, otelcolTempoRBACBinding := otelCollectorTempoRBAC(instance)
+	instanceObjects = append(instanceObjects, otelcolTempoRBAC)
+	instanceObjects = append(instanceObjects, otelcolTempoRBACBinding)
 
-	secrets, err := tempoStackSecrets(ctx, k8sClient, k8sReader, *instance)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create TempoStack secret: %w", err)
-	}
-	if secrets.objectStorage != nil {
-		instanceObjects = append(instanceObjects, secrets.objectStorage)
-	}
-	if secrets.objectStorageTLSSecret != nil {
-		instanceObjects = append(instanceObjects, secrets.objectStorageTLSSecret)
+	// TempoStack, secrets, and UIPlugin via kustomize overlay
+	overlay := NewOverlay(instance.Namespace)
+	if err := addTempoStack(overlay, instance); err != nil {
+		return nil, fmt.Errorf("failed to build TempoStack overlay: %w", err)
 	}
-	if secrets.objectStorageCAConfigMap != nil {
-		instanceObjects = append(instanceObjects, secrets.objectStorageCAConfigMap)
+	if err := addTempoSecrets(ctx, overlay, k8sClient, k8sReader, instance); err != nil {
+		return nil, fmt.Errorf("failed to build TempoStack secrets: %w", err)
 	}
+	addUIPlugin(overlay)
 
-	otelcolTempoRBAC, otelcolTempoRBACBinding := otelCollectorTempoRBAC(instance)
-	instanceObjects = append(instanceObjects, otelcolTempoRBAC)
-	instanceObjects = append(instanceObjects, otelcolTempoRBACBinding)
-	instanceObjects = append(instanceObjects, uiPlugin())
+	overlayObjects, err := overlay.Build()
+	if err != nil {
+		return nil, fmt.Errorf("failed to build kustomize overlay: %w", err)
+	}
+	instanceObjects = append(instanceObjects, overlayObjects...)
 
 	if instance.ObjectMeta.DeletionTimestamp != nil {
 		for _, obj := range instanceObjects {