Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,9 @@ OM_PUBLIC_URL=http://localhost:8080

# --- API ---
OM_API_PORT=8080
# om serve applies pending migrations before it listens. Set false to keep
# schema changes on an explicit `om migrate` (the Helm init container does that).
# OM_AUTO_MIGRATE=true

# Optional. When set, the setup form can only create this first admin.
# OM_ADMIN_EMAIL=
Expand Down
8 changes: 3 additions & 5 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -80,15 +80,13 @@ git clone https://github.com/OpenSourceOM/core.git
cd core
cp .env.example .env

# Start Postgres + API (http://localhost:8080)
# Start Postgres + API (http://localhost:8080).
# om serve applies pending migrations before it listens.
docker compose up -d

# Build the CLI
go build -o om ./cmd/om

# Apply graph schema migrations
./om migrate

# Load the sample graph (no cloud credentials)
./om scan demo

Expand Down Expand Up @@ -141,7 +139,7 @@ An API token is created only when an admin asks for one, from the console or wit
./om scan plugin -- ./my-collector
```

The CLI does not read `.env`. It defaults `POSTGRES_HOST` to `localhost`, which matches the Postgres port Compose publishes, so the host commands above reach the database. Set `POSTGRES_HOST=localhost` only if your shell already exports `POSTGRES_HOST=postgres`. For CLI-only use without the API container, run `docker compose up -d postgres`.
The CLI does not read `.env`. It defaults `POSTGRES_HOST` to `localhost`, which matches the Postgres port Compose publishes, so the host commands above reach the database. Set `POSTGRES_HOST=localhost` only if your shell already exports `POSTGRES_HOST=postgres`. For CLI-only use without the API container, run `docker compose up -d postgres` and then `./om migrate`. `om serve` applies the same files on startup. Set `OM_AUTO_MIGRATE=false` when a separate `om migrate` should be the only writer. The Helm chart still runs `om migrate` in an init container before `om serve`.

**Kubernetes:**

Expand Down
4 changes: 4 additions & 0 deletions deploy/helm/opensourceom/render_test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,10 @@ helm lint "$chart" "${base[@]}"

render default
assert_absent "$workdir/default.yaml" "OM_API_SECRET" "default install still set a shared API secret"
if ! grep -A1 'name: OM_AUTO_MIGRATE' "$workdir/default.yaml" | grep -q 'value: "true"'; then
echo "API container does not apply migrations on startup" >&2
exit 1
fi
assert_absent "$workdir/default.yaml" "kind: CronJob" "default install created a CronJob"
assert_absent "$workdir/default.yaml" "kind: ClusterRole$" "default install created scan RBAC"
assert_absent "$workdir/default.yaml" "- demo" "default install scheduled the demo scan"
Expand Down
2 changes: 2 additions & 0 deletions deploy/helm/opensourceom/templates/api.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,8 @@ spec:
value: {{ .Values.api.env | quote }}
- name: OM_API_PORT
value: "8080"
- name: OM_AUTO_MIGRATE
value: {{ .Values.api.autoMigrate | quote }}
{{- with .Values.api.publicURL }}
- name: OM_PUBLIC_URL
value: {{ . | quote }}
Expand Down
2 changes: 2 additions & 0 deletions deploy/helm/opensourceom/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ service:

api:
env: production
# om serve applies pending migrations. The migrate init container still runs first.
autoMigrate: true
# Browser origin for the console and optional OAuth callbacks. No trailing path.
publicURL: ""
# When set, only this address can become the first admin.
Expand Down
2 changes: 2 additions & 0 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
# SPDX-License-Identifier: Apache-2.0
# Local dev stack: Postgres + API
# Usage: cp .env.example .env && docker compose up -d
# The API applies pending migrations when it starts.
# The API image compiles aws-sdk-go-v2/service/ec2. Docker needs at least 6 GiB of RAM.

services:
Expand Down Expand Up @@ -30,6 +31,7 @@ services:
environment:
OM_ENV: ${OM_ENV:-development}
OM_API_PORT: 8080
OM_AUTO_MIGRATE: ${OM_AUTO_MIGRATE:-true}
OM_PUBLIC_URL: ${OM_PUBLIC_URL:-http://localhost:8080}
OM_ADMIN_EMAIL: ${OM_ADMIN_EMAIL:-}
OM_GOOGLE_CLIENT_ID: ${OM_GOOGLE_CLIENT_ID:-}
Expand Down
4 changes: 2 additions & 2 deletions docs/ARCHITECTURE.md
Original file line number Diff line number Diff line change
Expand Up @@ -67,8 +67,8 @@ See [ADR 001](./adr/001-graph-schema-v0.md), [ADR 002](./adr/002-phase1-findings

## Deployment

- **Dev:** Docker Compose — Postgres + API (`docker compose up -d`); CLI can also target Postgres directly
- **Prod:** Helm chart in `deploy/helm/opensourceom/` (API, optional Postgres, optional scheduled collectors)
- **Dev:** Docker Compose — Postgres + API (`docker compose up -d`). `om serve` applies pending migrations before it listens. The CLI can also target Postgres directly; `om migrate` is the schema step when the API is not running.
- **Prod:** Helm chart in `deploy/helm/opensourceom/` (API, optional Postgres, optional scheduled collectors). An init container runs `om migrate`, then the main container runs `om serve`, which applies any still-pending files unless `OM_AUTO_MIGRATE=false`.

## Phase 3

Expand Down
2 changes: 1 addition & 1 deletion docs/adr/001-graph-schema-v0.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ OpenSourceOM needs a minimal graph model to connect cloud inventory (workloads,
### Storage

- **Backend:** PostgreSQL with two tables — `nodes` and `edges`.
- **Migrations:** Versioned SQL files in `migrations/`, applied by `om migrate`.
- **Migrations:** Versioned SQL files in `migrations/`, applied by `om migrate` and by `om serve` unless `OM_AUTO_MIGRATE=false`.
- **Properties:** Flexible `JSONB` column on nodes and edges for provider-specific attributes.

### Node types (v0)
Expand Down
2 changes: 1 addition & 1 deletion internal/api/blastradius_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,7 @@ func openBlastRadiusStore(t *testing.T) *graph.Store {
}
databaseURL.Path = "/" + dbName

if err := migrate.Run(ctx, databaseURL.String(), blastRadiusMigrationsDir(t)); err != nil {
if _, err := migrate.Run(ctx, databaseURL.String(), blastRadiusMigrationsDir(t)); err != nil {
t.Fatalf("migrate: %v", err)
}
store, err := graph.NewStore(ctx, databaseURL.String())
Expand Down
2 changes: 1 addition & 1 deletion internal/auth/store_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -266,7 +266,7 @@ func openAuthStore(t *testing.T) *Store {
t.Fatal("locate test file")
}
migrations := filepath.Join(filepath.Dir(file), "..", "..", "migrations")
if err := migrate.Run(ctx, databaseURL.String(), migrations); err != nil {
if _, err := migrate.Run(ctx, databaseURL.String(), migrations); err != nil {
t.Fatalf("migrate: %v", err)
}
store, err := NewStore(ctx, databaseURL.String())
Expand Down
4 changes: 2 additions & 2 deletions internal/cmd/helpers.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,10 +50,10 @@ func loadConfig() config.Config {
return config.Load()
}

func runMigrations(ctx context.Context, cfg config.Config) error {
func runMigrations(ctx context.Context, cfg config.Config) ([]string, error) {
migrationsDir, err := repoPath("migrations")
if err != nil {
return err
return nil, err
}
return migrate.Run(ctx, cfg.DatabaseURL, migrationsDir)
}
Expand Down
10 changes: 9 additions & 1 deletion internal/cmd/migrate.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,17 @@ var migrateCmd = &cobra.Command{
Short: "Apply database migrations",
RunE: func(cmd *cobra.Command, args []string) error {
cfg := loadConfig()
if err := runMigrations(cmd.Context(), cfg); err != nil {
applied, err := runMigrations(cmd.Context(), cfg)
if err != nil {
return err
}
if len(applied) == 0 {
fmt.Println("Database schema is current.")
return nil
}
for _, version := range applied {
fmt.Printf("applied %s\n", version)
}
fmt.Println("Migrations applied.")
return nil
},
Expand Down
25 changes: 25 additions & 0 deletions internal/cmd/serve.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,9 @@ var serveCmd = &cobra.Command{
Short: "Start the HTTP API server",
RunE: func(cmd *cobra.Command, args []string) error {
cfg := loadConfig()
if err := applyMigrations(cmd.Context(), cfg); err != nil {
return err
}
store, err := openGraphStore(cmd.Context(), cfg)
if err != nil {
return err
Expand Down Expand Up @@ -49,6 +52,28 @@ var serveCmd = &cobra.Command{
},
}

func applyMigrations(ctx context.Context, cfg config.Config) error {
auto, err := config.ServeAutoMigrate()
if err != nil {
return err
}
if !auto {
log.Printf("skipping database migrations because OM_AUTO_MIGRATE is false")
return nil
}

applied, err := runMigrations(ctx, cfg)
if err != nil {
return fmt.Errorf("apply database migrations: %w", err)
}
if len(applied) == 0 {
log.Printf("database schema is current")
return nil
}
log.Printf("applied database migrations: %s", strings.Join(applied, ", "))
return nil
}

func warnOAuthConfig(cfg config.Config) {
if (cfg.GoogleClientID == "") != (cfg.GoogleClientSecret == "") {
log.Printf("Google console login needs both OM_GOOGLE_CLIENT_ID and OM_GOOGLE_CLIENT_SECRET")
Expand Down
15 changes: 15 additions & 0 deletions internal/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"fmt"
"os"
"strconv"
"strings"
)

type Config struct {
Expand Down Expand Up @@ -91,6 +92,20 @@ func Load() Config {
}
}

// ServeAutoMigrate reports whether om serve applies pending migrations before it listens.
// An unset or empty OM_AUTO_MIGRATE means yes. A present value must be a boolean.
func ServeAutoMigrate() (bool, error) {
raw, ok := os.LookupEnv("OM_AUTO_MIGRATE")
if !ok || strings.TrimSpace(raw) == "" {
return true, nil
}
value, err := strconv.ParseBool(strings.TrimSpace(raw))
if err != nil {
return false, fmt.Errorf("OM_AUTO_MIGRATE=%q is not a boolean", raw)
}
return value, nil
}

func getEnv(key, fallback string) string {
if v := os.Getenv(key); v != "" {
return v
Expand Down
58 changes: 58 additions & 0 deletions internal/config/config_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
// Copyright 2026 OpenSourceOM
// SPDX-License-Identifier: Apache-2.0

package config

import (
"os"
"testing"
)

func TestServeAutoMigrateDefaultsWhenUnset(t *testing.T) {
t.Setenv("OM_AUTO_MIGRATE", "false")
if err := os.Unsetenv("OM_AUTO_MIGRATE"); err != nil {
t.Fatal(err)
}

got, err := ServeAutoMigrate()
if err != nil {
t.Fatal(err)
}
if !got {
t.Fatal("unset OM_AUTO_MIGRATE should apply migrations")
}
}

func TestServeAutoMigrate(t *testing.T) {
tests := []struct {
name string
value string
want bool
wantErr bool
}{
{name: "empty", value: " ", want: true},
{name: "true", value: "true", want: true},
{name: "false", value: "false", want: false},
{name: "zero", value: "0", want: false},
{name: "junk", value: "sometimes", wantErr: true},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Setenv("OM_AUTO_MIGRATE", tt.value)
got, err := ServeAutoMigrate()
if tt.wantErr {
if err == nil {
t.Fatal("expected an error")
}
return
}
if err != nil {
t.Fatalf("ServeAutoMigrate: %v", err)
}
if got != tt.want {
t.Fatalf("ServeAutoMigrate() = %v, want %v", got, tt.want)
}
})
}
}
2 changes: 1 addition & 1 deletion internal/graph/query_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -418,7 +418,7 @@ func openQueryFixture(t *testing.T) *graph.Store {
}
databaseURL.Path = "/" + dbName

if err := migrate.Run(ctx, databaseURL.String(), migrationsDir(t)); err != nil {
if _, err := migrate.Run(ctx, databaseURL.String(), migrationsDir(t)); err != nil {
t.Fatalf("migrate: %v", err)
}
store, err := graph.NewStore(ctx, databaseURL.String())
Expand Down
Loading
Loading