From 8700d2a2581904d2907ef67434f77bf52d9566b6 Mon Sep 17 00:00:00 2001 From: milosjovanovic Date: Mon, 8 Jun 2026 18:10:08 +0200 Subject: [PATCH 1/7] feat: add YAML schema validation before terraform plan Validates repos/*.yaml files against a JSON schema before terraform plan runs on PRs. Fails fast with a clear error if validation fails, preventing wasted CI time on malformed configs. --- .../actions/validate-repo-configs/action.yaml | 15 +++++ .../actions/validate-repo-configs/validate.py | 64 +++++++++++++++++++ .github/workflows/tf-plan.yaml | 31 ++++++++- 3 files changed, 108 insertions(+), 2 deletions(-) create mode 100644 .github/actions/validate-repo-configs/action.yaml create mode 100644 .github/actions/validate-repo-configs/validate.py diff --git a/.github/actions/validate-repo-configs/action.yaml b/.github/actions/validate-repo-configs/action.yaml new file mode 100644 index 0000000..e8ee21a --- /dev/null +++ b/.github/actions/validate-repo-configs/action.yaml @@ -0,0 +1,15 @@ +name: "Validate repository configuration files" +description: "Validates repository config YAML files against a JSON schema, failing fast with a clear error before Terraform runs" +inputs: + config-path: + description: "Path to the configuration repository containing repos/*.yaml files to validate" + required: true + fallback-schema-path: + description: "Path to the base JSON schema to validate against. Can be overridden by placing a schema at /.schemas/repository-config.schema.json" + required: true +runs: + using: "composite" + steps: + - name: Validate repository config files against schema + shell: bash + run: python3 ${{ github.action_path }}/validate.py ${{ inputs.config-path }} ${{ inputs.fallback-schema-path }} diff --git a/.github/actions/validate-repo-configs/validate.py b/.github/actions/validate-repo-configs/validate.py new file mode 100644 index 0000000..8659cc2 --- /dev/null +++ b/.github/actions/validate-repo-configs/validate.py @@ -0,0 +1,64 @@ +import json +import sys +import glob +import os +import yaml +import jsonschema + + +def resolve_schema(config_path, fallback_schema_path): + config_root = os.path.realpath(config_path) + override = os.path.realpath(os.path.join(config_path, ".schemas", "repository-config.schema.json")) + + if not override.startswith(config_root): + print(f"Error: schema override path escapes config directory: {override}") + sys.exit(1) + + if os.path.exists(override): + print(f"Using org schema override: {override}") + return override + + print(f"Using default schema: {fallback_schema_path}") + return fallback_schema_path + + +def validate(config_path, fallback_schema_path): + schema_path = resolve_schema(config_path, fallback_schema_path) + + try: + with open(schema_path) as f: + schema = json.load(f) + except (OSError, json.JSONDecodeError) as e: + print(f"Error: failed to load schema from {schema_path}: {e}") + sys.exit(1) + + files = glob.glob(os.path.join(config_path, "repos", "*.yaml")) + if not files: + print("No repos/*.yaml files found, skipping validation") + return [] + + errors = [] + for f in files: + with open(f) as fh: + data = yaml.safe_load(fh) + try: + jsonschema.validate(data, schema) + print(f"ok -- {f}") + except jsonschema.ValidationError as e: + errors.append(f"{f}: {e.message} at {e.json_path}") + + return errors + + +if __name__ == "__main__": + if len(sys.argv) != 3: + print("Usage: validate.py ") + sys.exit(1) + + validation_errors = validate(sys.argv[1], sys.argv[2]) + + if validation_errors: + print("\nSchema validation errors were encountered:") + for err in validation_errors: + print(f" {err}") + sys.exit(1) diff --git a/.github/workflows/tf-plan.yaml b/.github/workflows/tf-plan.yaml index 80ceebf..583c5f4 100644 --- a/.github/workflows/tf-plan.yaml +++ b/.github/workflows/tf-plan.yaml @@ -1,4 +1,4 @@ -name: Terraform Plan +name: Validate and Plan on: workflow_call: @@ -25,7 +25,34 @@ on: required: true jobs: + validate: + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout GCSS + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + repository: G-Research/github-terraformer + ref: ${{ inputs.gcss_ref }} + persist-credentials: false + + - name: Checkout config repo + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + ref: ${{ inputs.commit_sha }} + token: ${{ secrets.gh_token }} + path: gcss_config + persist-credentials: false + + - name: Validate repos/*.yaml + uses: ./.github/actions/validate-repo-configs + with: + config-path: gcss_config + fallback-schema-path: .schemas/repository-config.schema.json + terraform-plan: + needs: validate runs-on: ubuntu-latest environment: plan permissions: @@ -235,4 +262,4 @@ jobs: summary: summary, text: text } - }); \ No newline at end of file + }); From 9384f0defe16cb5d17c2103dcef989c0ce2a8507 Mon Sep 17 00:00:00 2001 From: milosjovanovic Date: Tue, 9 Jun 2026 10:52:29 +0200 Subject: [PATCH 2/7] refactor: migrate YAML schema validation from Python to Go Replaces the Python-based YAML schema validation with a Go implementation integrated into `github-repo-importer`. Streamlines workflow by leveraging the importer's built-in schema generator and validator. Updates CI to ensure the schema is current before validation. --- .../actions/validate-repo-configs/action.yaml | 19 +- .../actions/validate-repo-configs/validate.py | 64 ----- .github/workflows/ci.yaml | 6 + .github/workflows/tf-plan.yaml | 1 - feature/github-repo-importer/cmd/schema.go | 57 +++-- feature/github-repo-importer/cmd/validate.go | 242 ++++++++++++++++++ .../github-repo-importer/cmd/validate_test.go | 132 ++++++++++ feature/github-repo-importer/go.mod | 2 + feature/github-repo-importer/go.sum | 6 + 9 files changed, 438 insertions(+), 91 deletions(-) delete mode 100644 .github/actions/validate-repo-configs/validate.py create mode 100644 feature/github-repo-importer/cmd/validate.go create mode 100644 feature/github-repo-importer/cmd/validate_test.go diff --git a/.github/actions/validate-repo-configs/action.yaml b/.github/actions/validate-repo-configs/action.yaml index e8ee21a..877661e 100644 --- a/.github/actions/validate-repo-configs/action.yaml +++ b/.github/actions/validate-repo-configs/action.yaml @@ -1,15 +1,22 @@ name: "Validate repository configuration files" -description: "Validates repository config YAML files against a JSON schema, failing fast with a clear error before Terraform runs" +description: "Validates repository config YAML files against the importer's JSON schema, failing fast with a clear error before Terraform runs" inputs: config-path: description: "Path to the configuration repository containing repos/*.yaml files to validate" required: true - fallback-schema-path: - description: "Path to the base JSON schema to validate against. Can be overridden by placing a schema at /.schemas/repository-config.schema.json" - required: true + importer-path: + description: "Path to the github-repo-importer Go module" + required: false + default: "feature/github-repo-importer" runs: using: "composite" steps: - - name: Validate repository config files against schema + - name: Setup Go + uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0 + with: + go-version-file: ${{ inputs.importer-path }}/go.mod + + - name: Validate repos/*.yaml against schema shell: bash - run: python3 ${{ github.action_path }}/validate.py ${{ inputs.config-path }} ${{ inputs.fallback-schema-path }} + working-directory: ${{ inputs.importer-path }} + run: go run . validate --config-dir "${{ github.workspace }}/${{ inputs.config-path }}" diff --git a/.github/actions/validate-repo-configs/validate.py b/.github/actions/validate-repo-configs/validate.py deleted file mode 100644 index 8659cc2..0000000 --- a/.github/actions/validate-repo-configs/validate.py +++ /dev/null @@ -1,64 +0,0 @@ -import json -import sys -import glob -import os -import yaml -import jsonschema - - -def resolve_schema(config_path, fallback_schema_path): - config_root = os.path.realpath(config_path) - override = os.path.realpath(os.path.join(config_path, ".schemas", "repository-config.schema.json")) - - if not override.startswith(config_root): - print(f"Error: schema override path escapes config directory: {override}") - sys.exit(1) - - if os.path.exists(override): - print(f"Using org schema override: {override}") - return override - - print(f"Using default schema: {fallback_schema_path}") - return fallback_schema_path - - -def validate(config_path, fallback_schema_path): - schema_path = resolve_schema(config_path, fallback_schema_path) - - try: - with open(schema_path) as f: - schema = json.load(f) - except (OSError, json.JSONDecodeError) as e: - print(f"Error: failed to load schema from {schema_path}: {e}") - sys.exit(1) - - files = glob.glob(os.path.join(config_path, "repos", "*.yaml")) - if not files: - print("No repos/*.yaml files found, skipping validation") - return [] - - errors = [] - for f in files: - with open(f) as fh: - data = yaml.safe_load(fh) - try: - jsonschema.validate(data, schema) - print(f"ok -- {f}") - except jsonschema.ValidationError as e: - errors.append(f"{f}: {e.message} at {e.json_path}") - - return errors - - -if __name__ == "__main__": - if len(sys.argv) != 3: - print("Usage: validate.py ") - sys.exit(1) - - validation_errors = validate(sys.argv[1], sys.argv[2]) - - if validation_errors: - print("\nSchema validation errors were encountered:") - for err in validation_errors: - print(f" {err}") - sys.exit(1) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 304fe5c..f40be34 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -23,3 +23,9 @@ jobs: - name: Test run: go test ./... + + - name: Check generated schema is up to date + run: | + go run . schema + git diff --exit-code ../../.schemas/repository-config.schema.json \ + || { echo "::error::.schemas/repository-config.schema.json is stale. Run 'go run . schema' in feature/github-repo-importer and commit the result."; exit 1; } diff --git a/.github/workflows/tf-plan.yaml b/.github/workflows/tf-plan.yaml index 583c5f4..95851d0 100644 --- a/.github/workflows/tf-plan.yaml +++ b/.github/workflows/tf-plan.yaml @@ -49,7 +49,6 @@ jobs: uses: ./.github/actions/validate-repo-configs with: config-path: gcss_config - fallback-schema-path: .schemas/repository-config.schema.json terraform-plan: needs: validate diff --git a/feature/github-repo-importer/cmd/schema.go b/feature/github-repo-importer/cmd/schema.go index cd12ab7..ee9457b 100644 --- a/feature/github-repo-importer/cmd/schema.go +++ b/feature/github-repo-importer/cmd/schema.go @@ -11,24 +11,50 @@ import ( orderedmap "github.com/wk8/go-ordered-map/v2" ) +const ( + schemaOutDir = ".schemas" + schemaOutFile = "repository-config.schema.json" +) + var schemaCmd = &cobra.Command{ Use: "schema", Short: "Generate JSON Schema for the repository config", RunE: func(cmd *cobra.Command, args []string) error { projectRoot := "../../" - outDir := ".schemas" - outFile := "repository-config.schema.json" - if err := os.MkdirAll(fmt.Sprintf("%s/%s", projectRoot, outDir), 0o755); err != nil { - return fmt.Errorf("create %s: %w", outDir, err) + if err := os.MkdirAll(fmt.Sprintf("%s/%s", projectRoot, schemaOutDir), 0o755); err != nil { + return fmt.Errorf("create %s: %w", schemaOutDir, err) } - outPath := filepath.Join(projectRoot, outDir, outFile) - f, err := os.Create(outPath) + outPath := filepath.Join(projectRoot, schemaOutDir, schemaOutFile) + + data, err := MarshalRepositoryConfigSchema() if err != nil { - return fmt.Errorf("create schema file: %w", err) + return err } - defer f.Close() + if err := os.WriteFile(outPath, data, 0o644); err != nil { + return fmt.Errorf("write schema file: %w", err) + } + + fmt.Fprintf(cmd.OutOrStdout(), "Schema written to %s\n", outPath) + return nil + }, +} +// MarshalRepositoryConfigSchema returns the JSON-encoded repository config schema. +func MarshalRepositoryConfigSchema() ([]byte, error) { + data, err := json.MarshalIndent(BuildRepositoryConfigSchema(), "", " ") + if err != nil { + return nil, fmt.Errorf("marshal schema: %w", err) + } + return data, nil +} + +// BuildRepositoryConfigSchema reflects the JSON schema for the repository +// configuration from the Go structs and applies the manual constraints that +// cannot be expressed through struct tags. It is the single source of truth +// shared by the `schema` and `validate` commands. +func BuildRepositoryConfigSchema() *jsonschema.Schema { + { reflector := &jsonschema.Reflector{ AllowAdditionalProperties: false, FieldNameTag: "yaml", @@ -36,7 +62,7 @@ var schemaCmd = &cobra.Command{ schema := reflector.Reflect(&RepositoryWithExpansionConfig{}) schema.Title = "Repository Configuration" - schema.ID = jsonschema.ID(fmt.Sprintf("https://raw.githubusercontent.com/G-Research/github-terraformer/refs/heads/main/%s/%s", outDir, outFile)) + schema.ID = jsonschema.ID(fmt.Sprintf("https://raw.githubusercontent.com/G-Research/github-terraformer/refs/heads/main/%s/%s", schemaOutDir, schemaOutFile)) squashIf := &jsonschema.Schema{ Properties: orderedmap.New[string, *jsonschema.Schema](), @@ -108,17 +134,8 @@ var schemaCmd = &cobra.Command{ }) } - data, err := json.MarshalIndent(schema, "", " ") - if err != nil { - return fmt.Errorf("marshal schema: %w", err) - } - if _, err := f.Write(data); err != nil { - return fmt.Errorf("write schema: %w", err) - } - - fmt.Fprintf(cmd.OutOrStdout(), "Schema written to %s\n", outPath) - return nil - }, + return schema + } } func init() { diff --git a/feature/github-repo-importer/cmd/validate.go b/feature/github-repo-importer/cmd/validate.go new file mode 100644 index 0000000..a6f385f --- /dev/null +++ b/feature/github-repo-importer/cmd/validate.go @@ -0,0 +1,242 @@ +package cmd + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/santhosh-tekuri/jsonschema/v6" + "github.com/spf13/cobra" + "golang.org/x/text/language" + "golang.org/x/text/message" + "gopkg.in/yaml.v3" +) + +// printer renders the localized validation messages. A non-nil printer is +// required: ErrorKind.LocalizedString panics on a nil printer. +var printer = message.NewPrinter(language.English) + +// orgSchemaRelPath is the location, relative to the config directory, where an +// organization can drop a schema that overrides the built-in one. +const orgSchemaRelPath = ".schemas/repository-config.schema.json" + +var ( + validateConfigDir string + validateSchemaPath string +) + +var validateCmd = &cobra.Command{ + Use: "validate", + Short: "Validate repository config YAML files against the JSON schema", + Long: `Validate validates every repos/*.yaml file in the config directory against the +repository configuration JSON schema, failing fast with the offending file and +JSON path before Terraform runs. + +By default it validates against the schema built from the importer's own structs +(the same one produced by the 'schema' command). If the config directory +contains an override at ` + orgSchemaRelPath + `, that schema is used instead.`, + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + return runValidate(cmd, validateConfigDir, validateSchemaPath) + }, +} + +func init() { + rootCmd.AddCommand(validateCmd) + + validateCmd.Flags().StringVar(&validateConfigDir, "config-dir", "", "Path to the config repository containing repos/*.yaml") + validateCmd.Flags().StringVar(&validateSchemaPath, "schema", "", "Path to a schema override (defaults to /"+orgSchemaRelPath+" when present, otherwise the built-in schema)") + validateCmd.MarkFlagRequired("config-dir") +} + +func runValidate(cmd *cobra.Command, configDir, schemaOverride string) error { + schema, err := loadValidationSchema(cmd, configDir, schemaOverride) + if err != nil { + return err + } + + files, err := filepath.Glob(filepath.Join(configDir, "repos", "*.yaml")) + if err != nil { + return fmt.Errorf("list repos/*.yaml: %w", err) + } + sort.Strings(files) + + if len(files) == 0 { + fmt.Fprintln(cmd.OutOrStdout(), "No repos/*.yaml files found, skipping validation") + return nil + } + + var failures []string + for _, file := range files { + fileErrs := validateFile(file, schema) + if len(fileErrs) > 0 { + failures = append(failures, fileErrs...) + continue + } + fmt.Fprintf(cmd.OutOrStdout(), "ok -- %s\n", file) + } + + if len(failures) > 0 { + fmt.Fprintln(cmd.OutOrStderr(), "\nSchema validation errors were encountered:") + for _, f := range failures { + fmt.Fprintf(cmd.OutOrStderr(), " %s\n", f) + } + return fmt.Errorf("schema validation failed for %d file(s)", failedFileCount(failures)) + } + + return nil +} + +// loadValidationSchema resolves and compiles the schema to validate against. An +// org-provided override at /.schemas/repository-config.schema.json +// (or an explicit --schema path) takes precedence over the built-in schema. +func loadValidationSchema(cmd *cobra.Command, configDir, schemaOverride string) (*jsonschema.Schema, error) { + compiler := jsonschema.NewCompiler() + + if schemaOverride == "" { + if candidate, ok := resolveOrgSchema(configDir); ok { + schemaOverride = candidate + } + } + + const schemaURL = "mem://repository-config.schema.json" + + var doc any + var err error + if schemaOverride != "" { + fmt.Fprintf(cmd.OutOrStdout(), "Using schema override: %s\n", schemaOverride) + f, openErr := os.Open(schemaOverride) + if openErr != nil { + return nil, fmt.Errorf("open schema override %s: %w", schemaOverride, openErr) + } + defer f.Close() + if doc, err = jsonschema.UnmarshalJSON(f); err != nil { + return nil, fmt.Errorf("parse schema override %s: %w", schemaOverride, err) + } + } else { + fmt.Fprintln(cmd.OutOrStdout(), "Using built-in schema") + raw, marshalErr := MarshalRepositoryConfigSchema() + if marshalErr != nil { + return nil, marshalErr + } + if doc, err = jsonschema.UnmarshalJSON(bytes.NewReader(raw)); err != nil { + return nil, fmt.Errorf("parse built-in schema: %w", err) + } + } + + if err := compiler.AddResource(schemaURL, doc); err != nil { + return nil, fmt.Errorf("load schema: %w", err) + } + schema, err := compiler.Compile(schemaURL) + if err != nil { + return nil, fmt.Errorf("compile schema: %w", err) + } + return schema, nil +} + +// resolveOrgSchema returns the override schema path if it exists and resolves to +// a location inside configDir, guarding against symlink escapes. +func resolveOrgSchema(configDir string) (string, bool) { + candidate := filepath.Join(configDir, orgSchemaRelPath) + if _, err := os.Stat(candidate); err != nil { + return "", false + } + + root, err := filepath.Abs(configDir) + if err != nil { + return "", false + } + if resolved, err := filepath.EvalSymlinks(root); err == nil { + root = resolved + } + + resolved, err := filepath.EvalSymlinks(candidate) + if err != nil { + return "", false + } + + rel, err := filepath.Rel(root, resolved) + if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(os.PathSeparator)) { + fmt.Fprintf(os.Stderr, "WARNING: ignoring schema override outside config directory: %s\n", candidate) + return "", false + } + return candidate, true +} + +// validateFile parses a single YAML file and validates it against the schema, +// returning one message per violation, each citing the file and JSON path. +func validateFile(path string, schema *jsonschema.Schema) []string { + data, err := os.ReadFile(path) + if err != nil { + return []string{fmt.Sprintf("%s: failed to read file: %v", path, err)} + } + + var raw any + if err := yaml.Unmarshal(data, &raw); err != nil { + return []string{fmt.Sprintf("%s: invalid YAML: %v", path, err)} + } + + instance, err := toJSONValue(raw) + if err != nil { + return []string{fmt.Sprintf("%s: %v", path, err)} + } + + err = schema.Validate(instance) + if err == nil { + return nil + } + + var ve *jsonschema.ValidationError + if !errors.As(err, &ve) { + return []string{fmt.Sprintf("%s: %v", path, err)} + } + + var messages []string + collectLeafErrors(path, ve, &messages) + if len(messages) == 0 { + // Should not happen, but never swallow a validation failure silently. + messages = append(messages, fmt.Sprintf("%s: %v", path, err)) + } + return messages +} + +// collectLeafErrors walks the validation error tree and emits one message per +// leaf failure, citing the file and the JSON path of the offending value. +func collectLeafErrors(path string, ve *jsonschema.ValidationError, out *[]string) { + if len(ve.Causes) == 0 { + loc := "/" + strings.Join(ve.InstanceLocation, "/") + if loc == "/" { + loc = "(root)" + } + *out = append(*out, fmt.Sprintf("%s: %s at %s", path, ve.ErrorKind.LocalizedString(printer), loc)) + return + } + for _, cause := range ve.Causes { + collectLeafErrors(path, cause, out) + } +} + +// toJSONValue normalizes a YAML-decoded value into JSON-compatible types so that +// schema validation follows JSON semantics (e.g. numbers, timestamps). +func toJSONValue(v any) (any, error) { + b, err := json.Marshal(v) + if err != nil { + return nil, fmt.Errorf("normalize YAML to JSON: %w", err) + } + return jsonschema.UnmarshalJSON(bytes.NewReader(b)) +} + +func failedFileCount(failures []string) int { + files := make(map[string]struct{}, len(failures)) + for _, f := range failures { + if i := strings.Index(f, ": "); i > 0 { + files[f[:i]] = struct{}{} + } + } + return len(files) +} diff --git a/feature/github-repo-importer/cmd/validate_test.go b/feature/github-repo-importer/cmd/validate_test.go new file mode 100644 index 0000000..603e0ac --- /dev/null +++ b/feature/github-repo-importer/cmd/validate_test.go @@ -0,0 +1,132 @@ +package cmd + +import ( + "bytes" + "os" + "path/filepath" + "testing" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// newConfigDir creates a temporary config directory containing the given +// repos/*.yaml files (name -> content). +func newConfigDir(t *testing.T, files map[string]string) string { + t.Helper() + dir := t.TempDir() + reposDir := filepath.Join(dir, "repos") + require.NoError(t, os.MkdirAll(reposDir, 0o755)) + for name, content := range files { + require.NoError(t, os.WriteFile(filepath.Join(reposDir, name), []byte(content), 0o644)) + } + return dir +} + +func runValidateCmd(t *testing.T, configDir, override string) (string, error) { + t.Helper() + var out bytes.Buffer + cmd := &cobra.Command{} + cmd.SetOut(&out) + cmd.SetErr(&out) + err := runValidate(cmd, configDir, override) + return out.String(), err +} + +func TestValidate_ValidConfigPasses(t *testing.T) { + dir := newConfigDir(t, map[string]string{ + "good.yaml": "default_branch: main\nvisibility: public\nhas_issues: true\n", + }) + + out, err := runValidateCmd(t, dir, "") + + assert.NoError(t, err) + assert.Contains(t, out, "Using built-in schema") + assert.Contains(t, out, "ok -- ") +} + +func TestValidate_SchemaViolationCitesFileAndPath(t *testing.T) { + dir := newConfigDir(t, map[string]string{ + "bad.yaml": "default_branch: main\nvisibility: banana\nhas_issues: nope\n", + }) + + out, err := runValidateCmd(t, dir, "") + + require.Error(t, err) + assert.Contains(t, out, filepath.Join(dir, "repos", "bad.yaml")) + assert.Contains(t, out, "/visibility") + assert.Contains(t, out, "/has_issues") +} + +func TestValidate_MissingRequiredField(t *testing.T) { + dir := newConfigDir(t, map[string]string{ + "missing.yaml": "visibility: public\n", + }) + + out, err := runValidateCmd(t, dir, "") + + require.Error(t, err) + assert.Contains(t, out, "missing property 'default_branch'") +} + +func TestValidate_MalformedYAMLReportedCleanly(t *testing.T) { + dir := newConfigDir(t, map[string]string{ + "malformed.yaml": "default_branch: main\n bad: indent\n", + }) + + out, err := runValidateCmd(t, dir, "") + + require.Error(t, err) + assert.Contains(t, out, "invalid YAML") + assert.Contains(t, out, filepath.Join(dir, "repos", "malformed.yaml")) +} + +func TestValidate_NoFilesSkips(t *testing.T) { + dir := newConfigDir(t, nil) + + out, err := runValidateCmd(t, dir, "") + + assert.NoError(t, err) + assert.Contains(t, out, "No repos/*.yaml files found") +} + +func TestValidate_OrgSchemaOverrideTakesPrecedence(t *testing.T) { + dir := newConfigDir(t, map[string]string{ + // Valid against the built-in schema, but the override only allows "internal". + "r.yaml": "default_branch: main\nvisibility: public\n", + }) + schemasDir := filepath.Join(dir, ".schemas") + require.NoError(t, os.MkdirAll(schemasDir, 0o755)) + require.NoError(t, os.WriteFile( + filepath.Join(schemasDir, "repository-config.schema.json"), + []byte(`{"type":"object","properties":{"visibility":{"enum":["internal"]}}}`), + 0o644, + )) + + out, err := runValidateCmd(t, dir, "") + + require.Error(t, err) + assert.Contains(t, out, "Using schema override") + assert.Contains(t, out, "/visibility") +} + +func TestValidate_SymlinkSchemaOverrideIsIgnored(t *testing.T) { + dir := newConfigDir(t, map[string]string{ + "r.yaml": "default_branch: main\n", + }) + + // Point .schemas at a directory outside the config dir. + outside := t.TempDir() + require.NoError(t, os.WriteFile( + filepath.Join(outside, "repository-config.schema.json"), + []byte(`{"type":"object","required":["never_present"]}`), + 0o644, + )) + require.NoError(t, os.Symlink(outside, filepath.Join(dir, ".schemas"))) + + out, err := runValidateCmd(t, dir, "") + + assert.NoError(t, err) + assert.Contains(t, out, "Using built-in schema") +} diff --git a/feature/github-repo-importer/go.mod b/feature/github-repo-importer/go.mod index 03a5c25..20807c4 100644 --- a/feature/github-repo-importer/go.mod +++ b/feature/github-repo-importer/go.mod @@ -5,11 +5,13 @@ go 1.22.3 require ( github.com/google/go-github/v67 v67.0.0 github.com/invopop/jsonschema v0.13.0 + github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 github.com/shurcooL/githubv4 v0.0.0-20240727222349-48295856cce7 github.com/spf13/cobra v1.8.1 github.com/stretchr/testify v1.11.1 github.com/wk8/go-ordered-map/v2 v2.1.8 golang.org/x/oauth2 v0.24.0 + golang.org/x/text v0.14.0 gopkg.in/yaml.v3 v3.0.1 ) diff --git a/feature/github-repo-importer/go.sum b/feature/github-repo-importer/go.sum index b6c5038..5e2eb92 100644 --- a/feature/github-repo-importer/go.sum +++ b/feature/github-repo-importer/go.sum @@ -6,6 +6,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI= +github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -29,6 +31,8 @@ github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLA github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= github.com/shurcooL/githubv4 v0.0.0-20240727222349-48295856cce7 h1:cYCy18SHPKRkvclm+pWm1Lk4YrREb4IOIb/YdFO0p2M= github.com/shurcooL/githubv4 v0.0.0-20240727222349-48295856cce7/go.mod h1:zqMwyHmnN/eDOZOdiTohqIUKUrTFX62PNlu7IJdu0q8= github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466 h1:17JxqqJY66GmZVHkmAsGEkcIu0oCe3AM420QDgGwZx0= @@ -43,6 +47,8 @@ github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/ github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE= golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= From e8ffa3c403ce53e8ca566af7be9d06870c7e5889 Mon Sep 17 00:00:00 2001 From: milosjovanovic Date: Tue, 9 Jun 2026 11:24:22 +0200 Subject: [PATCH 3/7] feat: support .yml files for schema validation in addition to .yaml Updates the validation logic to include both `.yaml` and `.yml` extensions. Refactors file globbing into a reusable function and adds a test case to ensure `.yml` files are validated correctly. --- feature/github-repo-importer/cmd/validate.go | 60 ++++++++++--------- .../github-repo-importer/cmd/validate_test.go | 14 ++++- 2 files changed, 46 insertions(+), 28 deletions(-) diff --git a/feature/github-repo-importer/cmd/validate.go b/feature/github-repo-importer/cmd/validate.go index a6f385f..222ebed 100644 --- a/feature/github-repo-importer/cmd/validate.go +++ b/feature/github-repo-importer/cmd/validate.go @@ -33,9 +33,9 @@ var ( var validateCmd = &cobra.Command{ Use: "validate", Short: "Validate repository config YAML files against the JSON schema", - Long: `Validate validates every repos/*.yaml file in the config directory against the -repository configuration JSON schema, failing fast with the offending file and -JSON path before Terraform runs. + Long: `Validate validates every repos/*.yaml and repos/*.yml file in the config +directory against the repository configuration JSON schema, failing fast with the +offending file and JSON path before Terraform runs. By default it validates against the schema built from the importer's own structs (the same one produced by the 'schema' command). If the config directory @@ -51,7 +51,7 @@ func init() { validateCmd.Flags().StringVar(&validateConfigDir, "config-dir", "", "Path to the config repository containing repos/*.yaml") validateCmd.Flags().StringVar(&validateSchemaPath, "schema", "", "Path to a schema override (defaults to /"+orgSchemaRelPath+" when present, otherwise the built-in schema)") - validateCmd.MarkFlagRequired("config-dir") + _ = validateCmd.MarkFlagRequired("config-dir") } func runValidate(cmd *cobra.Command, configDir, schemaOverride string) error { @@ -60,38 +60,54 @@ func runValidate(cmd *cobra.Command, configDir, schemaOverride string) error { return err } - files, err := filepath.Glob(filepath.Join(configDir, "repos", "*.yaml")) + files, err := globRepoConfigs(configDir) if err != nil { - return fmt.Errorf("list repos/*.yaml: %w", err) + return fmt.Errorf("list repo config files: %w", err) } sort.Strings(files) if len(files) == 0 { - fmt.Fprintln(cmd.OutOrStdout(), "No repos/*.yaml files found, skipping validation") + cmd.Println("No repos/*.yaml or repos/*.yml files found, skipping validation") return nil } var failures []string + failedFiles := 0 for _, file := range files { fileErrs := validateFile(file, schema) if len(fileErrs) > 0 { failures = append(failures, fileErrs...) + failedFiles++ continue } - fmt.Fprintf(cmd.OutOrStdout(), "ok -- %s\n", file) + cmd.Printf("ok -- %s\n", file) } if len(failures) > 0 { - fmt.Fprintln(cmd.OutOrStderr(), "\nSchema validation errors were encountered:") + cmd.PrintErrln("\nSchema validation errors were encountered:") for _, f := range failures { - fmt.Fprintf(cmd.OutOrStderr(), " %s\n", f) + cmd.PrintErrf(" %s\n", f) } - return fmt.Errorf("schema validation failed for %d file(s)", failedFileCount(failures)) + return fmt.Errorf("schema validation failed for %d file(s)", failedFiles) } return nil } +// globRepoConfigs returns the repos/*.yaml and repos/*.yml files under +// configDir, matching the extensions the importer's expand command accepts. +func globRepoConfigs(configDir string) ([]string, error) { + var files []string + for _, ext := range []string{"*.yaml", "*.yml"} { + matches, err := filepath.Glob(filepath.Join(configDir, "repos", ext)) + if err != nil { + return nil, err + } + files = append(files, matches...) + } + return files, nil +} + // loadValidationSchema resolves and compiles the schema to validate against. An // org-provided override at /.schemas/repository-config.schema.json // (or an explicit --schema path) takes precedence over the built-in schema. @@ -99,7 +115,7 @@ func loadValidationSchema(cmd *cobra.Command, configDir, schemaOverride string) compiler := jsonschema.NewCompiler() if schemaOverride == "" { - if candidate, ok := resolveOrgSchema(configDir); ok { + if candidate, ok := resolveOrgSchema(cmd, configDir); ok { schemaOverride = candidate } } @@ -109,17 +125,17 @@ func loadValidationSchema(cmd *cobra.Command, configDir, schemaOverride string) var doc any var err error if schemaOverride != "" { - fmt.Fprintf(cmd.OutOrStdout(), "Using schema override: %s\n", schemaOverride) + cmd.Printf("Using schema override: %s\n", schemaOverride) f, openErr := os.Open(schemaOverride) if openErr != nil { return nil, fmt.Errorf("open schema override %s: %w", schemaOverride, openErr) } - defer f.Close() + defer func() { _ = f.Close() }() if doc, err = jsonschema.UnmarshalJSON(f); err != nil { return nil, fmt.Errorf("parse schema override %s: %w", schemaOverride, err) } } else { - fmt.Fprintln(cmd.OutOrStdout(), "Using built-in schema") + cmd.Println("Using built-in schema") raw, marshalErr := MarshalRepositoryConfigSchema() if marshalErr != nil { return nil, marshalErr @@ -141,7 +157,7 @@ func loadValidationSchema(cmd *cobra.Command, configDir, schemaOverride string) // resolveOrgSchema returns the override schema path if it exists and resolves to // a location inside configDir, guarding against symlink escapes. -func resolveOrgSchema(configDir string) (string, bool) { +func resolveOrgSchema(cmd *cobra.Command, configDir string) (string, bool) { candidate := filepath.Join(configDir, orgSchemaRelPath) if _, err := os.Stat(candidate); err != nil { return "", false @@ -162,7 +178,7 @@ func resolveOrgSchema(configDir string) (string, bool) { rel, err := filepath.Rel(root, resolved) if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(os.PathSeparator)) { - fmt.Fprintf(os.Stderr, "WARNING: ignoring schema override outside config directory: %s\n", candidate) + cmd.PrintErrf("WARNING: ignoring schema override outside config directory: %s\n", candidate) return "", false } return candidate, true @@ -230,13 +246,3 @@ func toJSONValue(v any) (any, error) { } return jsonschema.UnmarshalJSON(bytes.NewReader(b)) } - -func failedFileCount(failures []string) int { - files := make(map[string]struct{}, len(failures)) - for _, f := range failures { - if i := strings.Index(f, ": "); i > 0 { - files[f[:i]] = struct{}{} - } - } - return len(files) -} diff --git a/feature/github-repo-importer/cmd/validate_test.go b/feature/github-repo-importer/cmd/validate_test.go index 603e0ac..2bacafc 100644 --- a/feature/github-repo-importer/cmd/validate_test.go +++ b/feature/github-repo-importer/cmd/validate_test.go @@ -59,6 +59,18 @@ func TestValidate_SchemaViolationCitesFileAndPath(t *testing.T) { assert.Contains(t, out, "/has_issues") } +func TestValidate_YmlExtensionIsValidated(t *testing.T) { + dir := newConfigDir(t, map[string]string{ + "bad.yml": "visibility: public\n", // missing required default_branch + }) + + out, err := runValidateCmd(t, dir, "") + + require.Error(t, err) + assert.Contains(t, out, filepath.Join(dir, "repos", "bad.yml")) + assert.Contains(t, out, "missing property 'default_branch'") +} + func TestValidate_MissingRequiredField(t *testing.T) { dir := newConfigDir(t, map[string]string{ "missing.yaml": "visibility: public\n", @@ -88,7 +100,7 @@ func TestValidate_NoFilesSkips(t *testing.T) { out, err := runValidateCmd(t, dir, "") assert.NoError(t, err) - assert.Contains(t, out, "No repos/*.yaml files found") + assert.Contains(t, out, "skipping validation") } func TestValidate_OrgSchemaOverrideTakesPrecedence(t *testing.T) { From d001c378bc2fccad6d845b82af8c82a4eb4578e2 Mon Sep 17 00:00:00 2001 From: milosjovanovic Date: Wed, 10 Jun 2026 12:43:07 +0200 Subject: [PATCH 4/7] feat: add fallback schema support for validation Enhances schema resolution logic to include a fallback schema option (`--fallback-schema`) when no org-level override is present. Updates command flags, adjusts resolution priority, and extends test cases to verify fallback behavior. --- .../actions/validate-repo-configs/action.yaml | 5 +- feature/github-repo-importer/cmd/validate.go | 50 +++++++++++----- .../github-repo-importer/cmd/validate_test.go | 60 ++++++++++++++++++- 3 files changed, 96 insertions(+), 19 deletions(-) diff --git a/.github/actions/validate-repo-configs/action.yaml b/.github/actions/validate-repo-configs/action.yaml index 877661e..75f1ec8 100644 --- a/.github/actions/validate-repo-configs/action.yaml +++ b/.github/actions/validate-repo-configs/action.yaml @@ -19,4 +19,7 @@ runs: - name: Validate repos/*.yaml against schema shell: bash working-directory: ${{ inputs.importer-path }} - run: go run . validate --config-dir "${{ github.workspace }}/${{ inputs.config-path }}" + run: | + go run . validate \ + --config-dir "${{ github.workspace }}/${{ inputs.config-path }}" \ + --fallback-schema "${{ github.workspace }}/.schemas/repository-config.schema.json" diff --git a/feature/github-repo-importer/cmd/validate.go b/feature/github-repo-importer/cmd/validate.go index 222ebed..0cfabc0 100644 --- a/feature/github-repo-importer/cmd/validate.go +++ b/feature/github-repo-importer/cmd/validate.go @@ -26,8 +26,9 @@ var printer = message.NewPrinter(language.English) const orgSchemaRelPath = ".schemas/repository-config.schema.json" var ( - validateConfigDir string - validateSchemaPath string + validateConfigDir string + validateSchemaPath string + validateFallbackPath string ) var validateCmd = &cobra.Command{ @@ -37,12 +38,14 @@ var validateCmd = &cobra.Command{ directory against the repository configuration JSON schema, failing fast with the offending file and JSON path before Terraform runs. -By default it validates against the schema built from the importer's own structs -(the same one produced by the 'schema' command). If the config directory -contains an override at ` + orgSchemaRelPath + `, that schema is used instead.`, +Schema resolution order: + 1. --schema flag (explicit override) + 2. /` + orgSchemaRelPath + ` (org-level override, if present) + 3. --fallback-schema flag (e.g. github-terraformer's own .schemas/ file) + 4. built-in schema generated from the importer's Go structs`, SilenceUsage: true, RunE: func(cmd *cobra.Command, args []string) error { - return runValidate(cmd, validateConfigDir, validateSchemaPath) + return runValidate(cmd, validateConfigDir, validateSchemaPath, validateFallbackPath) }, } @@ -50,12 +53,13 @@ func init() { rootCmd.AddCommand(validateCmd) validateCmd.Flags().StringVar(&validateConfigDir, "config-dir", "", "Path to the config repository containing repos/*.yaml") - validateCmd.Flags().StringVar(&validateSchemaPath, "schema", "", "Path to a schema override (defaults to /"+orgSchemaRelPath+" when present, otherwise the built-in schema)") + validateCmd.Flags().StringVar(&validateSchemaPath, "schema", "", "Explicit schema path (takes precedence over all other sources)") + validateCmd.Flags().StringVar(&validateFallbackPath, "fallback-schema", "", "Fallback schema path used when no org override exists (e.g. github-terraformer/.schemas/repository-config.schema.json)") _ = validateCmd.MarkFlagRequired("config-dir") } -func runValidate(cmd *cobra.Command, configDir, schemaOverride string) error { - schema, err := loadValidationSchema(cmd, configDir, schemaOverride) +func runValidate(cmd *cobra.Command, configDir, schemaOverride, fallbackSchema string) error { + schema, err := loadValidationSchema(cmd, configDir, schemaOverride, fallbackSchema) if err != nil { return err } @@ -108,10 +112,13 @@ func globRepoConfigs(configDir string) ([]string, error) { return files, nil } -// loadValidationSchema resolves and compiles the schema to validate against. An -// org-provided override at /.schemas/repository-config.schema.json -// (or an explicit --schema path) takes precedence over the built-in schema. -func loadValidationSchema(cmd *cobra.Command, configDir, schemaOverride string) (*jsonschema.Schema, error) { +// loadValidationSchema resolves and compiles the schema to validate against. +// Resolution order: +// 1. explicit schemaOverride (--schema flag) +// 2. org-level override at /.schemas/repository-config.schema.json +// 3. fallbackSchema (--fallback-schema flag, e.g. github-terraformer's own .schemas/ file) +// 4. built-in schema generated from the importer's Go structs +func loadValidationSchema(cmd *cobra.Command, configDir, schemaOverride, fallbackSchema string) (*jsonschema.Schema, error) { compiler := jsonschema.NewCompiler() if schemaOverride == "" { @@ -124,8 +131,9 @@ func loadValidationSchema(cmd *cobra.Command, configDir, schemaOverride string) var doc any var err error - if schemaOverride != "" { - cmd.Printf("Using schema override: %s\n", schemaOverride) + switch { + case schemaOverride != "": + cmd.Printf("Using org schema override: %s\n", schemaOverride) f, openErr := os.Open(schemaOverride) if openErr != nil { return nil, fmt.Errorf("open schema override %s: %w", schemaOverride, openErr) @@ -134,7 +142,17 @@ func loadValidationSchema(cmd *cobra.Command, configDir, schemaOverride string) if doc, err = jsonschema.UnmarshalJSON(f); err != nil { return nil, fmt.Errorf("parse schema override %s: %w", schemaOverride, err) } - } else { + case fallbackSchema != "": + cmd.Printf("Using base schema: %s\n", fallbackSchema) + f, openErr := os.Open(fallbackSchema) + if openErr != nil { + return nil, fmt.Errorf("open fallback schema %s: %w", fallbackSchema, openErr) + } + defer func() { _ = f.Close() }() + if doc, err = jsonschema.UnmarshalJSON(f); err != nil { + return nil, fmt.Errorf("parse fallback schema %s: %w", fallbackSchema, err) + } + default: cmd.Println("Using built-in schema") raw, marshalErr := MarshalRepositoryConfigSchema() if marshalErr != nil { diff --git a/feature/github-repo-importer/cmd/validate_test.go b/feature/github-repo-importer/cmd/validate_test.go index 2bacafc..835ba50 100644 --- a/feature/github-repo-importer/cmd/validate_test.go +++ b/feature/github-repo-importer/cmd/validate_test.go @@ -30,7 +30,7 @@ func runValidateCmd(t *testing.T, configDir, override string) (string, error) { cmd := &cobra.Command{} cmd.SetOut(&out) cmd.SetErr(&out) - err := runValidate(cmd, configDir, override) + err := runValidate(cmd, configDir, override, "") return out.String(), err } @@ -103,6 +103,62 @@ func TestValidate_NoFilesSkips(t *testing.T) { assert.Contains(t, out, "skipping validation") } +func TestValidate_FallbackSchemaUsedWhenNoOrgOverride(t *testing.T) { + dir := newConfigDir(t, map[string]string{ + "r.yaml": "default_branch: main\nvisibility: public\n", + }) + + // Write a fallback schema that only allows "internal" — should be used + // when no org override exists. + fallbackFile := filepath.Join(t.TempDir(), "fallback.schema.json") + require.NoError(t, os.WriteFile(fallbackFile, + []byte(`{"type":"object","properties":{"visibility":{"enum":["internal"]}}}`), + 0o644, + )) + + var out bytes.Buffer + cmd := &cobra.Command{} + cmd.SetOut(&out) + cmd.SetErr(&out) + err := runValidate(cmd, dir, "", fallbackFile) + + require.Error(t, err) + assert.Contains(t, out.String(), "Using base schema") + assert.Contains(t, out.String(), "/visibility") +} + +func TestValidate_OrgOverrideTakesPrecedenceOverFallback(t *testing.T) { + dir := newConfigDir(t, map[string]string{ + // valid against the org override (allows "public"), invalid against fallback (only "internal") + "r.yaml": "default_branch: main\nvisibility: public\n", + }) + + // Org override allows "public" + schemasDir := filepath.Join(dir, ".schemas") + require.NoError(t, os.MkdirAll(schemasDir, 0o755)) + require.NoError(t, os.WriteFile( + filepath.Join(schemasDir, "repository-config.schema.json"), + []byte(`{"type":"object","properties":{"visibility":{"enum":["public","private","internal"]}}}`), + 0o644, + )) + + // Fallback only allows "internal" + fallbackFile := filepath.Join(t.TempDir(), "fallback.schema.json") + require.NoError(t, os.WriteFile(fallbackFile, + []byte(`{"type":"object","properties":{"visibility":{"enum":["internal"]}}}`), + 0o644, + )) + + var out bytes.Buffer + cmd := &cobra.Command{} + cmd.SetOut(&out) + cmd.SetErr(&out) + err := runValidate(cmd, dir, "", fallbackFile) + + assert.NoError(t, err) + assert.Contains(t, out.String(), "Using org schema override") +} + func TestValidate_OrgSchemaOverrideTakesPrecedence(t *testing.T) { dir := newConfigDir(t, map[string]string{ // Valid against the built-in schema, but the override only allows "internal". @@ -119,7 +175,7 @@ func TestValidate_OrgSchemaOverrideTakesPrecedence(t *testing.T) { out, err := runValidateCmd(t, dir, "") require.Error(t, err) - assert.Contains(t, out, "Using schema override") + assert.Contains(t, out, "Using org schema override") assert.Contains(t, out, "/visibility") } From baef9531dd30614770128a5311e0a423ece5c013 Mon Sep 17 00:00:00 2001 From: milosjovanovic Date: Wed, 10 Jun 2026 15:25:59 +0200 Subject: [PATCH 5/7] feat: add configurable fallback schema path for validation Introduces `fallback-schema-path` input to the validation action and `base_schema_path` to the workflow, allowing customization of schema resolution. Uses fallback schema when no override is provided, defaulting to the bundled schema. --- .github/actions/validate-repo-configs/action.yaml | 8 +++++++- .github/workflows/tf-plan.yaml | 6 ++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/actions/validate-repo-configs/action.yaml b/.github/actions/validate-repo-configs/action.yaml index 75f1ec8..d2b4e89 100644 --- a/.github/actions/validate-repo-configs/action.yaml +++ b/.github/actions/validate-repo-configs/action.yaml @@ -4,6 +4,10 @@ inputs: config-path: description: "Path to the configuration repository containing repos/*.yaml files to validate" required: true + fallback-schema-path: + description: "Path to the base schema file to use when the config repository does not provide its own override. Defaults to the schema bundled with github-terraformer." + required: false + default: "" importer-path: description: "Path to the github-repo-importer Go module" required: false @@ -19,7 +23,9 @@ runs: - name: Validate repos/*.yaml against schema shell: bash working-directory: ${{ inputs.importer-path }} + env: + FALLBACK_SCHEMA: ${{ inputs.fallback-schema-path != '' && inputs.fallback-schema-path || format('{0}/.schemas/repository-config.schema.json', github.workspace) }} run: | go run . validate \ --config-dir "${{ github.workspace }}/${{ inputs.config-path }}" \ - --fallback-schema "${{ github.workspace }}/.schemas/repository-config.schema.json" + --fallback-schema "$FALLBACK_SCHEMA" diff --git a/.github/workflows/tf-plan.yaml b/.github/workflows/tf-plan.yaml index 95851d0..7628786 100644 --- a/.github/workflows/tf-plan.yaml +++ b/.github/workflows/tf-plan.yaml @@ -16,6 +16,11 @@ on: type: string description: 'The Terraform Cloud organization' required: true + base_schema_path: + type: string + description: "Path to the base JSON schema file for repos/*.yaml validation. When omitted, the schema bundled with github-terraformer is used." + required: false + default: "" secrets: app_private_key: required: true @@ -49,6 +54,7 @@ jobs: uses: ./.github/actions/validate-repo-configs with: config-path: gcss_config + fallback-schema-path: ${{ inputs.base_schema_path }} terraform-plan: needs: validate From e2f4bdaa0d6e7c840f9033cac3c481c727e9a5e5 Mon Sep 17 00:00:00 2001 From: milosjovanovic Date: Wed, 10 Jun 2026 15:46:32 +0200 Subject: [PATCH 6/7] refactor: rename base_schema_path to fallback_schema_path in tf-plan workflow --- .github/workflows/tf-plan.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tf-plan.yaml b/.github/workflows/tf-plan.yaml index 7628786..3d53700 100644 --- a/.github/workflows/tf-plan.yaml +++ b/.github/workflows/tf-plan.yaml @@ -16,9 +16,9 @@ on: type: string description: 'The Terraform Cloud organization' required: true - base_schema_path: + fallback_schema_path: type: string - description: "Path to the base JSON schema file for repos/*.yaml validation. When omitted, the schema bundled with github-terraformer is used." + description: "Path to the fallback JSON schema for repos/*.yaml validation, used when the config repo provides no override. When omitted, the schema bundled with github-terraformer is used." required: false default: "" secrets: @@ -54,7 +54,7 @@ jobs: uses: ./.github/actions/validate-repo-configs with: config-path: gcss_config - fallback-schema-path: ${{ inputs.base_schema_path }} + fallback-schema-path: ${{ inputs.fallback_schema_path }} terraform-plan: needs: validate From cba3850864ae31e2e978b46e7f9ba6eaf2ddb12e Mon Sep 17 00:00:00 2001 From: milosjovanovic Date: Wed, 10 Jun 2026 16:12:07 +0200 Subject: [PATCH 7/7] refactor: rename base_schema_path to fallback_schema_path in tf-plan workflow --- .github/actions/validate-repo-configs/action.yaml | 6 +++--- .github/workflows/tf-plan.yaml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/actions/validate-repo-configs/action.yaml b/.github/actions/validate-repo-configs/action.yaml index d2b4e89..fc9637d 100644 --- a/.github/actions/validate-repo-configs/action.yaml +++ b/.github/actions/validate-repo-configs/action.yaml @@ -5,9 +5,9 @@ inputs: description: "Path to the configuration repository containing repos/*.yaml files to validate" required: true fallback-schema-path: - description: "Path to the base schema file to use when the config repository does not provide its own override. Defaults to the schema bundled with github-terraformer." + description: "Workspace-relative path to the fallback schema, used when the config repository provides no override. Resolved against the runner workspace (e.g. 'gcss_config/.schemas/my.schema.json'). Defaults to the schema bundled with github-terraformer." required: false - default: "" + default: ".schemas/repository-config.schema.json" importer-path: description: "Path to the github-repo-importer Go module" required: false @@ -24,7 +24,7 @@ runs: shell: bash working-directory: ${{ inputs.importer-path }} env: - FALLBACK_SCHEMA: ${{ inputs.fallback-schema-path != '' && inputs.fallback-schema-path || format('{0}/.schemas/repository-config.schema.json', github.workspace) }} + FALLBACK_SCHEMA: ${{ format('{0}/{1}', github.workspace, inputs.fallback-schema-path != '' && inputs.fallback-schema-path || '.schemas/repository-config.schema.json') }} run: | go run . validate \ --config-dir "${{ github.workspace }}/${{ inputs.config-path }}" \ diff --git a/.github/workflows/tf-plan.yaml b/.github/workflows/tf-plan.yaml index 3d53700..8e70f08 100644 --- a/.github/workflows/tf-plan.yaml +++ b/.github/workflows/tf-plan.yaml @@ -18,7 +18,7 @@ on: required: true fallback_schema_path: type: string - description: "Path to the fallback JSON schema for repos/*.yaml validation, used when the config repo provides no override. When omitted, the schema bundled with github-terraformer is used." + description: "Workspace-relative path to the fallback JSON schema for repos/*.yaml validation, used when the config repo provides no override (e.g. 'gcss_config/.schemas/my.schema.json'). When omitted, the schema bundled with github-terraformer is used." required: false default: "" secrets: