diff --git a/CLAUDE.md b/CLAUDE.md index 2e47ad8..83566b7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -14,8 +14,11 @@ Expression Language (CEL) expressions. - `skills/`: Agent skills directory containing guidelines and workflows: - `cel-authoring`: Authoring, configuring, and testing expressions and JSON environment definitions. + - `cel-testing`: Testing and validating expressions against test cases. - `cel-debugging`: Diagnosing and resolving compilation and evaluation errors. + - `cel-conformance-testing`: Authoring, evaluating, and publishing CEL + conformance tests using the cel-spec textproto format. ## Build and Test Commands @@ -43,6 +46,8 @@ The `cel-expr-mcp` MCP server provides the following tools for AI agents: - `cel_compile`: Compiles a CEL expression to validate syntax, correctness, and type checking against an environment definition. - `cel_evaluate`: Evaluates a compiled expression against provided test cases. +- `cel_evaluate_conformance`: Evaluates CEL conformance tests from a cel-spec + textproto file or inline content. ### Startup Flags @@ -60,8 +65,11 @@ To understand how to best use these tools, refer to the skills in `skills/`: - `cel-authoring`: Guidelines for creating CEL environment configurations, generating expressions, and compiling. +- `cel-testing`: Writing and evaluating unit test suites for CEL expressions. - `cel-debugging`: Diagnostic steps for troubleshooting syntax, type, and evaluation errors. +- `cel-conformance-testing`: Executing and authoring conformance test suites from + `cel-spec` textproto definitions. ## Release & Supply Chain Security diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 697d7cb..dff165c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -63,4 +63,4 @@ asking for a new review. There is no commitment to resolution -- merging or closing a pull request, or fixing or closing an issue -- because some issues will require more discussion than others. -[1]: https://github.com/google/cel-spec +[1]: https://github.com/cel-expr/cel-spec diff --git a/MODULE.bazel b/MODULE.bazel index 39a40e5..704a8bb 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -20,10 +20,15 @@ go_deps.gazelle_default_attributes( "gazelle:proto disable_global", ], ) +go_deps.gazelle_override( + build_file_generation = "off", + path = "cel.dev/expr", +) go_deps.from_file(go_mod = "//:go.mod") use_repo( go_deps, "com_github_modelcontextprotocol_go_sdk", "dev_cel_cel_go", + "dev_cel_expr", "org_golang_google_protobuf", ) diff --git a/cmd/cel-expr/main.go b/cmd/cel-expr/main.go index 4f098b2..0b09300 100644 --- a/cmd/cel-expr/main.go +++ b/cmd/cel-expr/main.go @@ -39,6 +39,7 @@ Usage: Available Commands: compile Compile a CEL expression against an environment configuration eval Evaluate a CEL expression against test cases or bindings + conformance Evaluate CEL conformance tests from cel-spec textproto files env Validate or inspect an environment configuration prompt Generate an LLM authoring prompt from an environment and requirement mcp Start the Model Context Protocol (MCP) server over stdio @@ -68,6 +69,8 @@ func run(args []string, stdout, stderr io.Writer, stdin io.Reader) error { return runCompile(cmdArgs, stdout, stderr, stdin) case "eval", "evaluate": return runEval(cmdArgs, stdout, stderr, stdin) + case "conformance": + return runConformance(cmdArgs, stdout, stderr, stdin) case "env": return runEnv(cmdArgs, stdout, stderr, stdin) case "prompt": @@ -189,11 +192,12 @@ func runEval(args []string, stdout, stderr io.Writer, stdin io.Reader) error { exprFlag := fs.String("expr", "", "CEL expression string") testsFlag := fs.String("tests", "", "Path to test cases JSON file or inline JSON array") fs.StringVar(testsFlag, "test_cases", "", "Alias for -tests") + conformanceFlag := fs.String("conformance", "", "Path to conformance textproto file or inline textproto content") bindingsFlag := fs.String("bindings", "", "Single test case variable bindings JSON file or inline JSON object") expectedFlag := fs.String("expected", "", "Expected output value for single test case binding (JSON literal or string)") fs.Usage = func() { - fmt.Fprintln(stderr, "Usage: cel-expr eval [-env ] [-tests ] [-bindings ] [expression]") + fmt.Fprintln(stderr, "Usage: cel-expr eval [-env ] [-tests ] [-conformance ] [-bindings ] [expression]") fmt.Fprintln(stderr, "\nFlags:") fs.PrintDefaults() } @@ -209,6 +213,21 @@ func runEval(args []string, stdout, stderr io.Writer, stdin io.Reader) error { if expr == "" && fs.NArg() > 0 { expr = strings.Join(fs.Args(), " ") } + + if *conformanceFlag != "" { + conformanceArgs := []string{"-tests", *conformanceFlag} + if *envFlag != "" { + conformanceArgs = append(conformanceArgs, "-env", *envFlag) + } + if *fdsFlag != "" { + conformanceArgs = append(conformanceArgs, "-fds", *fdsFlag) + } + if expr != "" { + conformanceArgs = append(conformanceArgs, "-expr", expr) + } + return runConformance(conformanceArgs, stdout, stderr, stdin) + } + if expr == "" { return errors.New("expression is required (use -expr or pass as argument)") } @@ -293,6 +312,86 @@ func runEval(args []string, stdout, stderr io.Writer, stdin io.Reader) error { return nil } +func runConformance(args []string, stdout, stderr io.Writer, stdin io.Reader) error { + fs := flag.NewFlagSet("conformance", flag.ContinueOnError) + fs.SetOutput(stderr) + envFlag, fdsFlag := parseCommonEnvFlags(fs) + testsFlag := fs.String("tests", "", "Path to conformance textproto file or inline textproto content") + filterFlag := fs.String("filter", "", "Optional substring filter for test names") + skipFlag := fs.String("skip", "", "Comma-separated test prefixes to skip") + exprFlag := fs.String("expr", "", "Optional fallback expression if test does not define one") + + fs.Usage = func() { + fmt.Fprintln(stderr, "Usage: cel-expr conformance [-tests ] [-filter ] [-skip ] [-env ] [-fds ] [tests]") + fmt.Fprintln(stderr, "\nFlags:") + fs.PrintDefaults() + } + + if err := fs.Parse(args); err != nil { + if errors.Is(err, flag.ErrHelp) { + return nil + } + return err + } + + testsVal := *testsFlag + if testsVal == "" && fs.NArg() > 0 { + testsVal = strings.Join(fs.Args(), " ") + } + if testsVal == "" { + return errors.New("conformance tests are required (use -tests or pass as argument)") + } + + rawTests, err := loadContentOrStdin(testsVal, stdin) + if err != nil { + return fmt.Errorf("reading conformance tests: %w", err) + } + + var cfg *tools.Config + var opts []cel.EnvOption + if *envFlag != "" || *fdsFlag != "" { + var err error + cfg, opts, err = loadEnvAndOpts(*envFlag, *fdsFlag, stdin, false) + if err != nil { + return err + } + } + + var skipTests []string + if *skipFlag != "" { + for _, s := range strings.Split(*skipFlag, ",") { + s = strings.TrimSpace(s) + if s != "" { + skipTests = append(skipTests, s) + } + } + } + + res, err := tools.EvaluateConformanceWithParams(tools.ConformanceParams{ + Tests: rawTests, + Filter: *filterFlag, + SkipTests: skipTests, + EnvConfig: cfg, + FallbackExpr: *exprFlag, + }, opts...) + if err != nil { + return fmt.Errorf("conformance evaluation failed: %w", err) + } + + outBytes, err := json.MarshalIndent(res, "", " ") + if err != nil { + return fmt.Errorf("failed formatting output JSON: %w", err) + } + + fmt.Fprintln(stdout, string(outBytes)) + + if res.HasFailures() { + return fmt.Errorf("%d of %d conformance tests failed", res.Failed, res.Total) + } + + return nil +} + func runEnv(args []string, stdout, stderr io.Writer, stdin io.Reader) error { fs := flag.NewFlagSet("env", flag.ContinueOnError) fs.SetOutput(stderr) diff --git a/cmd/cel-expr/main_test.go b/cmd/cel-expr/main_test.go index bd8c2ba..6652009 100644 --- a/cmd/cel-expr/main_test.go +++ b/cmd/cel-expr/main_test.go @@ -41,6 +41,7 @@ func TestCLI_UsageAndHelp(t *testing.T) { {"eval help", []string{"help", "eval"}}, {"env help", []string{"help", "env"}}, {"prompt help", []string{"help", "prompt"}}, + {"conformance help", []string{"help", "conformance"}}, {"mcp help", []string{"help", "mcp"}}, } @@ -342,3 +343,79 @@ func TestCLI_MCP_Errors(t *testing.T) { t.Error("expected error when starting MCP server with invalid env, got nil") } } + +func TestCLI_Conformance(t *testing.T) { + textprotoContent := ` +name: "cli_conformance" +section { + name: "s1" + test { + name: "t1" + expr: "1 + 2" + value { int64_value: 3 } + } + test { + name: "t2" + expr: "3 * 4" + value { int64_value: 12 } + } +} +` + tmpFile := filepath.Join(t.TempDir(), "tests.textproto") + if err := os.WriteFile(tmpFile, []byte(textprotoContent), 0644); err != nil { + t.Fatal(err) + } + + tests := []struct { + name string + args []string + wantErr bool + }{ + { + name: "conformance with -tests flag", + args: []string{"conformance", "-tests", tmpFile}, + wantErr: false, + }, + { + name: "conformance with positional argument", + args: []string{"conformance", tmpFile}, + wantErr: false, + }, + { + name: "conformance with -filter flag", + args: []string{"conformance", "-tests", tmpFile, "-filter", "t1"}, + wantErr: false, + }, + { + name: "eval with -conformance flag alias", + args: []string{"eval", "-conformance", tmpFile}, + wantErr: false, + }, + { + name: "eval with -conformance and positional expr", + args: []string{"eval", "-conformance", tmpFile, "1 + 2"}, + wantErr: false, + }, + { + name: "conformance missing tests", + args: []string{"conformance"}, + wantErr: true, + }, + { + name: "conformance failing test", + args: []string{"conformance", `name:"fail" section { name:"s" test { name:"t" expr:"1" value { int64_value: 2 } } }`}, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var stdout, stderr bytes.Buffer + err := run(tt.args, &stdout, &stderr, nil) + if (err != nil) != tt.wantErr { + t.Errorf("run(%v) error = %v, wantErr %v (stdout: %s, stderr: %s)", tt.args, err, tt.wantErr, stdout.String(), stderr.String()) + } + }) + } +} + diff --git a/go.mod b/go.mod index 077cae0..8f82739 100644 --- a/go.mod +++ b/go.mod @@ -4,12 +4,12 @@ go 1.25.0 require ( cel.dev/cel-go v0.32.0 + cel.dev/expr v0.25.3 github.com/modelcontextprotocol/go-sdk v1.6.0 google.golang.org/protobuf v1.36.10 ) require ( - cel.dev/expr v0.25.1 // indirect github.com/antlr4-go/antlr/v4 v4.13.1 // indirect github.com/google/jsonschema-go v0.4.3 // indirect github.com/segmentio/asm v1.1.3 // indirect diff --git a/go.sum b/go.sum index 3c6c23d..45d5f92 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ cel.dev/cel-go v0.32.0 h1:irvpFKr5EuGPyxeME03ERh0rii1TX+BDAnB9eL3IvNk= cel.dev/cel-go v0.32.0/go.mod h1:DnVip7tpJSsgZymwfT+m1tnEVy3ivAjSMXPx12YrMkU= -cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4= -cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +cel.dev/expr v0.25.3 h1:A2jO8jwOugrrovveCWfj0KEZOfqiLgAcwjpHPhzIGw0= +cel.dev/expr v0.25.3/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= diff --git a/internal/mcp/server.go b/internal/mcp/server.go index 6cb72dc..cf5a43e 100644 --- a/internal/mcp/server.go +++ b/internal/mcp/server.go @@ -115,6 +115,10 @@ func NewServer(fixedEnv *tools.Config, opts ...cel.EnvOption) *mcp.Server { Name: "cel_evaluate", Description: "Evaluates a CEL expression against provided test cases. Returns test case results and coverage.", }, h.handleFixedEnvEvaluate) + mcp.AddTool[FixedEnvEvaluateConformanceArgs, any](s, &mcp.Tool{ + Name: "cel_evaluate_conformance", + Description: "Evaluates CEL conformance tests from a cel-spec textproto file or inline content. Returns test case results and status.", + }, h.handleFixedEnvEvaluateConformance) } else { mcp.AddTool[CreateEnvConfigArgs, any](s, &mcp.Tool{ Name: "cel_create_environment", @@ -132,6 +136,10 @@ func NewServer(fixedEnv *tools.Config, opts ...cel.EnvOption) *mcp.Server { Name: "cel_evaluate", Description: "Evaluates a CEL expression against provided test cases. Returns test case results and coverage.", }, h.handleEvaluate) + mcp.AddTool[EvaluateConformanceArgs, any](s, &mcp.Tool{ + Name: "cel_evaluate_conformance", + Description: "Evaluates CEL conformance tests from a cel-spec textproto file or inline content. Returns test case results and status.", + }, h.handleEvaluateConformance) } return s @@ -177,6 +185,23 @@ type FixedEnvEvaluateArgs struct { TestCases []tools.TestCase `json:"testCases" jsonschema_description:"The test cases for evaluation."` } +// EvaluateConformanceArgs is the arguments for the cel_evaluate_conformance tool. +type EvaluateConformanceArgs struct { + Tests string `json:"tests" jsonschema_description:"Path to conformance textproto file or inline textproto content."` + Filter string `json:"filter,omitempty" jsonschema_description:"Optional substring filter for test names."` + SkipTests []string `json:"skipTests,omitempty" jsonschema_description:"Optional list of test name prefixes to skip."` + EnvConfig *tools.Config `json:"envConfig,omitempty" jsonschema_description:"Optional JSON environment configuration."` + FallbackExpr string `json:"fallbackExpr,omitempty" jsonschema_description:"Optional fallback CEL expression."` +} + +// FixedEnvEvaluateConformanceArgs is the arguments for the cel_evaluate_conformance tool when a static environment is configured. +type FixedEnvEvaluateConformanceArgs struct { + Tests string `json:"tests" jsonschema_description:"Path to conformance textproto file or inline textproto content."` + Filter string `json:"filter,omitempty" jsonschema_description:"Optional substring filter for test names."` + SkipTests []string `json:"skipTests,omitempty" jsonschema_description:"Optional list of test name prefixes to skip."` + FallbackExpr string `json:"fallbackExpr,omitempty" jsonschema_description:"Optional fallback CEL expression."` +} + func (h *toolsHandler) handleCreateEnvConfig(ctx context.Context, request *mcp.CallToolRequest, args CreateEnvConfigArgs) (*mcp.CallToolResult, any, error) { _, err := tools.EnvFromConfig(args.EnvConfig, h.opts...) if err != nil { @@ -247,3 +272,31 @@ func (h *toolsHandler) handleFixedEnvEvaluate(ctx context.Context, request *mcp. TestCases: args.TestCases, }) } + +func (h *toolsHandler) handleEvaluateConformance(ctx context.Context, request *mcp.CallToolRequest, args EvaluateConformanceArgs) (*mcp.CallToolResult, any, error) { + envConfig := args.EnvConfig + if envConfig == nil { + envConfig = h.fixedEnv + } + res, err := tools.EvaluateConformanceWithParams(tools.ConformanceParams{ + Tests: args.Tests, + Filter: args.Filter, + SkipTests: args.SkipTests, + EnvConfig: envConfig, + FallbackExpr: args.FallbackExpr, + }, h.opts...) + if err != nil { + return nil, nil, err + } + return nil, res, nil +} + +func (h *toolsHandler) handleFixedEnvEvaluateConformance(ctx context.Context, request *mcp.CallToolRequest, args FixedEnvEvaluateConformanceArgs) (*mcp.CallToolResult, any, error) { + return h.handleEvaluateConformance(ctx, request, EvaluateConformanceArgs{ + Tests: args.Tests, + Filter: args.Filter, + SkipTests: args.SkipTests, + FallbackExpr: args.FallbackExpr, + }) +} + diff --git a/internal/mcp/server_test.go b/internal/mcp/server_test.go index 32de53f..cccfcb7 100644 --- a/internal/mcp/server_test.go +++ b/internal/mcp/server_test.go @@ -48,10 +48,11 @@ func TestListTools(t *testing.T) { resp := session.Tools(ctx, nil) expectedTools := map[string]bool{ - "cel_compile": true, - "cel_evaluate": true, - "cel_generate_prompt": true, - "cel_create_environment": true, + "cel_compile": true, + "cel_evaluate": true, + "cel_evaluate_conformance": true, + "cel_generate_prompt": true, + "cel_create_environment": true, } foundTools := make(map[string]bool) @@ -104,7 +105,7 @@ func TestListTools_WithFixedEnvironment(t *testing.T) { t.Error("cel_create_environment should NOT be exposed when a fixed environment is present") } - expectedTools := []string{"cel_compile", "cel_evaluate", "cel_generate_prompt"} + expectedTools := []string{"cel_compile", "cel_evaluate", "cel_evaluate_conformance", "cel_generate_prompt"} for _, name := range expectedTools { if !foundTools[name] { t.Errorf("expected tool %s not found in ListTools response with fixed environment", name) @@ -181,6 +182,34 @@ func TestFixedEnvironmentToolsExecution(t *testing.T) { if promptRes.IsError { t.Errorf("cel_generate_prompt failed: %v", promptRes.Content) } + + // 4. Test cel_evaluate_conformance without envConfig + conformanceRes, err := session.CallTool(ctx, &mcp.CallToolParams{ + Name: "cel_evaluate_conformance", + Arguments: map[string]any{ + "tests": ` +name: "fixed_env_suite" +section { + name: "s" + test { + name: "t" + expr: "foo == 'bar'" + bindings { + key: "foo" + value { value { string_value: "bar" } } + } + value { bool_value: true } + } +} +`, + }, + }) + if err != nil { + t.Fatalf("CallTool cel_evaluate_conformance failed: %v", err) + } + if conformanceRes.IsError { + t.Errorf("cel_evaluate_conformance failed: %v", conformanceRes.Content) + } } func TestLoadEnvConfig(t *testing.T) { @@ -390,6 +419,71 @@ func TestHandleGeneratePrompt(t *testing.T) { } } +func TestHandleEvaluateConformance(t *testing.T) { + ctx := context.Background() + h := &toolsHandler{} + + args := EvaluateConformanceArgs{ + Tests: ` +name: "mcp_conformance" +section { + name: "s" + test { + name: "t" + expr: "10 + 20" + value { int64_value: 30 } + } +} +`, + } + + _, rawResult, err := h.handleEvaluateConformance(ctx, &mcp.CallToolRequest{}, args) + if err != nil { + t.Fatalf("handleEvaluateConformance failed: %v", err) + } + + out, ok := rawResult.(*tools.ConformanceOutput) + if !ok { + t.Fatalf("expected *tools.ConformanceOutput, got %T", rawResult) + } + if out.Passed != 1 || out.Total != 1 { + t.Errorf("expected 1 passed test, got passed=%d, total=%d", out.Passed, out.Total) + } + + // Test with SkipTests + argsSkip := EvaluateConformanceArgs{ + Tests: ` +name: "mcp_conformance" +section { + name: "s" + test { + name: "t" + expr: "10 + 20" + value { int64_value: 30 } + } +} +`, + SkipTests: []string{"mcp_conformance/s/t"}, + } + _, rawResult, err = h.handleEvaluateConformance(ctx, &mcp.CallToolRequest{}, argsSkip) + if err != nil { + t.Fatalf("handleEvaluateConformance with SkipTests failed: %v", err) + } + out = rawResult.(*tools.ConformanceOutput) + if out.Skipped != 1 { + t.Errorf("expected 1 skipped test, got %d", out.Skipped) + } + + // Test error case (invalid syntax) + argsErr := EvaluateConformanceArgs{ + Tests: "invalid textproto content", + } + _, _, err = h.handleEvaluateConformance(ctx, &mcp.CallToolRequest{}, argsErr) + if err == nil { + t.Error("expected error for invalid conformance content, got nil") + } +} + func TestSetupServer(t *testing.T) { tmpDir := t.TempDir() envPath := filepath.Join(tmpDir, "env.json") diff --git a/internal/tools/BUILD b/internal/tools/BUILD index 30f7ea4..780d704 100644 --- a/internal/tools/BUILD +++ b/internal/tools/BUILD @@ -11,6 +11,7 @@ go_library( srcs = [ "compile.go", "config.go", + "conformance.go", "coverage.go", "evaluate.go", "input_schema.go", @@ -18,12 +19,18 @@ go_library( ], deps = [ "@dev_cel_cel_go//cel", + "@dev_cel_cel_go//common", "@dev_cel_cel_go//common/ast", "@dev_cel_cel_go//common/env", "@dev_cel_cel_go//common/types", "@dev_cel_cel_go//common/types/ref", "@dev_cel_cel_go//ext", + "@dev_cel_expr//:expr", + "@dev_cel_expr//conformance/proto2:go_default_library", + "@dev_cel_expr//conformance/proto3:go_default_library", + "@dev_cel_expr//conformance/test:go_default_library", "@org_golang_google_protobuf//encoding/protojson", + "@org_golang_google_protobuf//encoding/prototext", "@org_golang_google_protobuf//proto", ], ) @@ -33,6 +40,7 @@ go_test( srcs = [ "compile_test.go", "config_test.go", + "conformance_test.go", "coverage_test.go", "evaluate_test.go", "input_schema_test.go", @@ -41,10 +49,14 @@ go_test( data = glob(["testdata/**"]), embed = [":tools"], deps = [ + "//internal/proto:test_schema_go_proto", "@dev_cel_cel_go//cel", "@dev_cel_cel_go//common/ast", "@dev_cel_cel_go//common/env", "@dev_cel_cel_go//common/types", - "//internal/proto:test_schema_go_proto", + "@dev_cel_expr//:expr", + "@dev_cel_expr//conformance/test:go_default_library", + "@org_golang_google_protobuf//proto", + "@org_golang_google_protobuf//types/known/anypb", ], ) diff --git a/internal/tools/conformance.go b/internal/tools/conformance.go new file mode 100644 index 0000000..14f9d84 --- /dev/null +++ b/internal/tools/conformance.go @@ -0,0 +1,1026 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tools + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "math" + "os" + "strings" + + "cel.dev/cel-go/cel" + "cel.dev/cel-go/common" + "cel.dev/cel-go/common/ast" + "cel.dev/cel-go/common/types" + "cel.dev/cel-go/common/types/ref" + "cel.dev/cel-go/ext" + expr "cel.dev/expr" + test2pb "cel.dev/expr/conformance/proto2" + test3pb "cel.dev/expr/conformance/proto3" + testpb "cel.dev/expr/conformance/test" + "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/encoding/prototext" + "google.golang.org/protobuf/proto" +) + +// ConformanceTestResult represents the result of evaluating a single conformance test case. +type ConformanceTestResult struct { + Name string `json:"name"` + Status string `json:"status"` // "pass" or detailed failure message + Section string `json:"section,omitempty"` + File string `json:"file,omitempty"` + Expr string `json:"expr,omitempty"` + Description string `json:"description,omitempty"` +} + +// Passed returns true if the test case passed. +func (r ConformanceTestResult) Passed() bool { + return r.Status == "pass" +} + +// ConformanceOutput represents the aggregated outcome of a conformance test suite evaluation. +type ConformanceOutput struct { + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + Total int `json:"total"` + Passed int `json:"passed"` + Failed int `json:"failed"` + Skipped int `json:"skipped"` + Results []ConformanceTestResult `json:"results"` + Coverage string `json:"coverage,omitempty"` +} + +// HasFailures returns true if one or more tests failed. +func (o *ConformanceOutput) HasFailures() bool { + return o.Failed > 0 +} + +// ConformanceParams configures the conformance test evaluator. +type ConformanceParams struct { + Tests string `json:"tests"` // Path to textproto file or inline textproto content + Filter string `json:"filter,omitempty"` // Optional substring filter for test names + SkipTests []string `json:"skipTests,omitempty"` // Optional test prefixes/names to skip + EnvConfig *Config `json:"envConfig,omitempty"` // Optional environment configuration + FallbackExpr string `json:"fallbackExpr,omitempty"` // Fallback expression if test does not define one +} + +// DefaultConformanceSkipTests contains the standard set of conformance test prefixes skipped by cel-go. +// These correspond to tests that require future specification enhancements, pending spec updates, +// or known cel-go platform limitations (derived from cel-expr/cel-spec/tests/simple/BUILD.bazel). +var defaultConformanceSkipTests = []string{ + // Failing conformance tests in cel-go + "fields/qualified_identifier_resolution/map_key_float", + "fields/qualified_identifier_resolution/map_key_null", + "fields/qualified_identifier_resolution/map_value_repeat_key", + "fields/qualified_identifier_resolution/map_value_repeat_key_heterogeneous", + "timestamps/duration_converters/get_milliseconds", + "optionals/optionals/map_optional_select_has", + + // Temporarily failing tests, need a spec update + "string_ext/value_errors/indexof_out_of_range", + "string_ext/value_errors/lastindexof_out_of_range", + + // Future enhancements + "enums/strong_proto2", + "enums/strong_proto3", + + // Type deductions + "type_deductions/wrappers/wrapper_promotion_2", + "type_deductions/legacy_nullable_types/null_assignable_to_message_parameter_candidate", + "type_deductions/legacy_nullable_types/null_assignable_to_duration_parameter_candidate", + "type_deductions/legacy_nullable_types/null_assignable_to_timestamp_parameter_candidate", + "type_deductions/legacy_nullable_types/null_assignable_to_abstract_parameter_candidate", +} + +// DefaultConformanceSkipTests returns a copy of the standard set of conformance test prefixes skipped by cel-go. +// These correspond to tests that require future specification enhancements, pending spec updates, +// or known cel-go platform limitations (derived from cel-expr/cel-spec/tests/simple/BUILD.bazel). +func DefaultConformanceSkipTests() []string { + cp := make([]string, len(defaultConformanceSkipTests)) + copy(cp, defaultConformanceSkipTests) + return cp +} + +// ConformanceEnvOptions returns the standard CEL environment options required for CEL conformance testing. +func ConformanceEnvOptions() []cel.EnvOption { + return []cel.EnvOption{ + cel.StdLib(), + cel.ClearMacros(), + cel.OptionalTypes(), + cel.EagerlyValidateDeclarations(true), + cel.EnableErrorOnBadPresenceTest(true), + cel.Types( + &test2pb.TestAllTypes{}, + &test2pb.Proto2ExtensionScopedMessage{}, + &test3pb.TestAllTypes{}, + ), + ext.Bindings(), + ext.Encoders(), + ext.Lists(), + ext.Math(), + ext.Protos(), + ext.Strings(), + ext.TwoVarComprehensions(), + cel.Lib(celBlockLib{}), + cel.EnableIdentifierEscapeSyntax(), + } +} + +// buildConformanceEnvs constructs the custom CEL environments with and without macros. +func buildConformanceEnvs(cfg *Config, opts ...cel.EnvOption) (*cel.Env, *cel.Env, error) { + baseOpts := ConformanceEnvOptions() + if cfg != nil { + celConfig, err := cfg.ToCELConfig() + if err != nil { + return nil, nil, fmt.Errorf("failed constructing env from config: %w", err) + } + baseOpts = append(baseOpts, cel.FromConfig(celConfig, ext.ExtensionOptionFactory)) + } + baseOpts = append(baseOpts, opts...) + + envNoMacros, err := cel.NewCustomEnv(baseOpts...) + if err != nil { + return nil, nil, fmt.Errorf("failed constructing env (no macros): %w", err) + } + envWithMacros, err := envNoMacros.Extend(cel.Macros(cel.StandardMacros...)) + if err != nil { + return nil, nil, fmt.Errorf("failed constructing env (with macros): %w", err) + } + return envNoMacros, envWithMacros, nil +} + +func formatTestName(file, section, test string) string { + var parts []string + if file != "" { + parts = append(parts, file) + } + if section != "" { + parts = append(parts, section) + } + parts = append(parts, test) + return strings.Join(parts, "/") +} + +// EvaluateConformance evaluates CEL conformance tests from a textproto file path or inline textproto string. +func EvaluateConformance(tests string, envConfig *Config, opts ...cel.EnvOption) (*ConformanceOutput, error) { + return EvaluateConformanceWithParams(ConformanceParams{ + Tests: tests, + EnvConfig: envConfig, + }, opts...) +} + +// EvaluateConformanceWithParams evaluates CEL conformance tests with full parameter control. +func EvaluateConformanceWithParams(params ConformanceParams, opts ...cel.EnvOption) (*ConformanceOutput, error) { + file, err := ParseConformanceFile(params.Tests) + if err != nil { + return nil, fmt.Errorf("failed parsing conformance tests: %w", err) + } + + envNoMacros, envWithMacros, err := buildConformanceEnvs(params.EnvConfig, opts...) + if err != nil { + return nil, err + } + + out := &ConformanceOutput{ + Name: file.GetName(), + Description: file.GetDescription(), + } + + shouldSkip := func(name string) bool { + for _, s := range params.SkipTests { + if strings.HasPrefix(name, s) { + rem := name[len(s):] + if rem == "" || strings.HasPrefix(rem, "/") { + return true + } + } + } + return false + } + + for _, section := range file.GetSection() { + for _, test := range section.GetTest() { + fullName := formatTestName(file.GetName(), section.GetName(), test.GetName()) + + if params.Filter != "" && !strings.Contains(fullName, params.Filter) { + continue + } + + out.Total++ + + if shouldSkip(fullName) { + out.Skipped++ + out.Results = append(out.Results, ConformanceTestResult{ + Name: fullName, + Section: section.GetName(), + File: file.GetName(), + Expr: test.GetExpr(), + Description: test.GetDescription(), + Status: "skipped", + }) + continue + } + + exprStr := test.GetExpr() + if exprStr == "" { + exprStr = params.FallbackExpr + } + + res := evaluateSingleConformanceTest(test, exprStr, fullName, section.GetName(), file.GetName(), envNoMacros, envWithMacros) + if res.Passed() { + out.Passed++ + } else { + out.Failed++ + } + out.Results = append(out.Results, res) + } + } + + return out, nil +} + +// EvaluateConformanceTest evaluates an individual SimpleTest protobuf message. +func EvaluateConformanceTest(test *testpb.SimpleTest, envConfig *Config, opts ...cel.EnvOption) (*ConformanceTestResult, error) { + if test == nil { + return nil, errors.New("nil SimpleTest") + } + envNoMacros, envWithMacros, err := buildConformanceEnvs(envConfig, opts...) + if err != nil { + return nil, err + } + + res := evaluateSingleConformanceTest(test, test.GetExpr(), test.GetName(), "", "", envNoMacros, envWithMacros) + return &res, nil +} + +func checkDeducedType(want *expr.Type, outputType *cel.Type) string { + outType, err := types.TypeToProto(outputType) + if err != nil { + return fmt.Sprintf("failed converting output type: %v", err) + } + if !equalTypes(want, outType) { + return fmt.Sprintf("type mismatch: want %v, got %v", want, outType) + } + return "" +} + +func evaluateSingleConformanceTest(test *testpb.SimpleTest, exprStr, fullName, sectionName, fileName string, envNoMacros, envWithMacros *cel.Env) ConformanceTestResult { + res := ConformanceTestResult{ + Name: fullName, + Section: sectionName, + File: fileName, + Expr: exprStr, + Description: test.GetDescription(), + } + + var baseEnv *cel.Env + if test.GetDisableMacros() { + baseEnv = envNoMacros + } else { + baseEnv = envWithMacros + } + + src := common.NewStringSource(exprStr, fullName) + ast, iss := baseEnv.ParseSource(src) + if iss.Err() != nil { + res.Status = fmt.Sprintf("parse error: %v", iss.Err()) + return res + } + + var testOpts []cel.EnvOption + if test.GetContainer() != "" { + testOpts = append(testOpts, cel.Container(test.GetContainer())) + } + for _, d := range test.GetTypeEnv() { + opt, err := cel.ProtoAsDeclaration(d) + if err != nil { + res.Status = fmt.Sprintf("proto declaration error: %v", err) + return res + } + testOpts = append(testOpts, opt) + } + + testEnv := baseEnv + if len(testOpts) > 0 { + var err error + testEnv, err = baseEnv.Extend(testOpts...) + if err != nil { + res.Status = fmt.Sprintf("env extension error: %v", err) + return res + } + } + + if !test.GetDisableCheck() { + ast, iss = testEnv.Check(ast) + if iss.Err() != nil { + res.Status = fmt.Sprintf("check error: %v", iss.Err()) + return res + } + } + + // check_only tests verify the deduced output type without evaluation. + if test.GetCheckOnly() { + m, ok := test.GetResultMatcher().(*testpb.SimpleTest_TypedResult) + if !ok || m.TypedResult == nil { + res.Status = fmt.Sprintf("unexpected matcher kind for check-only test: %T", test.GetResultMatcher()) + return res + } + if status := checkDeducedType(m.TypedResult.GetDeducedType(), ast.OutputType()); status != "" { + res.Status = status + return res + } + res.Status = "pass" + return res + } + + prg, err := testEnv.Program(ast) + if err != nil { + res.Status = fmt.Sprintf("program creation error: %v", err) + return res + } + + bindings := make(map[string]any, len(test.GetBindings())) + for k, v := range test.GetBindings() { + val, err := exprValueToRefValue(testEnv.CELTypeAdapter(), v) + if err != nil { + res.Status = fmt.Sprintf("binding conversion error for %q: %v", k, err) + return res + } + bindings[k] = val + } + + ret, _, evalErr := prg.Eval(bindings) + if evalErr == nil && ret != nil && types.IsError(ret) { + if errVal, ok := ret.(*types.Err); ok { + evalErr = errVal.Unwrap() + } else { + evalErr = fmt.Errorf("%v", ret) + } + } + + // Default result matcher is boolean true. + matcher := test.GetResultMatcher() + if matcher == nil { + matcher = &testpb.SimpleTest_Value{ + Value: &expr.Value{ + Kind: &expr.Value_BoolValue{BoolValue: true}, + }, + } + } + + switch m := matcher.(type) { + case *testpb.SimpleTest_Value: + if evalErr != nil { + res.Status = fmt.Sprintf("unexpected eval error: %v", evalErr) + return res + } + val, err := refValueToExprValue(ret) + if err != nil { + res.Status = fmt.Sprintf("failed converting result to value proto: %v", err) + return res + } + if !equalValues(m.Value, val.GetValue()) { + res.Status = fmt.Sprintf("value mismatch: want %v, got %v", m.Value, val.GetValue()) + return res + } + res.Status = "pass" + return res + + case *testpb.SimpleTest_TypedResult: + if evalErr != nil { + res.Status = fmt.Sprintf("unexpected eval error: %v", evalErr) + return res + } + val, err := refValueToExprValue(ret) + if err != nil { + res.Status = fmt.Sprintf("failed converting result to value proto: %v", err) + return res + } + if m.TypedResult != nil { + if m.TypedResult.GetResult() != nil && !equalValues(m.TypedResult.GetResult(), val.GetValue()) { + res.Status = fmt.Sprintf("value mismatch: want %v, got %v", m.TypedResult.GetResult(), val.GetValue()) + return res + } + if m.TypedResult.GetDeducedType() != nil { + if status := checkDeducedType(m.TypedResult.GetDeducedType(), ast.OutputType()); status != "" { + res.Status = status + return res + } + } + } + res.Status = "pass" + return res + + case *testpb.SimpleTest_EvalError, *testpb.SimpleTest_AnyEvalErrors: + if evalErr == nil { + res.Status = fmt.Sprintf("expected eval error, got %v", ret) + return res + } + res.Status = "pass" + return res + + case *testpb.SimpleTest_Unknown, *testpb.SimpleTest_AnyUnknowns: + if ret == nil || !types.IsUnknown(ret) { + res.Status = fmt.Sprintf("expected unknown, got %v", ret) + return res + } + res.Status = "pass" + return res + + default: + res.Status = fmt.Sprintf("unsupported result matcher: %T", matcher) + return res + } +} + +// ParseConformanceFile parses a conformance test from a file path or direct content. +// It first attempts to read from disk as a file; if successful, it unmarshals from textproto +// or protojson into a SimpleTestFile (or wrapped SimpleTest). Otherwise, it treats the content +// itself as a test case parsable from textproto or protojson. +func ParseConformanceFile(pathOrContent string) (*testpb.SimpleTestFile, error) { + trimmed := strings.TrimSpace(pathOrContent) + if len(trimmed) == 0 { + return nil, errors.New("empty conformance test content") + } + + isExplicitPath := strings.HasPrefix(trimmed, "/") || + strings.HasPrefix(trimmed, "./") || + strings.HasPrefix(trimmed, "../") || + strings.HasSuffix(trimmed, ".textproto") || + strings.HasSuffix(trimmed, ".proto") || + strings.HasSuffix(trimmed, ".json") || + strings.HasSuffix(trimmed, ".pb") + + if fi, err := os.Stat(pathOrContent); err == nil { + if fi.IsDir() { + return nil, fmt.Errorf("conformance path %q is a directory, expected a file", pathOrContent) + } + fileBytes, err := os.ReadFile(pathOrContent) + if err != nil { + return nil, fmt.Errorf("failed reading conformance test file %q: %w", pathOrContent, err) + } + return parseConformanceBytes(bytes.TrimSpace(fileBytes)) + } else if isExplicitPath { + return nil, fmt.Errorf("failed reading conformance test file %q: %w", pathOrContent, err) + } + + // Otherwise, treat content itself as parsable from textproto or protojson. + return parseConformanceBytes([]byte(trimmed)) +} + +func parseConformanceBytes(data []byte) (*testpb.SimpleTestFile, error) { + if len(data) == 0 { + return nil, errors.New("empty conformance test content") + } + + type unmarshalFunc func([]byte, proto.Message) error + var formats []unmarshalFunc + if bytes.HasPrefix(data, []byte("{")) { + formats = []unmarshalFunc{protojson.Unmarshal, prototext.Unmarshal} + } else { + formats = []unmarshalFunc{prototext.Unmarshal, protojson.Unmarshal} + } + + var lastErr error + for _, unmarshal := range formats { + // 1. Try SimpleTestFile + file := &testpb.SimpleTestFile{} + if err := unmarshal(data, file); err == nil && (len(file.GetSection()) > 0 || file.GetName() != "") { + return file, nil + } else if err != nil { + lastErr = err + } + + // 2. Try standalone SimpleTest + singleTest := &testpb.SimpleTest{} + if err := unmarshal(data, singleTest); err == nil && (singleTest.GetExpr() != "" || singleTest.GetName() != "") { + return wrapSingleTest(singleTest), nil + } else if err != nil { + lastErr = err + } + + // 3. Try SimpleTestSection + section := &testpb.SimpleTestSection{} + if err := unmarshal(data, section); err == nil && len(section.GetTest()) > 0 { + return wrapSection(section), nil + } else if err != nil { + lastErr = err + } + } + + sample := string(data) + if len(sample) > 200 { + sample = sample[:200] + "... (truncated)" + } + if lastErr != nil { + return nil, fmt.Errorf("unable to parse conformance test: %w (input sample: %q)", lastErr, sample) + } + return nil, fmt.Errorf("unable to parse conformance test (input sample: %q)", sample) +} + +func wrapSingleTest(st *testpb.SimpleTest) *testpb.SimpleTestFile { + name := st.GetName() + if name == "" { + name = "standalone" + } + return &testpb.SimpleTestFile{ + Name: name, + Section: []*testpb.SimpleTestSection{ + { + Name: "default", + Test: []*testpb.SimpleTest{st}, + }, + }, + } +} + +func wrapSection(sec *testpb.SimpleTestSection) *testpb.SimpleTestFile { + return &testpb.SimpleTestFile{ + Name: sec.GetName(), + Description: sec.GetDescription(), + Section: []*testpb.SimpleTestSection{sec}, + } +} + +// SimpleTestToTestCase converts a SimpleTest to the unit test TestCase format. +func SimpleTestToTestCase(st *testpb.SimpleTest) (*TestCase, error) { + if st == nil { + return nil, errors.New("nil SimpleTest") + } + bindings := make(map[string]any, len(st.GetBindings())) + for k, v := range st.GetBindings() { + if v.GetValue() != nil { + jsVal, err := valueToJSON(v.GetValue()) + if err != nil { + return nil, fmt.Errorf("failed converting binding %q: %w", k, err) + } + bindings[k] = jsVal + } + } + + var expected any + if v := st.GetValue(); v != nil { + var err error + expected, err = valueToJSON(v) + if err != nil { + return nil, fmt.Errorf("failed converting expected value: %w", err) + } + } else if tr := st.GetTypedResult(); tr != nil && tr.GetResult() != nil { + var err error + expected, err = valueToJSON(tr.GetResult()) + if err != nil { + return nil, fmt.Errorf("failed converting expected value: %w", err) + } + } + + return &TestCase{ + TestCase: st.GetName(), + Bindings: bindings, + Expected: expected, + }, nil +} + +// TestCaseToSimpleTest converts a unit test TestCase into a SimpleTest. +func TestCaseToSimpleTest(tc TestCase, exprStr string) (*testpb.SimpleTest, error) { + st := &testpb.SimpleTest{ + Name: tc.TestCase, + Expr: exprStr, + Bindings: make(map[string]*expr.ExprValue, len(tc.Bindings)), + } + + for k, v := range tc.Bindings { + pbVal, err := jsonToValue(v) + if err != nil { + return nil, fmt.Errorf("failed converting binding %q: %w", k, err) + } + st.Bindings[k] = &expr.ExprValue{ + Kind: &expr.ExprValue_Value{Value: pbVal}, + } + } + + if tc.Expected != nil { + pbVal, err := jsonToValue(tc.Expected) + if err != nil { + return nil, fmt.Errorf("failed converting expected value: %w", err) + } + st.ResultMatcher = &testpb.SimpleTest_Value{Value: pbVal} + } + + return st, nil +} + +// FormatSimpleTestFile formats a SimpleTestFile as textproto. +func FormatSimpleTestFile(file *testpb.SimpleTestFile) (string, error) { + if file == nil { + return "", errors.New("nil SimpleTestFile") + } + opts := prototext.MarshalOptions{Multiline: true, Indent: " "} + b, err := opts.Marshal(file) + if err != nil { + return "", err + } + return string(b), nil +} + +// FormatSimpleTest formats a SimpleTest as textproto. +func FormatSimpleTest(test *testpb.SimpleTest) (string, error) { + if test == nil { + return "", errors.New("nil SimpleTest") + } + opts := prototext.MarshalOptions{Multiline: true, Indent: " "} + b, err := opts.Marshal(test) + if err != nil { + return "", err + } + return string(b), nil +} + +// refValueToExprValue converts a CEL ref.Val result into an ExprValue protobuf message. +// Derived from cel-expr/cel-go/conformance/conformance_test.go (refValueToExprValue). +func refValueToExprValue(res ref.Val) (*expr.ExprValue, error) { + if types.IsUnknown(res) { + return &expr.ExprValue{ + Kind: &expr.ExprValue_Unknown{ + Unknown: &expr.UnknownSet{ + Exprs: res.Value().([]int64), + }, + }, + }, nil + } + v, err := cel.ValueAsProto(res) + if err != nil { + return nil, err + } + return &expr.ExprValue{ + Kind: &expr.ExprValue_Value{Value: v}, + }, nil +} + +// exprValueToRefValue converts an ExprValue protobuf message to a CEL ref.Val. +// Derived from cel-expr/cel-go/conformance/conformance_test.go (exprValueToRefValue). +func exprValueToRefValue(adapter types.Adapter, ev *expr.ExprValue) (ref.Val, error) { + if ev == nil { + return nil, nil + } + switch k := ev.Kind.(type) { + case *expr.ExprValue_Value: + return cel.ProtoAsValue(adapter, k.Value) + case *expr.ExprValue_Error: + return types.NewErr("evaluation error"), nil + case *expr.ExprValue_Unknown: + var unk *types.Unknown + for _, id := range k.Unknown.GetExprs() { + if unk == nil { + unk = types.NewUnknown(id, nil) + } else { + unk = types.MergeUnknowns(types.NewUnknown(id, nil), unk) + } + } + return unk, nil + default: + return nil, fmt.Errorf("unknown ExprValue kind: %T", ev.Kind) + } +} + +// equalValues compares expected and actual expr.Value protobuf messages for semantic equality. +// Handles IEEE-754 NaN semantics, list order preservation, and order-insensitive map entry comparison. +// Derived from cel-expr/cel-go/conformance/conformance_test.go (diffValue / equalValues). +func equalValues(want, got *expr.Value) bool { + if want == nil || got == nil { + return want == got + } + switch w := want.Kind.(type) { + case *expr.Value_NullValue: + g, ok := got.Kind.(*expr.Value_NullValue) + return ok && w.NullValue == g.NullValue + case *expr.Value_BoolValue: + g, ok := got.Kind.(*expr.Value_BoolValue) + return ok && w.BoolValue == g.BoolValue + case *expr.Value_Int64Value: + g, ok := got.Kind.(*expr.Value_Int64Value) + return ok && w.Int64Value == g.Int64Value + case *expr.Value_Uint64Value: + g, ok := got.Kind.(*expr.Value_Uint64Value) + return ok && w.Uint64Value == g.Uint64Value + case *expr.Value_DoubleValue: + g, ok := got.Kind.(*expr.Value_DoubleValue) + if !ok { + return false + } + if math.IsNaN(w.DoubleValue) && math.IsNaN(g.DoubleValue) { + return true + } + return w.DoubleValue == g.DoubleValue + case *expr.Value_StringValue: + g, ok := got.Kind.(*expr.Value_StringValue) + return ok && w.StringValue == g.StringValue + case *expr.Value_BytesValue: + g, ok := got.Kind.(*expr.Value_BytesValue) + return ok && bytes.Equal(w.BytesValue, g.BytesValue) + case *expr.Value_TypeValue: + g, ok := got.Kind.(*expr.Value_TypeValue) + return ok && w.TypeValue == g.TypeValue + case *expr.Value_EnumValue: + g, ok := got.Kind.(*expr.Value_EnumValue) + return ok && proto.Equal(w.EnumValue, g.EnumValue) + case *expr.Value_ObjectValue: + g, ok := got.Kind.(*expr.Value_ObjectValue) + if !ok { + return false + } + if proto.Equal(w.ObjectValue, g.ObjectValue) { + return true + } + if w.ObjectValue == nil || g.ObjectValue == nil { + return w.ObjectValue == g.ObjectValue + } + wMsg, wErr := w.ObjectValue.UnmarshalNew() + gMsg, gErr := g.ObjectValue.UnmarshalNew() + if wErr == nil && gErr == nil { + return proto.Equal(wMsg, gMsg) + } + return false + case *expr.Value_ListValue: + g, ok := got.Kind.(*expr.Value_ListValue) + if !ok { + return false + } + wVals := w.ListValue.GetValues() + gVals := g.ListValue.GetValues() + if len(wVals) != len(gVals) { + return false + } + for i, elem := range wVals { + if !equalValues(elem, gVals[i]) { + return false + } + } + return true + case *expr.Value_MapValue: + g, ok := got.Kind.(*expr.Value_MapValue) + if !ok { + return false + } + wEntries := w.MapValue.GetEntries() + gEntries := g.MapValue.GetEntries() + if len(wEntries) != len(gEntries) { + return false + } + matched := make([]bool, len(gEntries)) + for _, we := range wEntries { + found := false + for j, ge := range gEntries { + if !matched[j] && equalValues(we.GetKey(), ge.GetKey()) && equalValues(we.GetValue(), ge.GetValue()) { + matched[j] = true + found = true + break + } + } + if !found { + return false + } + } + return true + } + return proto.Equal(want, got) +} + +// equalTypes compares expected and actual expr.Type protobuf messages for equality. +// Derived from cel-expr/cel-go/conformance/conformance_test.go (diffType). +func equalTypes(want, got *expr.Type) bool { + return proto.Equal(want, got) +} + +func valueToJSON(v *expr.Value) (any, error) { + if v == nil { + return nil, nil + } + switch k := v.Kind.(type) { + case *expr.Value_NullValue: + return nil, nil + case *expr.Value_BoolValue: + return k.BoolValue, nil + case *expr.Value_Int64Value: + return k.Int64Value, nil + case *expr.Value_Uint64Value: + return k.Uint64Value, nil + case *expr.Value_DoubleValue: + return k.DoubleValue, nil + case *expr.Value_StringValue: + return k.StringValue, nil + case *expr.Value_BytesValue: + return k.BytesValue, nil + case *expr.Value_TypeValue: + return k.TypeValue, nil + case *expr.Value_ListValue: + res := make([]any, 0, len(k.ListValue.GetValues())) + for _, elem := range k.ListValue.GetValues() { + jv, err := valueToJSON(elem) + if err != nil { + return nil, err + } + res = append(res, jv) + } + return res, nil + case *expr.Value_MapValue: + entries := k.MapValue.GetEntries() + res := make(map[string]any, len(entries)) + for _, entry := range entries { + var keyStr string + if keyVal := entry.GetKey(); keyVal != nil { + if keyVal.GetStringValue() != "" { + keyStr = keyVal.GetStringValue() + } else { + kJSON, err := valueToJSON(keyVal) + if err == nil && kJSON != nil { + keyStr = fmt.Sprintf("%v", kJSON) + } else { + keyStr = fmt.Sprintf("%v", keyVal) + } + } + } + jv, err := valueToJSON(entry.GetValue()) + if err != nil { + return nil, err + } + res[keyStr] = jv + } + return res, nil + case *expr.Value_EnumValue: + if k.EnumValue == nil { + return nil, nil + } + return k.EnumValue.GetValue(), nil + case *expr.Value_ObjectValue: + if k.ObjectValue == nil { + return nil, nil + } + msg, err := k.ObjectValue.UnmarshalNew() + if err != nil { + return nil, fmt.Errorf("failed unmarshaling ObjectValue: %w", err) + } + jsonBytes, err := protojson.Marshal(msg) + if err != nil { + return nil, fmt.Errorf("failed marshaling ObjectValue to JSON: %w", err) + } + var jsVal any + if err := json.Unmarshal(jsonBytes, &jsVal); err != nil { + return nil, fmt.Errorf("failed unmarshaling ObjectValue JSON: %w", err) + } + return jsVal, nil + default: + return nil, fmt.Errorf("unsupported value type: %T", v.Kind) + } +} + +func jsonToValue(val any) (*expr.Value, error) { + if val == nil { + return &expr.Value{Kind: &expr.Value_NullValue{}}, nil + } + switch v := val.(type) { + case bool: + return &expr.Value{Kind: &expr.Value_BoolValue{BoolValue: v}}, nil + case int: + return &expr.Value{Kind: &expr.Value_Int64Value{Int64Value: int64(v)}}, nil + case int64: + return &expr.Value{Kind: &expr.Value_Int64Value{Int64Value: v}}, nil + case uint: + return &expr.Value{Kind: &expr.Value_Uint64Value{Uint64Value: uint64(v)}}, nil + case uint64: + return &expr.Value{Kind: &expr.Value_Uint64Value{Uint64Value: v}}, nil + case float64: + if math.Floor(v) == v && !math.IsNaN(v) && !math.IsInf(v, 0) && v >= math.MinInt64 && v <= math.MaxInt64 { + // Integer value represented in JSON float + return &expr.Value{Kind: &expr.Value_Int64Value{Int64Value: int64(v)}}, nil + } + return &expr.Value{Kind: &expr.Value_DoubleValue{DoubleValue: v}}, nil + case string: + return &expr.Value{Kind: &expr.Value_StringValue{StringValue: v}}, nil + case []byte: + return &expr.Value{Kind: &expr.Value_BytesValue{BytesValue: v}}, nil + case []any: + elements := make([]*expr.Value, 0, len(v)) + for _, elem := range v { + pbVal, err := jsonToValue(elem) + if err != nil { + return nil, err + } + elements = append(elements, pbVal) + } + return &expr.Value{Kind: &expr.Value_ListValue{ListValue: &expr.ListValue{Values: elements}}}, nil + case map[string]any: + entries := make([]*expr.MapValue_Entry, 0, len(v)) + for k, elem := range v { + pbKey := &expr.Value{Kind: &expr.Value_StringValue{StringValue: k}} + pbVal, err := jsonToValue(elem) + if err != nil { + return nil, err + } + entries = append(entries, &expr.MapValue_Entry{Key: pbKey, Value: pbVal}) + } + return &expr.Value{Kind: &expr.Value_MapValue{MapValue: &expr.MapValue{Entries: entries}}}, nil + default: + return nil, fmt.Errorf("unsupported JSON value type: %T", val) + } +} + +// celBlockLib simulates indexed arguments and test-only macros for cel.block conformance tests. +// Derived from cel-expr/cel-go/conformance/conformance_test.go (celBlockLib, celBlock, celIndex, celCompreVar). +type celBlockLib struct{} + +func (celBlockLib) LibraryName() string { + return "cel.lib.ext.cel.block.conformance" +} + +func (celBlockLib) CompileOptions() []cel.EnvOption { + maxIndices := 30 + indexOpts := make([]cel.EnvOption, maxIndices) + for i := 0; i < maxIndices; i++ { + indexOpts[i] = cel.Variable(fmt.Sprintf("@index%d", i), cel.DynType) + } + return append([]cel.EnvOption{ + cel.Macros( + // cel.block([args], expr) + cel.ReceiverMacro("block", 2, celBlock), + // cel.index(int) + cel.ReceiverMacro("index", 1, celIndex), + // cel.iterVar(int, int) + cel.ReceiverMacro("iterVar", 2, celCompreVar("cel.iterVar", "@it")), + // cel.accuVar(int, int) + cel.ReceiverMacro("accuVar", 2, celCompreVar("cel.accuVar", "@ac")), + ), + }, indexOpts...) +} + +func (celBlockLib) ProgramOptions() []cel.ProgramOption { + return []cel.ProgramOption{} +} + +func celBlock(mef cel.MacroExprFactory, target ast.Expr, args []ast.Expr) (ast.Expr, *cel.Error) { + if !isCELNamespace(target) { + return nil, nil + } + bindings := args[0] + if bindings.Kind() != ast.ListKind { + return bindings, mef.NewError(bindings.ID(), "cel.block requires the first arg to be a list literal") + } + return mef.NewCall("cel.@block", args...), nil +} + +func celIndex(mef cel.MacroExprFactory, target ast.Expr, args []ast.Expr) (ast.Expr, *cel.Error) { + if !isCELNamespace(target) { + return nil, nil + } + index := args[0] + if !isNonNegativeInt(index) { + return index, mef.NewError(index.ID(), "cel.index requires a single non-negative int constant arg") + } + indexVal := index.AsLiteral().(types.Int) + return mef.NewIdent(fmt.Sprintf("@index%d", indexVal)), nil +} + +func celCompreVar(funcName, varPrefix string) cel.MacroFactory { + return func(mef cel.MacroExprFactory, target ast.Expr, args []ast.Expr) (ast.Expr, *cel.Error) { + if !isCELNamespace(target) { + return nil, nil + } + depth := args[0] + if !isNonNegativeInt(depth) { + return depth, mef.NewError(depth.ID(), fmt.Sprintf("%s requires two non-negative int constant args", funcName)) + } + unique := args[1] + if !isNonNegativeInt(unique) { + return unique, mef.NewError(unique.ID(), fmt.Sprintf("%s requires two non-negative int constant args", funcName)) + } + depthVal := depth.AsLiteral().(types.Int) + uniqueVal := unique.AsLiteral().(types.Int) + return mef.NewIdent(fmt.Sprintf("%s:%d:%d", varPrefix, depthVal, uniqueVal)), nil + } +} + +func isCELNamespace(target ast.Expr) bool { + return target.Kind() == ast.IdentKind && target.AsIdent() == "cel" +} + +func isNonNegativeInt(expr ast.Expr) bool { + if expr.Kind() != ast.LiteralKind { + return false + } + val := expr.AsLiteral() + return val.Type() == cel.IntType && val.(types.Int) >= 0 +} diff --git a/internal/tools/conformance_test.go b/internal/tools/conformance_test.go new file mode 100644 index 0000000..85e55b7 --- /dev/null +++ b/internal/tools/conformance_test.go @@ -0,0 +1,706 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tools + +import ( + "os" + "path/filepath" + "strings" + "testing" + + expr "cel.dev/expr" + testpb "cel.dev/expr/conformance/test" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" +) + +const sampleTextproto = ` +name: "sample_conformance" +description: "Sample test suite for CEL conformance evaluation" +section { + name: "primitives" + description: "Basic literal values" + test { + name: "int_zero" + expr: "0" + value { int64_value: 0 } + } + test { + name: "uint_val" + expr: "42u" + value { uint64_value: 42 } + } + test { + name: "float_val" + expr: "3.14" + value { double_value: 3.14 } + } + test { + name: "string_val" + expr: "'hello world'" + value { string_value: "hello world" } + } + test { + name: "bool_val" + expr: "true" + value { bool_value: true } + } + test { + name: "default_bool_true" + expr: "1 < 2" + } +} +section { + name: "bindings_and_env" + test { + name: "type_env_and_binding" + expr: "x + y" + type_env { + name: "x" + ident { + type { primitive: INT64 } + } + } + type_env { + name: "y" + ident { + type { primitive: INT64 } + } + } + bindings { + key: "x" + value { + value { int64_value: 10 } + } + } + bindings { + key: "y" + value { + value { int64_value: 20 } + } + } + value { int64_value: 30 } + } +} +section { + name: "errors_and_checks" + test { + name: "runtime_error" + expr: "1 / 0" + eval_error { + errors { message: "division by zero" } + } + } + test { + name: "unchecked_unbound" + expr: "unbound_var" + disable_check: true + eval_error {} + } + test { + name: "check_only_deduced_type" + expr: "size('test')" + check_only: true + typed_result { + deduced_type { + primitive: INT64 + } + } + } +} +section { + name: "block_macros" + test { + name: "cel_block_test" + expr: "cel.block([1, cel.index(0) + 2, cel.index(1) + 3], cel.index(2))" + value { int64_value: 6 } + } +} +` + +func TestEvaluateConformance(t *testing.T) { + out, err := EvaluateConformance(sampleTextproto, nil) + if err != nil { + t.Fatalf("EvaluateConformance failed: %v", err) + } + + if out.Name != "sample_conformance" { + t.Errorf("expected name sample_conformance, got %s", out.Name) + } + if out.Failed > 0 { + for _, r := range out.Results { + if !r.Passed() { + t.Errorf("test %s failed: %s", r.Name, r.Status) + } + } + } + if out.Passed != out.Total { + t.Errorf("expected all %d tests to pass, but passed: %d, failed: %d, skipped: %d", out.Total, out.Passed, out.Failed, out.Skipped) + } +} + +func TestEvaluateConformanceParams(t *testing.T) { + // Test filtering + out, err := EvaluateConformanceWithParams(ConformanceParams{ + Tests: sampleTextproto, + Filter: "primitives", + }) + if err != nil { + t.Fatalf("failed: %v", err) + } + if out.Total != 6 { + t.Errorf("expected 6 filtered tests, got %d", out.Total) + } + + // Test skip + out, err = EvaluateConformanceWithParams(ConformanceParams{ + Tests: sampleTextproto, + SkipTests: []string{"sample_conformance/block_macros"}, + }) + if err != nil { + t.Fatalf("failed: %v", err) + } + if out.Skipped != 1 { + t.Errorf("expected 1 skipped test, got %d", out.Skipped) + } +} + +func TestEvaluateConformanceSingleTest(t *testing.T) { + single := ` +name: "single_test" +expr: "2 * 3" +value { int64_value: 6 } +` + out, err := EvaluateConformance(single, nil) + if err != nil { + t.Fatalf("failed: %v", err) + } + if out.Total != 1 || out.Passed != 1 { + t.Errorf("expected 1 passed test, got total=%d passed=%d", out.Total, out.Passed) + } +} + +func TestEvaluateConformanceSection(t *testing.T) { + section := ` +name: "section_only" +test { + name: "t1" + expr: "true && false" + value { bool_value: false } +} +test { + name: "t2" + expr: "true || false" + value { bool_value: true } +} +` + out, err := EvaluateConformance(section, nil) + if err != nil { + t.Fatalf("failed: %v", err) + } + if out.Total != 2 || out.Passed != 2 { + t.Errorf("expected 2 passed tests, got total=%d passed=%d", out.Total, out.Passed) + } +} + +func TestEvaluateConformanceTestFunc(t *testing.T) { + st := &testpb.SimpleTest{ + Name: "direct_test", + Expr: "'abc' + 'def'", + ResultMatcher: &testpb.SimpleTest_Value{ + Value: &expr.Value{ + Kind: &expr.Value_StringValue{StringValue: "abcdef"}, + }, + }, + } + res, err := EvaluateConformanceTest(st, nil) + if err != nil { + t.Fatalf("EvaluateConformanceTest failed: %v", err) + } + if !res.Passed() { + t.Errorf("expected pass, got %s", res.Status) + } +} + +func TestSimpleTestConversion(t *testing.T) { + tc := TestCase{ + TestCase: "tc_conversion", + Bindings: map[string]any{ + "str": "hello", + "num": 42, + }, + Expected: true, + } + + st, err := TestCaseToSimpleTest(tc, "str == 'hello' && num == 42") + if err != nil { + t.Fatalf("TestCaseToSimpleTest failed: %v", err) + } + + if st.GetName() != "tc_conversion" { + t.Errorf("expected name tc_conversion, got %s", st.GetName()) + } + if len(st.GetBindings()) != 2 { + t.Errorf("expected 2 bindings, got %d", len(st.GetBindings())) + } + + convertedBack, err := SimpleTestToTestCase(st) + if err != nil { + t.Fatalf("SimpleTestToTestCase failed: %v", err) + } + if convertedBack.TestCase != tc.TestCase { + t.Errorf("expected %s, got %s", tc.TestCase, convertedBack.TestCase) + } + + text, err := FormatSimpleTest(st) + if err != nil { + t.Fatalf("FormatSimpleTest failed: %v", err) + } + if len(text) == 0 { + t.Error("expected non-empty formatted textproto") + } +} + +func TestEqualValues(t *testing.T) { + // List equality + v1 := &expr.Value{ + Kind: &expr.Value_ListValue{ + ListValue: &expr.ListValue{ + Values: []*expr.Value{ + {Kind: &expr.Value_Int64Value{Int64Value: 1}}, + {Kind: &expr.Value_Int64Value{Int64Value: 2}}, + }, + }, + }, + } + v2 := proto.Clone(v1).(*expr.Value) + if !equalValues(v1, v2) { + t.Error("identical list values should be equal") + } + + // Map equality regardless of order + m1 := &expr.Value{ + Kind: &expr.Value_MapValue{ + MapValue: &expr.MapValue{ + Entries: []*expr.MapValue_Entry{ + { + Key: &expr.Value{Kind: &expr.Value_StringValue{StringValue: "a"}}, + Value: &expr.Value{Kind: &expr.Value_Int64Value{Int64Value: 1}}, + }, + { + Key: &expr.Value{Kind: &expr.Value_StringValue{StringValue: "b"}}, + Value: &expr.Value{Kind: &expr.Value_Int64Value{Int64Value: 2}}, + }, + }, + }, + }, + } + m2 := &expr.Value{ + Kind: &expr.Value_MapValue{ + MapValue: &expr.MapValue{ + Entries: []*expr.MapValue_Entry{ + { + Key: &expr.Value{Kind: &expr.Value_StringValue{StringValue: "b"}}, + Value: &expr.Value{Kind: &expr.Value_Int64Value{Int64Value: 2}}, + }, + { + Key: &expr.Value{Kind: &expr.Value_StringValue{StringValue: "a"}}, + Value: &expr.Value{Kind: &expr.Value_Int64Value{Int64Value: 1}}, + }, + }, + }, + }, + } + if !equalValues(m1, m2) { + t.Error("map values with different entry ordering should be equal") + } +} + +func TestEvaluateConformanceFileOnDisk(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "test.textproto") + if err := os.WriteFile(path, []byte(sampleTextproto), 0644); err != nil { + t.Fatal(err) + } + + out, err := EvaluateConformance(path, nil) + if err != nil { + t.Fatalf("EvaluateConformance with file path failed: %v", err) + } + if out.Passed != out.Total || out.Total == 0 { + t.Errorf("expected all tests to pass, got total=%d passed=%d", out.Total, out.Passed) + } +} + +func findCelSpecTestdataDir() string { + if dir := os.Getenv("CEL_SPEC_TESTDATA"); dir != "" { + if fi, err := os.Stat(dir); err == nil && fi.IsDir() { + return dir + } + } + if dir := os.Getenv("CEL_SPEC_DIR"); dir != "" { + sub := filepath.Join(dir, "tests", "simple", "testdata") + if fi, err := os.Stat(sub); err == nil && fi.IsDir() { + return sub + } + } + gopath := os.Getenv("GOPATH") + if gopath == "" { + if home, err := os.UserHomeDir(); err == nil { + gopath = filepath.Join(home, "go") + } + } + candidates := []string{ + filepath.Join(gopath, "src", "github.com", "google", "cel-spec", "tests", "simple", "testdata"), + filepath.Join("..", "..", "..", "google", "cel-spec", "tests", "simple", "testdata"), + filepath.Join("..", "..", "cel-spec", "tests", "simple", "testdata"), + } + for _, c := range candidates { + if fi, err := os.Stat(c); err == nil && fi.IsDir() { + return c + } + } + return "" +} + +func TestExternalCelSpecBasicTextproto(t *testing.T) { + dir := findCelSpecTestdataDir() + if dir == "" { + t.Skip("cel-spec testdata directory not found") + } + celSpecBasic := filepath.Join(dir, "basic.textproto") + if _, err := os.Stat(celSpecBasic); os.IsNotExist(err) { + t.Skip("cel-spec basic.textproto not available in local environment") + } + + out, err := EvaluateConformance(celSpecBasic, nil) + if err != nil { + t.Fatalf("failed evaluating cel-spec basic.textproto: %v", err) + } + + t.Logf("Evaluated cel-spec basic.textproto: %d tests, %d passed, %d failed", out.Total, out.Passed, out.Failed) + if out.Failed > 0 { + for _, r := range out.Results { + if !r.Passed() { + t.Errorf("test %s failed: %s", r.Name, r.Status) + } + } + } + if out.Passed != out.Total { + t.Errorf("expected all %d tests in basic.textproto to pass, got %d passed", out.Total, out.Passed) + } +} + +func TestExternalCelSpecBlockExtTextproto(t *testing.T) { + dir := findCelSpecTestdataDir() + if dir == "" { + t.Skip("cel-spec testdata directory not found") + } + celSpecBlock := filepath.Join(dir, "block_ext.textproto") + if _, err := os.Stat(celSpecBlock); os.IsNotExist(err) { + t.Skip("cel-spec block_ext.textproto not available in local environment") + } + + out, err := EvaluateConformance(celSpecBlock, nil) + if err != nil { + t.Fatalf("failed evaluating cel-spec block_ext.textproto: %v", err) + } + + t.Logf("Evaluated cel-spec block_ext.textproto: %d tests, %d passed, %d failed", out.Total, out.Passed, out.Failed) + if out.Failed > 0 { + for _, r := range out.Results { + if !r.Passed() { + t.Errorf("test %s failed: %s", r.Name, r.Status) + } + } + } + if out.Passed != out.Total { + t.Errorf("expected all %d tests in block_ext.textproto to pass, got %d passed", out.Total, out.Passed) + } +} + +func TestEvaluateConformanceProtoJSON(t *testing.T) { + // Single test in ProtoJSON format + jsonSingle := `{ + "name": "json_single_test", + "expr": "10 * 20", + "value": { "int64Value": "200" } + }` + out, err := EvaluateConformance(jsonSingle, nil) + if err != nil { + t.Fatalf("EvaluateConformance with protojson single test failed: %v", err) + } + if out.Total != 1 || out.Passed != 1 { + t.Errorf("expected 1 passed test, got total=%d passed=%d", out.Total, out.Passed) + } + + // Full SimpleTestFile in ProtoJSON format + jsonFile := `{ + "name": "json_file_suite", + "description": "Suite in ProtoJSON", + "section": [ + { + "name": "arithmetic", + "test": [ + { + "name": "add", + "expr": "5 + 5", + "value": { "int64Value": "10" } + }, + { + "name": "sub", + "expr": "15 - 5", + "value": { "int64Value": "10" } + } + ] + } + ] + }` + out, err = EvaluateConformance(jsonFile, nil) + if err != nil { + t.Fatalf("EvaluateConformance with protojson suite failed: %v", err) + } + if out.Total != 2 || out.Passed != 2 { + t.Errorf("expected 2 passed tests, got total=%d passed=%d", out.Total, out.Passed) + } +} + +func TestEndToEndAllCelSpecConformance(t *testing.T) { + celSpecDir := findCelSpecTestdataDir() + if celSpecDir == "" { + t.Skip("cel-spec testdata directory not found") + } + + files, err := filepath.Glob(filepath.Join(celSpecDir, "*.textproto")) + if err != nil { + t.Fatalf("failed globbing cel-spec textproto files: %v", err) + } + if len(files) == 0 { + t.Fatalf("no cel-spec textproto files found in %s", celSpecDir) + } + + totalAll := 0 + passedAll := 0 + failedAll := 0 + skippedAll := 0 + + for _, file := range files { + fileName := filepath.Base(file) + // Skip network_ext as it tests IP extensions not included in cel-go core + if fileName == "network_ext.textproto" { + t.Logf("Skipping %s (requires network extensions outside cel-go core)", fileName) + continue + } + + t.Run(fileName, func(t *testing.T) { + out, err := EvaluateConformanceWithParams(ConformanceParams{ + Tests: file, + SkipTests: DefaultConformanceSkipTests(), + }) + if err != nil { + t.Fatalf("failed evaluating %s: %v", fileName, err) + } + + totalAll += out.Total + passedAll += out.Passed + failedAll += out.Failed + skippedAll += out.Skipped + + t.Logf("[%s] Total: %d, Passed: %d, Failed: %d, Skipped: %d", + fileName, out.Total, out.Passed, out.Failed, out.Skipped) + + if out.Failed > 0 { + for _, r := range out.Results { + if !r.Passed() { + t.Errorf("FAIL in %s: %s -> %s", fileName, r.Name, r.Status) + } + } + } + }) + } + + t.Logf("OVERALL CONFORMANCE SUMMARY: Total: %d, Passed: %d, Skipped: %d, Failed: %d across %d files", + totalAll, passedAll, skippedAll, failedAll, len(files)-1) + if failedAll > 0 { + t.Errorf("conformance evaluation had %d failures out of %d tests", failedAll, totalAll) + } +} + +func TestConformanceReviewEdgeCases(t *testing.T) { + t.Run("equalValues_nil_ObjectValue", func(t *testing.T) { + vNil := &expr.Value{Kind: &expr.Value_ObjectValue{ObjectValue: nil}} + vVal := &expr.Value{Kind: &expr.Value_ObjectValue{ObjectValue: &anypb.Any{TypeUrl: "test"}}} + + // Must not panic + if equalValues(vNil, vVal) { + t.Error("nil ObjectValue should not equal non-nil ObjectValue") + } + if equalValues(vVal, vNil) { + t.Error("non-nil ObjectValue should not equal nil ObjectValue") + } + if !equalValues(vNil, vNil) { + t.Error("nil ObjectValue should equal nil ObjectValue") + } + }) + + t.Run("formatTestName_symmetry", func(t *testing.T) { + tests := []struct { + file, section, test string + want string + }{ + {"suite", "sec", "t1", "suite/sec/t1"}, + {"suite", "", "t1", "suite/t1"}, + {"", "sec", "t1", "sec/t1"}, + {"", "", "t1", "t1"}, + } + for _, tc := range tests { + got := formatTestName(tc.file, tc.section, tc.test) + if got != tc.want { + t.Errorf("formatTestName(%q, %q, %q) = %q, want %q", tc.file, tc.section, tc.test, got, tc.want) + } + } + }) + + t.Run("DefaultConformanceSkipTests_defensive_copy", func(t *testing.T) { + s1 := DefaultConformanceSkipTests() + s2 := DefaultConformanceSkipTests() + if len(s1) == 0 { + t.Fatal("expected non-empty skip list") + } + s1[0] = "mutated_value" + if s2[0] == "mutated_value" { + t.Error("DefaultConformanceSkipTests did not return an independent defensive copy") + } + }) + + t.Run("parseConformanceBytes_error_diagnostics", func(t *testing.T) { + longInvalid := "invalid_syntax_key: {" + strings.Repeat("x", 500) + _, err := ParseConformanceFile(longInvalid) + if err == nil { + t.Fatal("expected error on invalid syntax, got nil") + } + errMsg := err.Error() + if len(errMsg) > 600 { + t.Errorf("error message unexpectedly large (%d bytes), expected bounded sample", len(errMsg)) + } + if !strings.Contains(errMsg, "truncated") { + t.Errorf("expected truncated notice in error message, got: %s", errMsg) + } + }) + + t.Run("jsonToValue_integer_bounds", func(t *testing.T) { + v, err := jsonToValue(float64(42)) + if err != nil { + t.Fatal(err) + } + if _, ok := v.Kind.(*expr.Value_Int64Value); !ok { + t.Errorf("expected Int64Value for whole number float, got %T", v.Kind) + } + + v2, err := jsonToValue(3.14) + if err != nil { + t.Fatal(err) + } + if _, ok := v2.Kind.(*expr.Value_DoubleValue); !ok { + t.Errorf("expected DoubleValue for fractional float, got %T", v2.Kind) + } + }) + + t.Run("FormatSimpleTestFile", func(t *testing.T) { + f := &testpb.SimpleTestFile{ + Name: "fmt_test", + Section: []*testpb.SimpleTestSection{ + { + Name: "s1", + Test: []*testpb.SimpleTest{ + {Name: "t1", Expr: "true"}, + }, + }, + }, + } + formatted, err := FormatSimpleTestFile(f) + if err != nil { + t.Fatalf("FormatSimpleTestFile failed: %v", err) + } + if !strings.Contains(formatted, "fmt_test") { + t.Errorf("expected formatted output to contain fmt_test, got: %s", formatted) + } + if _, err := FormatSimpleTestFile(nil); err == nil { + t.Error("expected error for nil SimpleTestFile, got nil") + } + }) + + t.Run("HasFailures", func(t *testing.T) { + oPass := &ConformanceOutput{Passed: 5, Failed: 0} + if oPass.HasFailures() { + t.Error("expected HasFailures() = false when Failed is 0") + } + oFail := &ConformanceOutput{Passed: 4, Failed: 1} + if !oFail.HasFailures() { + t.Error("expected HasFailures() = true when Failed > 0") + } + }) + + t.Run("EvaluateConformanceTest_nil", func(t *testing.T) { + _, err := EvaluateConformanceTest(nil, nil) + if err == nil { + t.Error("expected error evaluating nil SimpleTest, got nil") + } + }) + + t.Run("ParseConformanceFile_explicit_path_errors", func(t *testing.T) { + // Non-existent explicit file path + _, err := ParseConformanceFile("non_existent_file.textproto") + if err == nil { + t.Error("expected error for missing .textproto file, got nil") + } else if !strings.Contains(err.Error(), "failed reading conformance test file") { + t.Errorf("expected 'failed reading conformance test file' error, got: %v", err) + } + + // Directory path + tmpDir := t.TempDir() + _, err = ParseConformanceFile(tmpDir) + if err == nil { + t.Error("expected error for directory path, got nil") + } else if !strings.Contains(err.Error(), "is a directory") { + t.Errorf("expected 'is a directory' error, got: %v", err) + } + }) + + t.Run("valueToJSON_exhaustive", func(t *testing.T) { + tests := []struct { + name string + val *expr.Value + }{ + {"null", &expr.Value{Kind: &expr.Value_NullValue{}}}, + {"uint", &expr.Value{Kind: &expr.Value_Uint64Value{Uint64Value: 100}}}, + {"double", &expr.Value{Kind: &expr.Value_DoubleValue{DoubleValue: 2.718}}}, + {"bytes", &expr.Value{Kind: &expr.Value_BytesValue{BytesValue: []byte("bytes_test")}}}, + {"type", &expr.Value{Kind: &expr.Value_TypeValue{TypeValue: "int"}}}, + {"enum", &expr.Value{Kind: &expr.Value_EnumValue{EnumValue: &expr.EnumValue{Type: "test.Enum", Value: 1}}}}, + } + for _, tc := range tests { + jv, err := valueToJSON(tc.val) + if err != nil { + t.Errorf("%s: valueToJSON failed: %v", tc.name, err) + } + if tc.name != "null" && jv == nil { + t.Errorf("%s: expected non-nil result", tc.name) + } + } + }) +} + + diff --git a/skills/cel-conformance-testing/SKILL.md b/skills/cel-conformance-testing/SKILL.md new file mode 100644 index 0000000..69585ee --- /dev/null +++ b/skills/cel-conformance-testing/SKILL.md @@ -0,0 +1,209 @@ +--- +name: cel-conformance-testing +description: >- + Skill for authoring, evaluating, and contributing Common Expression Language (CEL) + conformance tests using the cel-spec textproto format. +--- + +# Common Expression Language (CEL) Conformance Testing Skill + +Use this skill to author, execute, validate, and publish CEL conformance tests. +Conformance tests define the formal behavioral specification of CEL across +implementations (cel-go, cel-cpp, cel-java). + +## Conformance Test Overview & Locations + +Conformance tests in CEL are defined in Protocol Buffer text format (`textproto`) +or Protocol Buffer JSON (`protojson`) using the protobuf schema defined in `cel-spec`: +`cel.expr.conformance.test.SimpleTestFile`. + +### Repository Location + +All standard conformance tests are maintained in the [cel-expr/cel-spec](https://github.com/cel-expr/cel-spec) +repository on GitHub under: +``` +tests/simple/testdata/ +``` + +Key test files include: +- `basic.textproto`: Fundamental self-evaluating forms and core operations. +- `comparisons.textproto`: Equality and order comparisons across numeric and complex types. +- `integer_math.textproto`: Arithmetic, overflows, and signed/unsigned math operations. +- `logic.textproto`: Short-circuiting logical operations (`&&`, `||`, `!`). +- `block_ext.textproto`: Scoped variable bindings via `cel.block`. +- `type_deduction.textproto`: Static type deduction and parameterized types. +- `proto2.textproto` & `proto3.textproto`: Protobuf field access, well-known types, and extensions. + +Every test file must begin with the protobuf header directives: +```textproto +# proto-file: ../../../proto/cel/expr/conformance/test/simple.proto +# proto-message: cel.expr.conformance.test.SimpleTestFile +``` + +## Structure & Mapping to Unit Tests + +A conformance test file (`SimpleTestFile`) organizes tests into sections: +- `SimpleTestFile`: High-level suite (`name`, `description`, repeated `section`). +- `SimpleTestSection`: Group of related cases (`name`, `description`, repeated `test`). +- `SimpleTest`: Individual test scenario. + +### Mapping between Unit Test and Conformance Test Structure + +The conformance test structure maps directly to CEL unit testing concepts (such as `cel-go/interpreter/interpreter_test.go`): + +| Unit Test Field | `SimpleTest` Mapping | Description | +| --------------- | -------------------- | ----------- | +| `expr` | `expr` | CEL expression under test | +| `container` | `container` | Namespace container for identifier resolution | +| `vars` (`VariableDecl`) | `type_env` entries | Type declarations for variables and functions | +| `in` (`any`) | `bindings` | Input variable bindings (`ExprValue`) | +| `out` (`any`) | `value` | Expected result value (`expr.Value`) | +| `err` (string) | `eval_error` | Expected evaluation error (`ErrorSet`) | +| `unchecked: true` | `disable_check: true` | Bypasses static type check to test runtime evaluation | +| - | `check_only: true` | Validates deduced output type against `typed_result` without evaluation | +| - | `disable_macros: true` | Disables macro expansion during parsing | + +### Example Conformance Test + +```textproto +name: "policy_conformance" +description: "Conformance tests for policy rule evaluation" +section { + name: "role_checks" + test { + name: "admin_access" + expr: "user.role == 'admin'" + type_env { + name: "user" + ident { + type { + message_type: "example.User" + } + } + } + bindings { + key: "user" + value { + value { + object_value { + [type.googleapis.com/example.User] { role: "admin" } + } + } + } + } + value { bool_value: true } + } +} +``` + +## What Makes a Good Conformance Test + +1. **Targeted & Minimal**: + - Each test case should exercise exactly one semantic rule or edge condition. + - Keep expressions as small as possible so test failures isolate the exact flaw. + +2. **Language-Agnostic Assertions**: + - Conformance tests must pass in Go, C++, Python, and Java runtimes (see [Target Runtimes Reference](references/target-runtimes.md)). + - For errors, use `eval_error` or `any_eval_errors`. Avoid asserting implementation-specific error messages unless verifying standardized error codes. + - For floating point values, account for IEEE-754 semantics (e.g. `NaN != NaN`). + +3. **Thorough Edge Case Coverage**: + - **Zero-ish and extremes**: `0`, `0u`, `0.0`, `-0.0`, empty strings, empty lists, empty maps, `int64.min`, `int64.max`. + - **Short-circuiting**: Verify that errors and unknowns in unreachable branches do not prevent evaluation (e.g. `false && (1/0 == 0)` evaluates to `false`). + - **Type Checking**: Use `check_only: true` and `typed_result { deduced_type { ... } }` to assert static type inference without needing runtime data. + - **Runtime Dynamic Checks**: Use `disable_check: true` to test dynamic evaluation behavior for undeclared variables or unsupported operations. + +4. **Self-Contained & Deterministic**: + - Explicitly declare all identifiers in `type_env` unless verifying undeclared identifier behavior. + - Supply all required input bindings in `bindings`. + +## Running Conformance Tests + +### Using the Suite Runner Script (`run-conformance.sh`) + +A dedicated test runner script is provided with this skill at `scripts/run-conformance.sh` to execute the CLI against all or filtered `cel-spec` test suites: + +```bash +# Run all cel-spec conformance tests (auto-detects testdata directory): +./skills/cel-conformance-testing/scripts/run-conformance.sh + +# Run only a specific test file: +./skills/cel-conformance-testing/scripts/run-conformance.sh --file basic.textproto + +# Filter across all suites by test name substring: +./skills/cel-conformance-testing/scripts/run-conformance.sh --filter int_zero + +# Specify an explicit testdata directory: +./skills/cel-conformance-testing/scripts/run-conformance.sh --dir /path/to/cel-spec/tests/simple/testdata +``` + +The script automatically applies standard `cel-go` skip rules, builds the CLI binary, executes tests across each file, and prints a summary report. + +### Using the CLI directly (`cel-expr conformance`) + +```bash +# Run an entire conformance test file +cel-expr conformance -tests path/to/suite.textproto + +# Filter to run a specific test or section +cel-expr conformance -tests path/to/suite.textproto -filter admin_access + +# Skip specific tests +cel-expr conformance -tests path/to/suite.textproto -skip prefix1,prefix2 + +# Provide external protobuf definitions (FileDescriptorSet) +cel-expr conformance -tests path/to/suite.textproto -fds path/to/descriptors.fds + +# Run via stdin +cat path/to/suite.textproto | cel-expr conformance -tests - +``` + +### Using the MCP Tool (`cel_evaluate_conformance`) + +Call the `cel_evaluate_conformance` tool with: +```json +{ + "tests": "path/to/tests.textproto", + "filter": "optional_name_filter" +} +``` + +Or pass inline textproto or protojson directly as the `tests` argument. + +## Publishing Conformance Tests to GitHub `cel-spec` + +Follow this workflow to contribute conformance tests to the upstream specification: + +1. **Fork and Clone**: + ```bash + git clone https://github.com//cel-spec + cd cel-spec + git checkout -b add-feature-conformance-tests + ``` + +2. **Locate or Create Test Files**: + - Add tests to an existing category file under `tests/simple/testdata/` (e.g., `tests/simple/testdata/string_ext.textproto`). + - If adding tests for a new extension or major capability, create a new file named `{feature}.textproto` or `{feature}_ext.textproto`. + +3. **Verify Header and Formatting**: + - Ensure the file begins with the required proto comments: + ```textproto + # proto-file: ../../../proto/cel/expr/conformance/test/simple.proto + # proto-message: cel.expr.conformance.test.SimpleTestFile + ``` + - Use 2-space indentation. + +4. **Validate against Runtimes**: + - Test locally using `cel-expr conformance`: + ```bash + cel-expr conformance -tests tests/simple/testdata/{feature}.textproto + ``` + - If Bazel is installed in `cel-spec`: + ```bash + bazel test //tests/simple:simple_test + ``` + - Validate across target runtimes as described in [Target Runtimes Reference](references/target-runtimes.md). + +5. **Submit a Pull Request**: + - Commit changes with a descriptive message referencing the relevant specification chapter or issue. + - Open a Pull Request against `cel-expr/cel-spec:master`. diff --git a/skills/cel-conformance-testing/references/target-runtimes.md b/skills/cel-conformance-testing/references/target-runtimes.md new file mode 100644 index 0000000..03286f6 --- /dev/null +++ b/skills/cel-conformance-testing/references/target-runtimes.md @@ -0,0 +1,66 @@ +# Running Conformance Tests across Target Runtimes + +CEL conformance tests define the behavioral specification of CEL and are validated against all core runtime implementations. Before contributing or proposing new conformance test cases to `cel-spec`, verify them across the respective language targets: + +## Go (`cel-go`) + +Tests can be evaluated using the local runner tooling provided in this repository or via Bazel in the [`cel-go`](https://github.com/cel-expr/cel-go) repository: + +- **Via local runner script** (using the `cel-expr` CLI against your local `cel-spec` directory): + ```bash + ./skills/cel-conformance-testing/scripts/run-conformance.sh --file .textproto + ``` +- **Via Bazel in `cel-go`**: + ```bash + # Run standard conformance test suite + bazel test //conformance:conformance + + # Run the full dashboard conformance suite + bazel test //conformance:conformance_dashboard + ``` + +## C++ (`cel-cpp`) + +In the [`cel-cpp`](https://github.com/cel-expr/cel-cpp) repository, conformance tests are executed via Bazel: + +```bash +# Run checked conformance tests +bazel test //conformance:conformance_checked + +# Run all conformance test targets +bazel test //conformance:... +``` + +## Java (`cel-java`) + +In the [`cel-java`](https://github.com/cel-expr/cel-java) repository, conformance tests are executed via Bazel: + +```bash +# Run the primary conformance test suite +bazel test //conformance/src/test/java/dev/cel/conformance:all + +# Run all conformance test packages +bazel test //conformance/... +``` + +## Python (`cel-python`) + +In [`cel-python`](https://github.com/cloud-custodian/cel-python), conformance tests are translated from `cel-spec` `.textproto` files into Gherkin feature scenarios using `tools/gherkinize.py` and run via `behave` or `pytest`: + +```bash +# Regenerate feature files from cel-spec textprotos +cd features && make all + +# Run conformance scenarios +behave + +# Or run via pytest +pytest +``` + +Also, in [`cel-expr/cel-python`](https://github.com/cel-expr/cel-python/blob/main/conformance/BUILD): + +```bash +bazel test //conformance:conformance_test +``` + diff --git a/skills/cel-conformance-testing/scripts/run-conformance.sh b/skills/cel-conformance-testing/scripts/run-conformance.sh new file mode 100755 index 0000000..bb27e00 --- /dev/null +++ b/skills/cel-conformance-testing/scripts/run-conformance.sh @@ -0,0 +1,343 @@ +#!/usr/bin/env bash +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" + +# Default skip list matching cel-go/conformance/BUILD.bazel +DEFAULT_SKIPS=( + "fields/qualified_identifier_resolution/map_key_float" + "fields/qualified_identifier_resolution/map_key_null" + "fields/qualified_identifier_resolution/map_value_repeat_key" + "fields/qualified_identifier_resolution/map_value_repeat_key_heterogeneous" + "timestamps/duration_converters/get_milliseconds" + "optionals/optionals/map_optional_select_has" + "string_ext/value_errors/indexof_out_of_range" + "string_ext/value_errors/lastindexof_out_of_range" + "enums/strong_proto2" + "enums/strong_proto3" + "type_deductions/wrappers/wrapper_promotion_2" + "type_deductions/legacy_nullable_types/null_assignable_to_message_parameter_candidate" + "type_deductions/legacy_nullable_types/null_assignable_to_duration_parameter_candidate" + "type_deductions/legacy_nullable_types/null_assignable_to_timestamp_parameter_candidate" + "type_deductions/legacy_nullable_types/null_assignable_to_abstract_parameter_candidate" +) +JOINED_DEFAULT_SKIPS=$(IFS=,; echo "${DEFAULT_SKIPS[*]}") + +usage() { + cat < Directory containing cel-spec .textproto files + (default: auto-detected from GOPATH or ../cel-spec) + -f, --filter Filter test cases by name substring + -s, --skip Test prefixes to skip (comma-separated) + --file Run only a specific test file (e.g. basic.textproto) + -b, --bin Path to pre-built cel-expr CLI binary + (default: builds a temporary binary via 'go build') + -v, --verbose Print verbose JSON output for each test file + -h, --help Show this help message and exit + +Examples: + # Run all conformance tests: + $(basename "$0") + + # Run only basic.textproto: + $(basename "$0") --file basic.textproto + + # Filter to a specific test name across all suites: + $(basename "$0") --filter int_zero + + # Specify a custom cel-spec testdata directory: + $(basename "$0") --dir /path/to/cel-spec/tests/simple/testdata +EOF +} + +# Locate testdata directory +find_testdata_dir() { + if [[ -n "${CEL_SPEC_TESTDATA:-}" && -d "${CEL_SPEC_TESTDATA}" ]]; then + echo "${CEL_SPEC_TESTDATA}" + return + fi + if [[ -n "${CEL_SPEC_DIR:-}" && -d "${CEL_SPEC_DIR}/tests/simple/testdata" ]]; then + echo "${CEL_SPEC_DIR}/tests/simple/testdata" + return + fi + + local candidates=( + "$(go env GOPATH 2>/dev/null || echo '')/src/github.com/cel-expr/cel-spec/tests/simple/testdata" + "${REPO_ROOT}/../cel-spec/tests/simple/testdata" + "${REPO_ROOT}/../../cel-expr/cel-spec/tests/simple/testdata" + ) + + for dir in "${candidates[@]}"; do + if [[ -n "${dir}" && -d "${dir}" ]]; then + echo "${dir}" + return + fi + done + + echo "" +} + +TESTDATA_DIR="" +FILTER="" +ADDITIONAL_SKIPS="" +SPECIFIC_FILE="" +CLI_BIN="" +VERBOSE=false + +while [[ $# -gt 0 ]]; do + case "$1" in + -d|--dir) + TESTDATA_DIR="$2" + shift 2 + ;; + -f|--filter) + FILTER="$2" + shift 2 + ;; + -s|--skip) + ADDITIONAL_SKIPS="$2" + shift 2 + ;; + --file) + SPECIFIC_FILE="$2" + shift 2 + ;; + -b|--bin) + CLI_BIN="$2" + shift 2 + ;; + -v|--verbose) + VERBOSE=true + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + if [[ -f "$1" ]]; then + SPECIFIC_FILE="$1" + elif [[ -d "$1" ]]; then + TESTDATA_DIR="$1" + else + echo "Unknown option or argument: $1" >&2 + usage + exit 1 + fi + shift + ;; + esac +done + +if [[ -z "${TESTDATA_DIR}" ]]; then + TESTDATA_DIR="$(find_testdata_dir)" +fi + +if [[ -z "${TESTDATA_DIR}" && -z "${SPECIFIC_FILE}" ]]; then + echo "Error: cel-spec testdata directory could not be located." >&2 + echo "Please specify with -d / --dir or set CEL_SPEC_TESTDATA." >&2 + exit 1 +fi + +# Build cel-expr CLI binary if not provided +TMP_DIR="" +if [[ -z "${CLI_BIN}" ]]; then + TMP_DIR="$(mktemp -d)" + trap 'rm -rf "${TMP_DIR}"' EXIT + CLI_BIN="${TMP_DIR}/cel-expr" + echo "Building cel-expr CLI binary..." + (cd "${REPO_ROOT}" && go build -o "${CLI_BIN}" ./cmd/cli) +elif [[ ! -x "${CLI_BIN}" ]]; then + echo "Error: Specified binary '${CLI_BIN}' does not exist or is not executable." >&2 + exit 1 +fi + +# Combine skip list +ALL_SKIPS="${JOINED_DEFAULT_SKIPS}" +if [[ -n "${ADDITIONAL_SKIPS}" ]]; then + ALL_SKIPS="${ALL_SKIPS},${ADDITIONAL_SKIPS}" +fi + +# Collect files to test +FILES=() +if [[ -n "${SPECIFIC_FILE}" ]]; then + if [[ -f "${SPECIFIC_FILE}" ]]; then + FILES+=("${SPECIFIC_FILE}") + elif [[ -f "${TESTDATA_DIR}/${SPECIFIC_FILE}" ]]; then + FILES+=("${TESTDATA_DIR}/${SPECIFIC_FILE}") + else + echo "Error: Test file '${SPECIFIC_FILE}' not found." >&2 + exit 1 + fi +else + while IFS= read -r f; do + FILES+=("$f") + done < <(find "${TESTDATA_DIR}" -maxdepth 1 -name "*.textproto" | sort) +fi + +if [[ ${#FILES[@]} -eq 0 ]]; then + echo "Error: No .textproto files found to test." >&2 + exit 1 +fi + +echo "=======================================================" +echo "CEL Conformance Test Evaluator" +echo "=======================================================" +if [[ -n "${TESTDATA_DIR}" ]]; then + echo "Testdata Dir: ${TESTDATA_DIR}" +fi +echo "Files Found: ${#FILES[@]}" +if [[ -n "${FILTER}" ]]; then + echo "Filter: ${FILTER}" +fi +echo "=======================================================" +echo "" + +TOTAL_FILES=0 +TOTAL_TESTS=0 +TOTAL_PASSED=0 +TOTAL_SKIPPED=0 +TOTAL_FAILED=0 +FAILED_FILES=() + +for file in "${FILES[@]}"; do + filename="$(basename "$file")" + + # Skip network_ext by default if testing all files (requires non-core IP extension) + if [[ -z "${SPECIFIC_FILE}" && "${filename}" == "network_ext.textproto" ]]; then + printf "%-35s %s\n" "${filename}:" "SKIPPED (requires non-core network extensions)" + continue + fi + + TOTAL_FILES=$((TOTAL_FILES + 1)) + + CMD=("${CLI_BIN}" "conformance" "-tests" "${file}" "-skip" "${ALL_SKIPS}") + if [[ -n "${FILTER}" ]]; then + CMD+=("-filter" "${FILTER}") + fi + + JSON_OUTPUT="" + STDERR_OUTPUT="" + EXIT_CODE=0 + STDERR_FILE="$(mktemp)" + if JSON_OUTPUT="$("${CMD[@]}" 2>"${STDERR_FILE}")"; then + EXIT_CODE=0 + else + EXIT_CODE=$? + fi + STDERR_OUTPUT="$(cat "${STDERR_FILE}")" + rm -f "${STDERR_FILE}" + + if [[ "${VERBOSE}" == true ]]; then + echo "--- ${filename} ---" + echo "${JSON_OUTPUT}" + if [[ -n "${STDERR_OUTPUT}" ]]; then + echo "${STDERR_OUTPUT}" >&2 + fi + fi + + # Parse JSON output using jq or python3 + FILE_TOTAL=0 + FILE_PASSED=0 + FILE_SKIPPED=0 + FILE_FAILED=0 + + if command -v jq >/dev/null 2>&1; then + FILE_TOTAL=$(echo "${JSON_OUTPUT}" | jq -r '.total // 0' 2>/dev/null || echo "0") + FILE_PASSED=$(echo "${JSON_OUTPUT}" | jq -r '.passed // 0' 2>/dev/null || echo "0") + FILE_SKIPPED=$(echo "${JSON_OUTPUT}" | jq -r '.skipped // 0' 2>/dev/null || echo "0") + FILE_FAILED=$(echo "${JSON_OUTPUT}" | jq -r '.failed // 0' 2>/dev/null || echo "0") + elif command -v python3 >/dev/null 2>&1; then + read -r FILE_TOTAL FILE_PASSED FILE_SKIPPED FILE_FAILED < <( + echo "${JSON_OUTPUT}" | python3 -c ' +import sys, json +try: + d = json.load(sys.stdin) + print(d.get("total", 0), d.get("passed", 0), d.get("skipped", 0), d.get("failed", 0)) +except Exception: + print("0 0 0 0") +' + ) + fi + + # Ensure numeric values + [[ "${FILE_TOTAL}" =~ ^[0-9]+$ ]] || FILE_TOTAL=0 + [[ "${FILE_PASSED}" =~ ^[0-9]+$ ]] || FILE_PASSED=0 + [[ "${FILE_SKIPPED}" =~ ^[0-9]+$ ]] || FILE_SKIPPED=0 + [[ "${FILE_FAILED}" =~ ^[0-9]+$ ]] || FILE_FAILED=0 + + TOTAL_TESTS=$((TOTAL_TESTS + FILE_TOTAL)) + TOTAL_PASSED=$((TOTAL_PASSED + FILE_PASSED)) + TOTAL_SKIPPED=$((TOTAL_SKIPPED + FILE_SKIPPED)) + TOTAL_FAILED=$((TOTAL_FAILED + FILE_FAILED)) + + if [[ ${FILE_FAILED} -gt 0 || ${EXIT_CODE} -ne 0 ]]; then + FAILED_FILES+=("${filename}") + printf "%-35s \033[31mFAIL\033[0m (%d total, %d passed, %d skipped, %d failed)\n" \ + "${filename}:" "${FILE_TOTAL}" "${FILE_PASSED}" "${FILE_SKIPPED}" "${FILE_FAILED}" + + # Print failure details + if command -v jq >/dev/null 2>&1; then + echo "${JSON_OUTPUT}" | jq -r '.results[]? | select(.status != "pass" and .status != "skipped") | " -> \(.name): \(.status)"' 2>/dev/null || true + elif command -v python3 >/dev/null 2>&1; then + echo "${JSON_OUTPUT}" | python3 -c ' +import sys, json +try: + d = json.load(sys.stdin) + for r in d.get("results", []): + if r.get("status") not in ("pass", "skipped"): + print(f" -> {r.get(\"name\")}: {r.get(\"status\")}") +except Exception: + pass +' 2>/dev/null || true + fi + if [[ -n "${STDERR_OUTPUT}" && ${FILE_TOTAL} -eq 0 ]]; then + echo " -> ${STDERR_OUTPUT}" >&2 + fi + else + printf "%-35s \033[32mPASS\033[0m (%d total, %d passed, %d skipped, %d failed)\n" \ + "${filename}:" "${FILE_TOTAL}" "${FILE_PASSED}" "${FILE_SKIPPED}" "${FILE_FAILED}" + fi +done + +echo "" +echo "=======================================================" +echo "CEL CONFORMANCE SUMMARY" +echo "=======================================================" +echo "Files Evaluated: ${TOTAL_FILES}" +echo "Total Tests: ${TOTAL_TESTS}" +echo "Passed: ${TOTAL_PASSED}" +echo "Skipped: ${TOTAL_SKIPPED}" +echo "Failed: ${TOTAL_FAILED}" +echo "=======================================================" + +if [[ ${TOTAL_FAILED} -gt 0 || ${#FAILED_FILES[@]} -gt 0 ]]; then + echo "" + echo "FAILED SUITES: ${FAILED_FILES[*]}" + exit 1 +else + echo "" + echo "ALL CONFORMANCE TESTS PASSED!" + exit 0 +fi diff --git a/skills/cel-testing/SKILL.md b/skills/cel-testing/SKILL.md index fa51781..528989a 100644 --- a/skills/cel-testing/SKILL.md +++ b/skills/cel-testing/SKILL.md @@ -2,7 +2,7 @@ name: cel-testing description: >- Skill for testing Google Common Expression Language (CEL) expressions. - Use to test or validate an existing CEL rule. + Use to test or validate expressions. --- # Google Common Expression Language (CEL) Testing Skill