CEL Conformance runner

CLAUDE.md

@@ -14,8 +14,11 @@ Expression Language (CEL) expressions.
 -   `skills/`: Agent skills directory containing guidelines and workflows:
     -   `cel-authoring`: Authoring, configuring, and testing expressions and
         JSON environment definitions.
+    -   `cel-testing`: Testing and validating expressions against test cases.
     -   `cel-debugging`: Diagnosing and resolving compilation and evaluation
         errors.
+    -   `cel-conformance-testing`: Authoring, evaluating, and publishing CEL
+        conformance tests using the cel-spec textproto format.
 
 ## Build and Test Commands
 
@@ -43,6 +46,8 @@ The `cel-expr-mcp` MCP server provides the following tools for AI agents:
 -   `cel_compile`: Compiles a CEL expression to validate syntax, correctness,
     and type checking against an environment definition.
 -   `cel_evaluate`: Evaluates a compiled expression against provided test cases.
+-   `cel_evaluate_conformance`: Evaluates CEL conformance tests from a cel-spec
+    textproto file or inline content.
 
 ### Startup Flags
 
@@ -60,8 +65,11 @@ To understand how to best use these tools, refer to the skills in `skills/`:
 
 -   `cel-authoring`: Guidelines for creating CEL environment configurations,
     generating expressions, and compiling.
+-   `cel-testing`: Writing and evaluating unit test suites for CEL expressions.
 -   `cel-debugging`: Diagnostic steps for troubleshooting syntax, type, and
     evaluation errors.
+-   `cel-conformance-testing`: Executing and authoring conformance test suites from
+    `cel-spec` textproto definitions.
 
 ## Release & Supply Chain Security
 

CONTRIBUTING.md

@@ -63,4 +63,4 @@ asking for a new review. There is no commitment to resolution -- merging
 or closing a pull request, or fixing or closing an issue -- because some
 issues will require more discussion than others.
 
-[1]:  https://github.com/google/cel-spec
+[1]:  https://github.com/cel-expr/cel-spec

MODULE.bazel

@@ -20,10 +20,15 @@ go_deps.gazelle_default_attributes(
         "gazelle:proto disable_global",
     ],
 )
+go_deps.gazelle_override(
+    build_file_generation = "off",
+    path = "cel.dev/expr",
+)
 go_deps.from_file(go_mod = "//:go.mod")
 use_repo(
     go_deps,
     "com_github_modelcontextprotocol_go_sdk",
     "dev_cel_cel_go",
+    "dev_cel_expr",
     "org_golang_google_protobuf",
 )

cmd/cel-expr/main.go

@@ -39,6 +39,7 @@ Usage:
 Available Commands:
   compile     Compile a CEL expression against an environment configuration
   eval        Evaluate a CEL expression against test cases or bindings
+  conformance Evaluate CEL conformance tests from cel-spec textproto files
   env         Validate or inspect an environment configuration
   prompt      Generate an LLM authoring prompt from an environment and requirement
   mcp         Start the Model Context Protocol (MCP) server over stdio
@@ -68,6 +69,8 @@ func run(args []string, stdout, stderr io.Writer, stdin io.Reader) error {
 		return runCompile(cmdArgs, stdout, stderr, stdin)
 	case "eval", "evaluate":
 		return runEval(cmdArgs, stdout, stderr, stdin)
+	case "conformance":
+		return runConformance(cmdArgs, stdout, stderr, stdin)
 	case "env":
 		return runEnv(cmdArgs, stdout, stderr, stdin)
 	case "prompt":
@@ -189,11 +192,12 @@ func runEval(args []string, stdout, stderr io.Writer, stdin io.Reader) error {
 	exprFlag := fs.String("expr", "", "CEL expression string")
 	testsFlag := fs.String("tests", "", "Path to test cases JSON file or inline JSON array")
 	fs.StringVar(testsFlag, "test_cases", "", "Alias for -tests")
+	conformanceFlag := fs.String("conformance", "", "Path to conformance textproto file or inline textproto content")
 	bindingsFlag := fs.String("bindings", "", "Single test case variable bindings JSON file or inline JSON object")
 	expectedFlag := fs.String("expected", "", "Expected output value for single test case binding (JSON literal or string)")
 
 	fs.Usage = func() {
-		fmt.Fprintln(stderr, "Usage: cel-expr eval [-env <path|json>] [-tests <path|json>] [-bindings <path|json>] [expression]")
+		fmt.Fprintln(stderr, "Usage: cel-expr eval [-env <path|json>] [-tests <path|json>] [-conformance <path|textproto>] [-bindings <path|json>] [expression]")
 		fmt.Fprintln(stderr, "\nFlags:")
 		fs.PrintDefaults()
 	}
@@ -209,6 +213,21 @@ func runEval(args []string, stdout, stderr io.Writer, stdin io.Reader) error {
 	if expr == "" && fs.NArg() > 0 {
 		expr = strings.Join(fs.Args(), " ")
 	}
+
+	if *conformanceFlag != "" {
+		conformanceArgs := []string{"-tests", *conformanceFlag}
+		if *envFlag != "" {
+			conformanceArgs = append(conformanceArgs, "-env", *envFlag)
+		}
+		if *fdsFlag != "" {
+			conformanceArgs = append(conformanceArgs, "-fds", *fdsFlag)
+		}
+		if expr != "" {
+			conformanceArgs = append(conformanceArgs, "-expr", expr)
+		}
+		return runConformance(conformanceArgs, stdout, stderr, stdin)
+	}
+
 	if expr == "" {
 		return errors.New("expression is required (use -expr <expr> or pass as argument)")
 	}
@@ -293,6 +312,86 @@ func runEval(args []string, stdout, stderr io.Writer, stdin io.Reader) error {
 	return nil
 }
 
+func runConformance(args []string, stdout, stderr io.Writer, stdin io.Reader) error {
+	fs := flag.NewFlagSet("conformance", flag.ContinueOnError)
+	fs.SetOutput(stderr)
+	envFlag, fdsFlag := parseCommonEnvFlags(fs)
+	testsFlag := fs.String("tests", "", "Path to conformance textproto file or inline textproto content")
+	filterFlag := fs.String("filter", "", "Optional substring filter for test names")
+	skipFlag := fs.String("skip", "", "Comma-separated test prefixes to skip")
+	exprFlag := fs.String("expr", "", "Optional fallback expression if test does not define one")
+
+	fs.Usage = func() {
+		fmt.Fprintln(stderr, "Usage: cel-expr conformance [-tests <path|textproto>] [-filter <name>] [-skip <prefixes>] [-env <path|json>] [-fds <path>] [tests]")
+		fmt.Fprintln(stderr, "\nFlags:")
+		fs.PrintDefaults()
+	}
+
+	if err := fs.Parse(args); err != nil {
+		if errors.Is(err, flag.ErrHelp) {
+			return nil
+		}
+		return err
+	}
+
+	testsVal := *testsFlag
+	if testsVal == "" && fs.NArg() > 0 {
+		testsVal = strings.Join(fs.Args(), " ")
+	}
+	if testsVal == "" {
+		return errors.New("conformance tests are required (use -tests <path|textproto> or pass as argument)")
+	}
+
+	rawTests, err := loadContentOrStdin(testsVal, stdin)
+	if err != nil {
+		return fmt.Errorf("reading conformance tests: %w", err)
+	}
+
+	var cfg *tools.Config
+	var opts []cel.EnvOption
+	if *envFlag != "" || *fdsFlag != "" {
+		var err error
+		cfg, opts, err = loadEnvAndOpts(*envFlag, *fdsFlag, stdin, false)
+		if err != nil {
+			return err
+		}
+	}
+
+	var skipTests []string
+	if *skipFlag != "" {
+		for _, s := range strings.Split(*skipFlag, ",") {
+			s = strings.TrimSpace(s)
+			if s != "" {
+				skipTests = append(skipTests, s)
+			}
+		}
+	}
+
+	res, err := tools.EvaluateConformanceWithParams(tools.ConformanceParams{
+		Tests:        rawTests,
+		Filter:       *filterFlag,
+		SkipTests:    skipTests,
+		EnvConfig:    cfg,
+		FallbackExpr: *exprFlag,
+	}, opts...)
+	if err != nil {
+		return fmt.Errorf("conformance evaluation failed: %w", err)
+	}
+
+	outBytes, err := json.MarshalIndent(res, "", "  ")
+	if err != nil {
+		return fmt.Errorf("failed formatting output JSON: %w", err)
+	}
+
+	fmt.Fprintln(stdout, string(outBytes))
+
+	if res.HasFailures() {
+		return fmt.Errorf("%d of %d conformance tests failed", res.Failed, res.Total)
+	}
+
+	return nil
+}
+
 func runEnv(args []string, stdout, stderr io.Writer, stdin io.Reader) error {
 	fs := flag.NewFlagSet("env", flag.ContinueOnError)
 	fs.SetOutput(stderr)

cmd/cel-expr/main_test.go

@@ -41,6 +41,7 @@ func TestCLI_UsageAndHelp(t *testing.T) {
 		{"eval help", []string{"help", "eval"}},
 		{"env help", []string{"help", "env"}},
 		{"prompt help", []string{"help", "prompt"}},
+		{"conformance help", []string{"help", "conformance"}},
 		{"mcp help", []string{"help", "mcp"}},
 	}
 
@@ -342,3 +343,79 @@ func TestCLI_MCP_Errors(t *testing.T) {
 		t.Error("expected error when starting MCP server with invalid env, got nil")
 	}
 }
+
+func TestCLI_Conformance(t *testing.T) {
+	textprotoContent := `
+name: "cli_conformance"
+section {
+  name: "s1"
+  test {
+    name: "t1"
+    expr: "1 + 2"
+    value { int64_value: 3 }
+  }
+  test {
+    name: "t2"
+    expr: "3 * 4"
+    value { int64_value: 12 }
+  }
+}
+`
+	tmpFile := filepath.Join(t.TempDir(), "tests.textproto")
+	if err := os.WriteFile(tmpFile, []byte(textprotoContent), 0644); err != nil {
+		t.Fatal(err)
+	}
+
+	tests := []struct {
+		name    string
+		args    []string
+		wantErr bool
+	}{
+		{
+			name:    "conformance with -tests flag",
+			args:    []string{"conformance", "-tests", tmpFile},
+			wantErr: false,
+		},
+		{
+			name:    "conformance with positional argument",
+			args:    []string{"conformance", tmpFile},
+			wantErr: false,
+		},
+		{
+			name:    "conformance with -filter flag",
+			args:    []string{"conformance", "-tests", tmpFile, "-filter", "t1"},
+			wantErr: false,
+		},
+		{
+			name:    "eval with -conformance flag alias",
+			args:    []string{"eval", "-conformance", tmpFile},
+			wantErr: false,
+		},
+		{
+			name:    "eval with -conformance and positional expr",
+			args:    []string{"eval", "-conformance", tmpFile, "1 + 2"},
+			wantErr: false,
+		},
+		{
+			name:    "conformance missing tests",
+			args:    []string{"conformance"},
+			wantErr: true,
+		},
+		{
+			name:    "conformance failing test",
+			args:    []string{"conformance", `name:"fail" section { name:"s" test { name:"t" expr:"1" value { int64_value: 2 } } }`},
+			wantErr: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var stdout, stderr bytes.Buffer
+			err := run(tt.args, &stdout, &stderr, nil)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("run(%v) error = %v, wantErr %v (stdout: %s, stderr: %s)", tt.args, err, tt.wantErr, stdout.String(), stderr.String())
+			}
+		})
+	}
+}
+

go.mod

@@ -4,12 +4,12 @@ go 1.25.0
 
 require (
 	cel.dev/cel-go v0.32.0
+	cel.dev/expr v0.25.3
 	github.com/modelcontextprotocol/go-sdk v1.6.0
 	google.golang.org/protobuf v1.36.10
 )
 
 require (
-	cel.dev/expr v0.25.1 // indirect
 	github.com/antlr4-go/antlr/v4 v4.13.1 // indirect
 	github.com/google/jsonschema-go v0.4.3 // indirect
 	github.com/segmentio/asm v1.1.3 // indirect

go.sum

@@ -1,7 +1,7 @@
 cel.dev/cel-go v0.32.0 h1:irvpFKr5EuGPyxeME03ERh0rii1TX+BDAnB9eL3IvNk=
 cel.dev/cel-go v0.32.0/go.mod h1:DnVip7tpJSsgZymwfT+m1tnEVy3ivAjSMXPx12YrMkU=
-cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4=
-cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4=
+cel.dev/expr v0.25.3 h1:A2jO8jwOugrrovveCWfj0KEZOfqiLgAcwjpHPhzIGw0=
+cel.dev/expr v0.25.3/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4=
 github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ=
 github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw=
 github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY=

internal/mcp/server.go

@@ -115,6 +115,10 @@ func NewServer(fixedEnv *tools.Config, opts ...cel.EnvOption) *mcp.Server {
 			Name:        "cel_evaluate",
 			Description: "Evaluates a CEL expression against provided test cases. Returns test case results and coverage.",
 		}, h.handleFixedEnvEvaluate)
+		mcp.AddTool[FixedEnvEvaluateConformanceArgs, any](s, &mcp.Tool{
+			Name:        "cel_evaluate_conformance",
+			Description: "Evaluates CEL conformance tests from a cel-spec textproto file or inline content. Returns test case results and status.",
+		}, h.handleFixedEnvEvaluateConformance)
 	} else {
 		mcp.AddTool[CreateEnvConfigArgs, any](s, &mcp.Tool{
 			Name:        "cel_create_environment",
@@ -132,6 +136,10 @@ func NewServer(fixedEnv *tools.Config, opts ...cel.EnvOption) *mcp.Server {
 			Name:        "cel_evaluate",
 			Description: "Evaluates a CEL expression against provided test cases. Returns test case results and coverage.",
 		}, h.handleEvaluate)
+		mcp.AddTool[EvaluateConformanceArgs, any](s, &mcp.Tool{
+			Name:        "cel_evaluate_conformance",
+			Description: "Evaluates CEL conformance tests from a cel-spec textproto file or inline content. Returns test case results and status.",
+		}, h.handleEvaluateConformance)
 	}
 
 	return s
@@ -177,6 +185,23 @@ type FixedEnvEvaluateArgs struct {
 	TestCases []tools.TestCase `json:"testCases" jsonschema_description:"The test cases for evaluation."`
 }
 
+// EvaluateConformanceArgs is the arguments for the cel_evaluate_conformance tool.
+type EvaluateConformanceArgs struct {
+	Tests        string        `json:"tests" jsonschema_description:"Path to conformance textproto file or inline textproto content."`
+	Filter       string        `json:"filter,omitempty" jsonschema_description:"Optional substring filter for test names."`
+	SkipTests    []string      `json:"skipTests,omitempty" jsonschema_description:"Optional list of test name prefixes to skip."`
+	EnvConfig    *tools.Config `json:"envConfig,omitempty" jsonschema_description:"Optional JSON environment configuration."`
+	FallbackExpr string        `json:"fallbackExpr,omitempty" jsonschema_description:"Optional fallback CEL expression."`
+}
+
+// FixedEnvEvaluateConformanceArgs is the arguments for the cel_evaluate_conformance tool when a static environment is configured.
+type FixedEnvEvaluateConformanceArgs struct {
+	Tests        string   `json:"tests" jsonschema_description:"Path to conformance textproto file or inline textproto content."`
+	Filter       string   `json:"filter,omitempty" jsonschema_description:"Optional substring filter for test names."`
+	SkipTests    []string `json:"skipTests,omitempty" jsonschema_description:"Optional list of test name prefixes to skip."`
+	FallbackExpr string   `json:"fallbackExpr,omitempty" jsonschema_description:"Optional fallback CEL expression."`
+}
+
 func (h *toolsHandler) handleCreateEnvConfig(ctx context.Context, request *mcp.CallToolRequest, args CreateEnvConfigArgs) (*mcp.CallToolResult, any, error) {
 	_, err := tools.EnvFromConfig(args.EnvConfig, h.opts...)
 	if err != nil {
@@ -247,3 +272,31 @@ func (h *toolsHandler) handleFixedEnvEvaluate(ctx context.Context, request *mcp.
 		TestCases: args.TestCases,
 	})
 }
+
+func (h *toolsHandler) handleEvaluateConformance(ctx context.Context, request *mcp.CallToolRequest, args EvaluateConformanceArgs) (*mcp.CallToolResult, any, error) {
+	envConfig := args.EnvConfig
+	if envConfig == nil {
+		envConfig = h.fixedEnv
+	}
+	res, err := tools.EvaluateConformanceWithParams(tools.ConformanceParams{
+		Tests:        args.Tests,
+		Filter:       args.Filter,
+		SkipTests:    args.SkipTests,
+		EnvConfig:    envConfig,
+		FallbackExpr: args.FallbackExpr,
+	}, h.opts...)
+	if err != nil {
+		return nil, nil, err
+	}
+	return nil, res, nil
+}
+
+func (h *toolsHandler) handleFixedEnvEvaluateConformance(ctx context.Context, request *mcp.CallToolRequest, args FixedEnvEvaluateConformanceArgs) (*mcp.CallToolResult, any, error) {
+	return h.handleEvaluateConformance(ctx, request, EvaluateConformanceArgs{
+		Tests:        args.Tests,
+		Filter:       args.Filter,
+		SkipTests:    args.SkipTests,
+		FallbackExpr: args.FallbackExpr,
+	})
+}
+