Source enum values from external proto enums and reference external enum types

config/site/support/doctest_helper.rb

@@ -104,6 +104,37 @@ module ElasticGraph
       end
     end
 
+    doctest.before "ElasticGraph::ProtoIngestion::SchemaDefinition::SchemaElements::EnumTypeExtension#external_proto_enum" do
+      # The examples source enum values from an app-defined proto enum class; provide one here.
+      # We define real constants instead of using rspec-mocks stub_const, because the mock
+      # lifecycle (setup/verify/teardown) is not run for individual doctests and the stubs
+      # would leak to every later doctest in the process.
+      proto_enum_entry = ::Data.define(:name, :number)
+
+      ::Object.const_set(:MyApp, ::Module.new) unless defined?(MyApp)
+      MyApp.const_set(:Protos, ::Module.new) unless MyApp.const_defined?(:Protos)
+
+      MyApp::Protos.const_set(:Currency, ::Class.new)
+      MyApp::Protos::Currency.define_singleton_method(:enums) do
+        [:CURRENCY_UNKNOWN_DO_NOT_USE, :CURRENCY_USD, :CURRENCY_CAD].each_with_index.map do |name, number|
+          proto_enum_entry.new(name: name, number: number)
+        end
+      end
+
+      # Its names match an ElasticGraph enum exactly, so it can be referenced rather than sourced.
+      MyApp::Protos.const_set(:CurrencyCode, ::Class.new)
+      MyApp::Protos::CurrencyCode.define_singleton_method(:enums) do
+        [proto_enum_entry.new(name: :USD, number: 1), proto_enum_entry.new(name: :CAD, number: 2)]
+      end
+    end
+
+    doctest.after "ElasticGraph::ProtoIngestion::SchemaDefinition::SchemaElements::EnumTypeExtension#external_proto_enum" do
+      MyApp::Protos.send(:remove_const, :Currency)
+      MyApp::Protos.send(:remove_const, :CurrencyCode)
+      MyApp.send(:remove_const, :Protos)
+      ::Object.send(:remove_const, :MyApp)
+    end
+
     doctest.before "ElasticGraph::SchemaDefinition::SchemaElements::ScalarType#coerce_with" do
       ::FileUtils.mkdir_p "coercion_adapters"
       ::File.write("coercion_adapters/phone_number.rb", <<~EOS)

elasticgraph-proto_ingestion/README.md

@@ -218,6 +218,108 @@ Each call to `protobuf` replaces the full protobuf configuration. The override a
 `import:`, so `schema.proto` no longer imports `google/protobuf/timestamp.proto`. An override that
 omits `field_comment:` likewise drops the built-in comment.
 
+### Sourcing Enum Values From Existing Protobuf Enums
+
+If your project already has canonical protobuf enum definitions, you can source an enum's
+generated proto values from them instead of maintaining the value list in two places. Call
+`external_proto_enum` on the enum type with a proto enum class (anything exposing `.enums`).
+In the examples below, this stand-in plays the role of your app's generated proto enum class:
+
+```ruby
+# in config/schema/app_protos.rb
+
+# Stands in for a proto enum class generated by your app's protobuf tooling.
+module MyApp
+  module Protos
+    EnumEntry = ::Data.define(:name, :number)
+
+    # Its value names carry a prefix and include a sentinel, so it needs a transform and an
+    # exclusion to line up with an ElasticGraph enum.
+    class Currency
+      def self.enums
+        [
+          EnumEntry.new(name: :CURRENCY_UNKNOWN_DO_NOT_USE, number: 0),
+          EnumEntry.new(name: :CURRENCY_USD, number: 1),
+          EnumEntry.new(name: :CURRENCY_CAD, number: 2)
+        ]
+      end
+    end
+
+    # Its value names already match the ElasticGraph enum exactly, so it can be referenced
+    # directly rather than regenerated locally.
+    class CurrencyCode
+      def self.enums
+        [
+          EnumEntry.new(name: :USD, number: 1),
+          EnumEntry.new(name: :CAD, number: 2)
+        ]
+      end
+    end
+  end
+end
+```
+
+Optional per-source options curate and transform the sourced values:
+
+```ruby
+# in config/schema/currency.rb
+
+ElasticGraph.define_schema do |schema|
+  schema.enum_type "Currency" do |t|
+    t.values "USD", "CAD"
+    t.external_proto_enum MyApp::Protos::Currency,
+      # Proto values to omit from the generated enum.
+      exclusions: [:UNKNOWN_DO_NOT_USE],
+      # Values expected in the generated enum that the proto enum lacks.
+      expected_extras: [:LEGACY],
+      # Optional transform applied to each proto value name.
+      name_transform: ->(name) { name.delete_prefix("CURRENCY_") }
+  end
+end
+```
+
+`name_transform` runs first, and `exclusions` and `expected_extras` then apply to the
+transformed names. In the example above the exclusion is therefore `UNKNOWN_DO_NOT_USE`, the
+name left after the transform strips `CURRENCY_`, rather than `CURRENCY_UNKNOWN_DO_NOT_USE`.
+
+When an enum has one or more external sources, `elasticgraph-proto_ingestion` uses them
+as the source of the generated enum's values. When multiple sources are registered for the
+same enum, they must all resolve to the same value set.
+
+### Referencing Existing Protobuf Types
+
+For enums that exactly match a canonical proto enum, you can go further and reference the
+existing proto type instead of generating a duplicate local enum. Pass `proto:` and `import:`
+to `external_proto_enum`, and `schema.proto` will import the named file and use the external
+type name directly:
+
+```ruby
+# in config/schema/currency.rb
+
+ElasticGraph.define_schema do |schema|
+  schema.enum_type "CurrencyCode" do |t|
+    t.values "USD", "CAD"
+    t.external_proto_enum MyApp::Protos::CurrencyCode,
+      proto: "myapp.types.CurrencyCode",
+      import: "myapp/types/currency_code.proto"
+  end
+end
+```
+
+A referenced enum must have exactly one `external_proto_enum` call that passes no
+`exclusions:`, `expected_extras:`, or `name_transform:`, whose values match the
+ElasticGraph enum's values; transformed, curated, or multi-source enums stay generated
+locally. Note that `MyApp::Protos::Currency` from the
+previous section cannot be referenced this way: its `CURRENCY_`-prefixed names only match
+after a transform, and referenced enums allow no transform.
+
+The source's enum entries must also expose `.number`. Those numbers are recorded in
+`proto_field_numbers.yaml`, and must agree with any numbers already pinned there — otherwise
+switching to the external type would silently reinterpret existing wire data. Recording them
+also means that dropping `proto:`/`import:` later regenerates the enum locally with its
+original numbers rather than renumbering it. No referenced value may use number 0, which is
+reserved for the zero value this gem generates for a local enum.
+
 ## Type Mappings
 
 The generated `schema.proto` uses these built-in scalar mappings:

elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/field_number_mappings.rb

@@ -253,6 +253,43 @@ def reserved_enum_value_numbers_for(enum_name, active_value_names)
           reserved_numbers_by_name(value_numbers, active_value_names)
         end
 
+        # Returns the previously pinned numbers for a protobuf enum.
+        #
+        # @param enum_name [String]
+        # @return [Hash<String, Integer>]
+        def pinned_enum_value_numbers(enum_name)
+          @enum_mappings_by_name[enum_name]&.value_numbers_by_name || {}
+        end
+
+        # Records numbers an external proto enum already owns, verbatim.
+        #
+        # Unlike {#enum_value_numbers_for}, this allocates nothing -- the external enum is the
+        # authority on its own numbers. Recording them keeps the artifact a complete record of the
+        # wire format, so that an enum which later stops being referenced externally is generated
+        # locally with the same numbers instead of being silently renumbered from 1.
+        #
+        # @param enum_name [String]
+        # @param numbers_by_name [Hash<String, Integer>]
+        # @return [void]
+        def pin_enum_value_numbers(enum_name, numbers_by_name)
+          numbers_by_name.each do |value_name, number|
+            unless number.is_a?(::Integer) && number >= 0 && number <= MAX_ENUM_VALUE_NUMBER
+              raise Errors::SchemaError, "External proto enum `#{enum_name}` assigns `#{value_name}` the number " \
+                "#{number.inspect}, which is not a valid protobuf enum value number (0..#{MAX_ENUM_VALUE_NUMBER})."
+            end
+          end
+
+          enum_mapping = enum_mapping_for(enum_name)
+          updated_value_numbers = enum_mapping.value_numbers_by_name.merge(numbers_by_name)
+          @enum_mappings_by_name = @enum_mappings_by_name.merge(
+            enum_name => enum_mapping.with(
+              value_numbers_by_name: updated_value_numbers,
+              next_number: [enum_mapping.next_number, updated_value_numbers.values.max.to_i + 1].max
+            )
+          )
+          nil
+        end
+
         # Serializes the mappings back to the `proto_field_numbers.yaml` artifact format, with
         # messages and enums sorted by name and their fields and values sorted by number.
         #

elasticgraph-proto_ingestion/lib/elastic_graph/proto_ingestion/schema_definition/schema.rb

@@ -112,13 +112,16 @@ def field_number_for(message_name:, type_name:, public_field_name:)
         end
 
         # @dynamic next_field_number_for, reserved_field_numbers_for, enum_value_numbers_for
-        # @dynamic next_enum_value_number_for, reserved_enum_value_numbers_for
+        # @dynamic next_enum_value_number_for, reserved_enum_value_numbers_for, pinned_enum_value_numbers
+        # @dynamic pin_enum_value_numbers
         def_delegators :@field_number_mappings,
           :next_field_number_for,
           :reserved_field_numbers_for,
           :enum_value_numbers_for,
           :next_enum_value_number_for,
-          :reserved_enum_value_numbers_for
+          :reserved_enum_value_numbers_for,
+          :pinned_enum_value_numbers,
+          :pin_enum_value_numbers
 
         # Returns the label prefix (including its trailing space) that a field declaration needs
         # under the configured syntax, or an empty string when the field takes no label.