Extract indexing field metadata behind a format-neutral value object

elasticgraph-indexer/lib/elastic_graph/indexer/record_preparer.rb

@@ -26,8 +26,8 @@ def self.prepare_for_index(type_name, value, mapping_properties)
 
       def initialize(indexing_preparer_by_scalar_type_name, type_metas)
         @indexing_preparer_by_scalar_type_name = indexing_preparer_by_scalar_type_name
-        @eg_meta_by_field_name_by_concrete_type = type_metas.to_h do |meta|
-          [meta.name, meta.eg_meta_by_field_name]
+        @fields_by_name_by_concrete_type = type_metas.to_h do |meta|
+          [meta.name, meta.fields_by_name]
         end
       end
 
@@ -59,12 +59,12 @@ def prepare_for_index(type_name, value, mapping_properties)
           element_type_name = type_name.delete_prefix("[").delete_suffix("]")
           value.map { |v| prepare_for_index(element_type_name, v, mapping_properties) }
         when ::Hash
-          # `@eg_meta_by_field_name_by_concrete_type` does not have abstract types in it (e.g. type unions).
+          # `@fields_by_name_by_concrete_type` does not have abstract types in it (e.g. type unions).
           # Instead, it'll have each concrete subtype in it.
           #
           # If `type_name` is an abstract type, we need to look at the `__typename` field to see
           # what the concrete subtype is. `__typename` is required on abstract types and indicates that.
-          eg_meta_by_field_name = @eg_meta_by_field_name_by_concrete_type.fetch(value["__typename"] || type_name)
+          fields_by_name = @fields_by_name_by_concrete_type.fetch(value["__typename"] || type_name)
 
           # We only want to consider __typename if it's in the per-record mapping in order to determine
           # whether __typename is required on records. When it's a constant_keyword it exists at the index
@@ -76,10 +76,10 @@ def prepare_for_index(type_name, value, mapping_properties)
             if field_name == "__typename"
               # Only include __typename if the index mapping has it at this position.
               [field_name, field_value] if typename_in_record_mapping
-            elsif (eg_meta = eg_meta_by_field_name[field_name])
-              name_in_index = eg_meta.fetch("nameInIndex")
+            elsif (field = fields_by_name[field_name])
+              name_in_index = field.name_in_index
               nested_mapping_properties = mapping_properties&.dig(name_in_index, "properties")
-              [name_in_index, prepare_for_index(eg_meta.fetch("type"), field_value, nested_mapping_properties)]
+              [name_in_index, prepare_for_index(field.type, field_value, nested_mapping_properties)]
             end
           end.to_h
 
@@ -98,12 +98,24 @@ def prepare_for_index(type_name, value, mapping_properties)
         end
       end
 
-      TypeMetadata = ::Data.define(
-        # The name of the type this metadata object is for.
-        :name,
-        # The per-field ElasticGraph metadata, keyed by field name.
-        :eg_meta_by_field_name
-      )
+      # Ingestion-format-neutral metadata about a single indexed field. Ingestion adapters build
+      # these from whatever source describes their format (e.g. `elasticgraph-json_ingestion`
+      # derives them from the versioned JSON schemas), so the indexer itself does not need to know
+      # how any particular ingestion format describes its fields.
+      #
+      # @!attribute [r] type
+      #   @return [String] name of the ElasticGraph type of this field
+      # @!attribute [r] name_in_index
+      #   @return [String] name of this field in the index
+      FieldMetadata = ::Data.define(:type, :name_in_index)
+
+      # Ingestion-format-neutral metadata about a single indexed type.
+      #
+      # @!attribute [r] name
+      #   @return [String] the name of the type this metadata object is for
+      # @!attribute [r] fields_by_name
+      #   @return [Hash<String, FieldMetadata>] metadata for each of the type's fields, keyed by field name
+      TypeMetadata = ::Data.define(:name, :fields_by_name)
     end
   end
 end

elasticgraph-indexer/sig/elastic_graph/indexer/record_preparer.rbs

@@ -5,8 +5,8 @@ module ElasticGraph
     end
 
     class RecordPreparer
-      type egMetaByFieldHash = ::Hash[::String, {"type" => ::String, "nameInIndex" => ::String}]
-      type egMetaByFieldByTypeHash = ::Hash[::String, egMetaByFieldHash]
+      type fieldsByNameHash = ::Hash[::String, FieldMetadata]
+      type fieldsByNameByTypeHash = ::Hash[::String, fieldsByNameHash]
 
       include _RecordPreparer
 
@@ -20,20 +20,35 @@ module ElasticGraph
       ) -> void
 
       @indexing_preparer_by_scalar_type_name: ::Hash[::String, SchemaArtifacts::RuntimeMetadata::extensionClass?]
-      @eg_meta_by_field_name_by_concrete_type: egMetaByFieldByTypeHash
+      @fields_by_name_by_concrete_type: fieldsByNameByTypeHash
+
+      class FieldMetadata
+        attr_reader type: ::String
+        attr_reader name_in_index: ::String
+
+        def initialize: (
+          type: ::String,
+          name_in_index: ::String
+        ) -> void
+
+        def with: (
+          ?type: ::String,
+          ?name_in_index: ::String
+        ) -> FieldMetadata
+      end
 
       class TypeMetadata
         attr_reader name: ::String
-        attr_reader eg_meta_by_field_name: egMetaByFieldHash
+        attr_reader fields_by_name: fieldsByNameHash
 
         def initialize: (
           name: ::String,
-          eg_meta_by_field_name: egMetaByFieldHash
+          fields_by_name: fieldsByNameHash
         ) -> void
 
         def with: (
           ?name: ::String,
-          ?eg_meta_by_field_name: egMetaByFieldHash
+          ?fields_by_name: fieldsByNameHash
         ) -> TypeMetadata
       end
     end

elasticgraph-indexer/spec/unit/elastic_graph/indexer/record_preparer_spec.rb

@@ -451,6 +451,36 @@ class Indexer
         end
       end
 
+      context "when built directly from ingestion-format-neutral metadata" do
+        it "prepares records without any JSON schema, so other ingestion formats can supply their own field metadata" do
+          preparer = RecordPreparer.new({}, [
+            RecordPreparer::TypeMetadata.new(
+              name: "MyType",
+              fields_by_name: {
+                "id" => RecordPreparer::FieldMetadata.new(type: "ID!", name_in_index: "id"),
+                "name" => RecordPreparer::FieldMetadata.new(type: "String", name_in_index: "name2"),
+                "options" => RecordPreparer::FieldMetadata.new(type: "WidgetOptions", name_in_index: "options")
+              }
+            ),
+            RecordPreparer::TypeMetadata.new(
+              name: "WidgetOptions",
+              fields_by_name: {
+                "color" => RecordPreparer::FieldMetadata.new(type: "String", name_in_index: "clr")
+              }
+            )
+          ])
+
+          record = preparer.prepare_for_index("MyType", {
+            "id" => "1",
+            "name" => "Winston",
+            "options" => {"color" => "RED"},
+            "not_in_schema" => "should be dropped"
+          }, {})
+
+          expect(record).to eq({"id" => "1", "name2" => "Winston", "options" => {"clr" => "RED"}})
+        end
+      end
+
       def build_preparer(**config_overrides, &schema_definition)
         build_preparer_with_artifacts(**config_overrides, &schema_definition).first
       end

elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/record_preparer_factory.rb

@@ -57,14 +57,18 @@ def build_type_metas_from(json_schemas)
             {} # : ::Hash[::String, untyped]
           end # : ::Hash[::String, untyped]
 
-          eg_meta_by_field_name = properties.filter_map do |prop_name, prop|
-            eg_meta = prop["ElasticGraph"]
-            [prop_name, eg_meta] if eg_meta
+          fields_by_name = properties.filter_map do |prop_name, prop|
+            if (eg_meta = prop["ElasticGraph"])
+              [prop_name, Indexer::RecordPreparer::FieldMetadata.new(
+                type: eg_meta.fetch("type"),
+                name_in_index: eg_meta.fetch("nameInIndex")
+              )]
+            end
           end.to_h
 
           Indexer::RecordPreparer::TypeMetadata.new(
             name: type,
-            eg_meta_by_field_name: eg_meta_by_field_name
+            fields_by_name: fields_by_name
           )
         end
       end