diff --git a/elasticgraph-indexer/lib/elastic_graph/indexer/record_preparer.rb b/elasticgraph-indexer/lib/elastic_graph/indexer/record_preparer.rb index f21b73e27..44c44b077 100644 --- a/elasticgraph-indexer/lib/elastic_graph/indexer/record_preparer.rb +++ b/elasticgraph-indexer/lib/elastic_graph/indexer/record_preparer.rb @@ -26,8 +26,8 @@ def self.prepare_for_index(type_name, value, mapping_properties) def initialize(indexing_preparer_by_scalar_type_name, type_metas) @indexing_preparer_by_scalar_type_name = indexing_preparer_by_scalar_type_name - @eg_meta_by_field_name_by_concrete_type = type_metas.to_h do |meta| - [meta.name, meta.eg_meta_by_field_name] + @fields_by_name_by_concrete_type = type_metas.to_h do |meta| + [meta.name, meta.fields_by_name] end end @@ -59,12 +59,12 @@ def prepare_for_index(type_name, value, mapping_properties) element_type_name = type_name.delete_prefix("[").delete_suffix("]") value.map { |v| prepare_for_index(element_type_name, v, mapping_properties) } when ::Hash - # `@eg_meta_by_field_name_by_concrete_type` does not have abstract types in it (e.g. type unions). + # `@fields_by_name_by_concrete_type` does not have abstract types in it (e.g. type unions). # Instead, it'll have each concrete subtype in it. # # If `type_name` is an abstract type, we need to look at the `__typename` field to see # what the concrete subtype is. `__typename` is required on abstract types and indicates that. - eg_meta_by_field_name = @eg_meta_by_field_name_by_concrete_type.fetch(value["__typename"] || type_name) + fields_by_name = @fields_by_name_by_concrete_type.fetch(value["__typename"] || type_name) # We only want to consider __typename if it's in the per-record mapping in order to determine # whether __typename is required on records. When it's a constant_keyword it exists at the index @@ -76,10 +76,10 @@ def prepare_for_index(type_name, value, mapping_properties) if field_name == "__typename" # Only include __typename if the index mapping has it at this position. [field_name, field_value] if typename_in_record_mapping - elsif (eg_meta = eg_meta_by_field_name[field_name]) - name_in_index = eg_meta.fetch("nameInIndex") + elsif (field = fields_by_name[field_name]) + name_in_index = field.name_in_index nested_mapping_properties = mapping_properties&.dig(name_in_index, "properties") - [name_in_index, prepare_for_index(eg_meta.fetch("type"), field_value, nested_mapping_properties)] + [name_in_index, prepare_for_index(field.type, field_value, nested_mapping_properties)] end end.to_h @@ -98,12 +98,24 @@ def prepare_for_index(type_name, value, mapping_properties) end end - TypeMetadata = ::Data.define( - # The name of the type this metadata object is for. - :name, - # The per-field ElasticGraph metadata, keyed by field name. - :eg_meta_by_field_name - ) + # Ingestion-format-neutral metadata about a single indexed field. Ingestion adapters build + # these from whatever source describes their format (e.g. `elasticgraph-json_ingestion` + # derives them from the versioned JSON schemas), so the indexer itself does not need to know + # how any particular ingestion format describes its fields. + # + # @!attribute [r] type + # @return [String] name of the ElasticGraph type of this field + # @!attribute [r] name_in_index + # @return [String] name of this field in the index + FieldMetadata = ::Data.define(:type, :name_in_index) + + # Ingestion-format-neutral metadata about a single indexed type. + # + # @!attribute [r] name + # @return [String] the name of the type this metadata object is for + # @!attribute [r] fields_by_name + # @return [Hash] metadata for each of the type's fields, keyed by field name + TypeMetadata = ::Data.define(:name, :fields_by_name) end end end diff --git a/elasticgraph-indexer/sig/elastic_graph/indexer/record_preparer.rbs b/elasticgraph-indexer/sig/elastic_graph/indexer/record_preparer.rbs index 694a5f307..be93067b9 100644 --- a/elasticgraph-indexer/sig/elastic_graph/indexer/record_preparer.rbs +++ b/elasticgraph-indexer/sig/elastic_graph/indexer/record_preparer.rbs @@ -5,8 +5,8 @@ module ElasticGraph end class RecordPreparer - type egMetaByFieldHash = ::Hash[::String, {"type" => ::String, "nameInIndex" => ::String}] - type egMetaByFieldByTypeHash = ::Hash[::String, egMetaByFieldHash] + type fieldsByNameHash = ::Hash[::String, FieldMetadata] + type fieldsByNameByTypeHash = ::Hash[::String, fieldsByNameHash] include _RecordPreparer @@ -20,20 +20,35 @@ module ElasticGraph ) -> void @indexing_preparer_by_scalar_type_name: ::Hash[::String, SchemaArtifacts::RuntimeMetadata::extensionClass?] - @eg_meta_by_field_name_by_concrete_type: egMetaByFieldByTypeHash + @fields_by_name_by_concrete_type: fieldsByNameByTypeHash + + class FieldMetadata + attr_reader type: ::String + attr_reader name_in_index: ::String + + def initialize: ( + type: ::String, + name_in_index: ::String + ) -> void + + def with: ( + ?type: ::String, + ?name_in_index: ::String + ) -> FieldMetadata + end class TypeMetadata attr_reader name: ::String - attr_reader eg_meta_by_field_name: egMetaByFieldHash + attr_reader fields_by_name: fieldsByNameHash def initialize: ( name: ::String, - eg_meta_by_field_name: egMetaByFieldHash + fields_by_name: fieldsByNameHash ) -> void def with: ( ?name: ::String, - ?eg_meta_by_field_name: egMetaByFieldHash + ?fields_by_name: fieldsByNameHash ) -> TypeMetadata end end diff --git a/elasticgraph-indexer/spec/unit/elastic_graph/indexer/record_preparer_spec.rb b/elasticgraph-indexer/spec/unit/elastic_graph/indexer/record_preparer_spec.rb index d2586c6a6..f6dba8646 100644 --- a/elasticgraph-indexer/spec/unit/elastic_graph/indexer/record_preparer_spec.rb +++ b/elasticgraph-indexer/spec/unit/elastic_graph/indexer/record_preparer_spec.rb @@ -451,6 +451,36 @@ class Indexer end end + context "when built directly from ingestion-format-neutral metadata" do + it "prepares records without any JSON schema, so other ingestion formats can supply their own field metadata" do + preparer = RecordPreparer.new({}, [ + RecordPreparer::TypeMetadata.new( + name: "MyType", + fields_by_name: { + "id" => RecordPreparer::FieldMetadata.new(type: "ID!", name_in_index: "id"), + "name" => RecordPreparer::FieldMetadata.new(type: "String", name_in_index: "name2"), + "options" => RecordPreparer::FieldMetadata.new(type: "WidgetOptions", name_in_index: "options") + } + ), + RecordPreparer::TypeMetadata.new( + name: "WidgetOptions", + fields_by_name: { + "color" => RecordPreparer::FieldMetadata.new(type: "String", name_in_index: "clr") + } + ) + ]) + + record = preparer.prepare_for_index("MyType", { + "id" => "1", + "name" => "Winston", + "options" => {"color" => "RED"}, + "not_in_schema" => "should be dropped" + }, {}) + + expect(record).to eq({"id" => "1", "name2" => "Winston", "options" => {"clr" => "RED"}}) + end + end + def build_preparer(**config_overrides, &schema_definition) build_preparer_with_artifacts(**config_overrides, &schema_definition).first end diff --git a/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/record_preparer_factory.rb b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/record_preparer_factory.rb index b1cfa2ab0..c7850a59b 100644 --- a/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/record_preparer_factory.rb +++ b/elasticgraph-json_ingestion/lib/elastic_graph/json_ingestion/record_preparer_factory.rb @@ -57,14 +57,18 @@ def build_type_metas_from(json_schemas) {} # : ::Hash[::String, untyped] end # : ::Hash[::String, untyped] - eg_meta_by_field_name = properties.filter_map do |prop_name, prop| - eg_meta = prop["ElasticGraph"] - [prop_name, eg_meta] if eg_meta + fields_by_name = properties.filter_map do |prop_name, prop| + if (eg_meta = prop["ElasticGraph"]) + [prop_name, Indexer::RecordPreparer::FieldMetadata.new( + type: eg_meta.fetch("type"), + name_in_index: eg_meta.fetch("nameInIndex") + )] + end end.to_h Indexer::RecordPreparer::TypeMetadata.new( name: type, - eg_meta_by_field_name: eg_meta_by_field_name + fields_by_name: fields_by_name ) end end