Skip to content
10 changes: 9 additions & 1 deletion config/schema/artifacts/datastore_config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,12 @@ index_templates:
index_patterns:
- teams_rollover__*
template:
aliases: {}
aliases:
teams_all: {}
teams_nfl:
filter:
term:
league: NFL
mappings:
dynamic: strict
properties:
Expand Down Expand Up @@ -1268,6 +1273,9 @@ index_templates:
__typename:
type: constant_keyword
value: Team
formed:
type: alias
path: formed_on
_routing:
required: true
_size:
Expand Down
10 changes: 9 additions & 1 deletion config/schema/artifacts_with_apollo/datastore_config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,12 @@ index_templates:
index_patterns:
- teams_rollover__*
template:
aliases: {}
aliases:
teams_all: {}
teams_nfl:
filter:
term:
league: NFL
mappings:
dynamic: strict
properties:
Expand Down Expand Up @@ -1268,6 +1273,9 @@ index_templates:
__typename:
type: constant_keyword
value: Team
formed:
type: alias
path: formed_on
_routing:
required: true
_size:
Expand Down
9 changes: 9 additions & 0 deletions config/schema/teams.rb
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,15 @@
i.route_with "league"
i.rollover :yearly, "formed_on"
i.has_had_multiple_sources!
# Exercises `customize_config` with both index aliases (plain and filtered) and a field alias.
i.customize_config do |config|
config["aliases"] = {
"teams_all" => {},
"teams_nfl" => {"filter" => {"term" => {"league" => "NFL"}}}
}

config["mappings"]["properties"]["formed"] = {"type" => "alias", "path" => "formed_on"}
end
end
end

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,8 @@ def put_index_mapping(*) = nil

def put_index_settings(*) = nil

def update_index_aliases(*) = nil

# Document APIs
def_delegators :@wrapped_client, :get, :search, :msearch

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,12 +32,14 @@ def initialize(datastore_client, index, env_agnostic_index_config, output)
# exposed by the `IndexDefinition` object. Based on the configuration of the passed index
# and the state of the index in the datastore, does one of the following:
#
# - If the index did not already exist: creates the index with the desired mappings and settings.
# - If the index did not already exist: creates the index with the desired aliases, mappings and settings.
# - If the desired mapping has fewer fields than what is in the index: leaves the existing fields
# alone, because the datastore provides no way to remove fields from a mapping.
# - If the settings have desired changes: updates the settings, restoring any setting that
# no longer has a desired value to its default.
# - If the mapping has desired changes: updates the mappings.
# - If the aliases have desired changes: adds or updates the desired aliases, leaving aliases
# that were not declared (e.g. ones created outside of ElasticGraph) alone.
#
# Note that any of the writes to the index may fail. There are many things that cannot
# be changed on an existing index (such as static settings, field mapping types, etc). We do not attempt
Expand All @@ -57,6 +59,10 @@ def configure!
update_settings if settings_updates.any?

update_mapping if has_mapping_updates?

# Update aliases after mappings, since an alias with a `filter` can reference fields that are
# only available once the mapping updates have been applied.
Comment thread
myronmarston marked this conversation as resolved.
update_aliases if alias_updates.any?
end

def validate
Expand Down Expand Up @@ -86,6 +92,17 @@ def update_settings
report_action "Updated settings for index `#{@index.name}`:\n#{settings_diff}"
end

def update_aliases
# An `add` action is an upsert: when the named alias already exists on the index, the
# datastore replaces its definition with the one provided here (rather than merging).
actions = alias_updates.map do |name, definition|
{"add" => definition.merge({"index" => @index.name, "alias" => name})}
Comment thread
myronmarston marked this conversation as resolved.
end

@datastore_client.update_index_aliases(body: {"actions" => actions})
report_action "Updated aliases for index `#{@index.name}`:\n#{alias_diff}"
end

def cannot_modify_mapping_field_type_error
"The datastore does not support modifying the type of a field from an existing index definition. " \
"You are attempting to update type of fields (#{mapping_type_changes.inspect}) from the #{@index.name} index definition."
Expand Down Expand Up @@ -118,6 +135,22 @@ def settings_updates
end
end

# Note: aliases that exist on the index but are not desired are intentionally left alone rather
# than removed. Undeclared aliases may have been created outside of ElasticGraph (which does
# nothing with aliases itself), so their absence from our desired configuration just means
# ElasticGraph doesn't manage them.
def alias_updates
@alias_updates ||= desired_aliases.reject { |name, definition| current_aliases[name] == definition }
end

def desired_aliases
desired_config["aliases"] || {}
end

def current_aliases
current_config["aliases"] || {}
end

def desired_mapping_for_update
@desired_mapping_for_update ||= MappingUpdate.build_mapping_update(desired: desired_mapping, current: current_mapping)
end
Expand Down Expand Up @@ -169,6 +202,10 @@ def settings_diff
@settings_diff ||= Indexer::HashDiffer.diff(current_settings, desired_settings) || "(no diff)"
end

def alias_diff
@alias_diff ||= Indexer::HashDiffer.diff(current_aliases, current_aliases.merge(alias_updates)) || "(no diff)"
end

def report_action(message)
@reporter.report_action(message)
end
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,12 +36,14 @@ def initialize(datastore_client, index_template, env_agnostic_index_config_paren
# exposed by the `IndexDefinition` object. Based on the configuration of the passed index
# and the state of the index in the datastore, does one of the following:
#
# - If the index did not already exist: creates the index with the desired mappings and settings.
# - If the index template did not already exist: creates the index template with the desired
# aliases, mappings and settings.
# - If the desired mapping has fewer fields than what is in the index template: leaves the existing
# fields alone (see `put_index_template` for why).
# - If the settings have desired changes: updates the settings, restoring any setting that
# no longer has a desired value to its default.
# - If the mapping has desired changes: updates the mappings.
# - If the aliases have desired changes: adds or updates the desired aliases, leaving aliases
# that were not declared (e.g. ones created outside of ElasticGraph) alone.
# - If any other part of the template configuration (settings, mappings, etc.) has desired
# changes: updates the template.
#
# Note that any of the writes to the index may fail. There are many things that cannot
# be changed on an existing index (such as static settings, field mapping types, etc). We do not attempt
Expand All @@ -50,8 +52,10 @@ def initialize(datastore_client, index_template, env_agnostic_index_config_paren
def configure!
related_index_configurators.each(&:configure!)

# there is no partial update for index template config and the same API both creates and updates it
put_index_template if has_mapping_updates? || settings_updates.any?
# There is no partial update for index template config and the same API both creates and updates it.
# Note that we diff the full template config (rather than just mappings and settings) so that a
# change to any part of it--such as `customize_config` customizations--triggers an update.
put_index_template if has_config_updates?
end

def validate
Expand Down Expand Up @@ -104,35 +108,46 @@ def mapping_type_changes
end
end

def has_mapping_updates?
current_mapping != desired_mapping_for_update
end

def settings_updates
@settings_updates ||= begin
# Updating a setting to null will cause the datastore to restore the default value of the setting.
restore_to_defaults = (current_settings.keys - desired_settings.keys).to_h { |key| [key, nil] }
desired_settings.select { |key, value| current_settings[key] != value }.merge(restore_to_defaults)
end
def has_config_updates?
desired_config_parent_for_update.fetch("template") != (current_config_parent["template"] || {})
end

def desired_mapping_for_update
@desired_mapping_for_update ||= MappingUpdate.build_mapping_update(desired: desired_mapping, current: current_mapping)
end

def desired_config_parent_for_update
@desired_config_parent_for_update ||= Support::HashUtil.deep_merge(
desired_config_parent,
{"template" => {"mappings" => desired_mapping_for_update}}
)
@desired_config_parent_for_update ||= begin
template = DatastoreCore::IndexConfigNormalizer.normalize(
desired_config_parent.fetch("template").merge({
"mappings" => desired_mapping_for_update,
"aliases" => aliases_for_update
})
)

desired_config_parent.merge({"template" => template})
end
end

def desired_mapping
desired_config_parent.fetch("template").fetch("mappings")
# Aliases that exist on the current template but are not desired are preserved rather than removed
# (the same policy `MappingUpdate` applies to no-longer-desired mapping fields). Undeclared aliases
# may have been created outside of ElasticGraph (which does nothing with aliases itself), and since
# `put_index_template` replaces the entire template, omitting them here would silently drop them
# from all future rollover indices.
def aliases_for_update
current_aliases.merge(desired_aliases)
end

def desired_aliases
desired_config_parent.fetch("template")["aliases"] || {}
end

def desired_settings
@desired_settings ||= desired_config_parent.fetch("template").fetch("settings")
def current_aliases
current_config_parent.dig("template", "aliases") || {}
end

def desired_mapping
desired_config_parent.fetch("template").fetch("mappings")
end

def desired_config_parent
Expand All @@ -159,10 +174,6 @@ def current_mapping
current_config_parent.dig("template", "mappings") || {}
end

def current_settings
@current_settings ||= current_config_parent.dig("template", "settings")
end

def current_config_parent
@current_config_parent ||= begin
config = @datastore_client.get_index_template(@index_template.name)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ module ElasticGraph
def create_new_index: () -> void
def update_mapping: () -> void
def update_settings: () -> void
def update_aliases: () -> void
def cannot_modify_mapping_field_type_error: () -> ::String
def index_exists?: () -> bool

Expand All @@ -36,6 +37,13 @@ module ElasticGraph
@settings_updates: DatastoreCore::indexSettingsHash?
def settings_updates: () -> DatastoreCore::indexSettingsHash

@alias_updates: DatastoreCore::indexAliasesHash?
def alias_updates: () -> DatastoreCore::indexAliasesHash

def desired_aliases: () -> DatastoreCore::indexAliasesHash

def current_aliases: () -> DatastoreCore::indexAliasesHash

@desired_mapping_for_update: DatastoreCore::indexMappingHash?
def desired_mapping_for_update: () -> DatastoreCore::indexMappingHash

Expand All @@ -61,6 +69,9 @@ module ElasticGraph
@settings_diff: ::String?
def settings_diff: () -> ::String

@alias_diff: ::String?
def alias_diff: () -> ::String

def report_action: (::String) -> void
end
end
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,30 +32,27 @@ module ElasticGraph
@mapping_type_changes: ::Array[::String]?
def mapping_type_changes: () -> ::Array[::String]

def has_mapping_updates?: () -> bool

@settings_updates: DatastoreCore::indexSettingsHash?
def settings_updates: () -> DatastoreCore::indexSettingsHash
def has_config_updates?: () -> bool

@desired_mapping_for_update: DatastoreCore::indexMappingHash?
def desired_mapping_for_update: () -> DatastoreCore::indexMappingHash

@desired_config_parent_for_update: ::Hash[::String, untyped]?
def desired_config_parent_for_update: () -> ::Hash[::String, untyped]

def desired_mapping: () -> DatastoreCore::indexMappingHash
def aliases_for_update: () -> DatastoreCore::indexAliasesHash

def desired_aliases: () -> DatastoreCore::indexAliasesHash

@desired_settings: DatastoreCore::indexSettingsHash?
def desired_settings: () -> DatastoreCore::indexSettingsHash
def current_aliases: () -> DatastoreCore::indexAliasesHash

def desired_mapping: () -> DatastoreCore::indexMappingHash

@desired_config_parent: ::Hash[::String, untyped]
def desired_config_parent: () -> ::Hash[::String, untyped]

def current_mapping: () -> DatastoreCore::indexMappingHash

@current_settings: DatastoreCore::indexSettingsHash?
def current_settings: () -> DatastoreCore::indexSettingsHash

@current_config_parent: ::Hash[::String, untyped]
def current_config_parent: () -> ::Hash[::String, untyped]

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,31 @@ module IndexDefinitionConfigurator
.and log_warning(/Can't update non dynamic setting/)
end

it "applies mapping updates before alias updates so that an alias filter can reference a newly declared field" do
nested_alias = "#{unique_index_name}_nested"
# The datastore rejects a `nested` alias filter when the path is not yet in the index mapping.
nested_filter = {"nested" => {"path" => "nested_options", "query" => {"term" => {"nested_options.size" => "large"}}}}

configure_index_definition(schema_def)

expect {
configure_index_definition(schema_def(
configure_widget: ->(t) {
t.field "nested_options", "[WidgetOptions!]!" do |f|
f.mapping type: "nested"
end
},
configure_index: ->(index) {
index.customize_config do |config|
config["aliases"] = {nested_alias => {"filter" => nested_filter}}
end
}
))
}.to change { aliases_of(unique_index_name) }
.from({})
.to({nested_alias => {"filter" => nested_filter}})
end

it "handles empty indexed types" do
schema = schema_def(define_no_widget_fields: true)

Expand All @@ -39,6 +64,10 @@ def make_datastore_calls_to_configure_index_def(index_name, subresource = nil)
make_datastore_write_calls("main", "PUT #{put_index_definition_url(index_name, subresource)}")
end

def make_datastore_calls_to_update_aliases(_index_name)
make_datastore_write_calls("main", "POST /_aliases")
end

def fetch_artifact_configuration(schema_artifacts, index_def_name)
schema_artifacts.indices.fetch(index_def_name)
end
Expand Down
Loading