From 8058c404be8ce9a3aef52ca19cbdbe762d19df39 Mon Sep 17 00:00:00 2001 From: Mark Yang Date: Mon, 24 Aug 2026 12:00:36 +0100 Subject: [PATCH 1/7] Add customize_config API for arbitrary index config customization Implements the general-purpose customization hook proposed in https://github.com/block/elasticgraph/discussions/1364: t.index "campaigns" do |i| i.rollover :monthly, "createdAt" i.customize_config do |config| config["aliases"] = { "campaigns_read" => {}, "campaigns_active" => {"filter" => {"term" => {"status" => "ACTIVE"}}} } end end The block mutates a defensive deep copy of the index configuration (the template body for rollover index definitions, the index config otherwise), and the result is baked into `datastore_config.yaml`. Three admin-side changes make customizations fully respected: * Absent means "leave alone": on template updates, desired aliases are merged over the template's current ones instead of replacing them, so an alias created outside of ElasticGraph is never clobbered by a routine `clusters:configure` (fixing that regression independently of the new API). * `ForIndexTemplate#configure!` now diffs the full desired vs. current template config instead of just mappings and settings, so a customization-only change still reaches the datastore. * `ForIndex#configure!` reconciles aliases on existing concrete indices via the `_aliases` API (adding/updating desired aliases, never removing undeclared ones), so a newly declared alias lands on pre-existing rollover indices rather than only ones created later. `IndexConfigNormalizer` now treats an empty `aliases` hash the same as an absent one, since the datastore doesn't consistently echo the key back, and both datastore clients gained an `update_index_aliases` method. Generated with Claude Code --- .../datastore_client_dry_run_decorator.rb | 2 + .../for_index.rb | 37 ++++- .../for_index_template.rb | 67 ++++---- .../for_index.rbs | 11 ++ .../for_index_template.rbs | 17 +-- .../for_index_spec.rb | 4 + .../for_index_template_spec.rb | 38 +++++ .../shared_examples.rb | 72 +++++++++ .../datastore_core/index_config_normalizer.rb | 6 + .../elastic_graph/datastore_core/client.rbs | 2 + .../index_config_normalizer_spec.rb | 25 +++ .../lib/elastic_graph/elasticsearch/client.rb | 4 + .../sig/elasticsearch.rbs | 1 + .../elasticsearch/client_spec.rb | 2 + .../lib/elastic_graph/opensearch/client.rb | 4 + elasticgraph-opensearch/sig/opensearch.rbs | 1 + .../elastic_graph/opensearch/client_spec.rb | 2 + .../schema_definition/indexing/index.rb | 73 ++++++++- .../schema_definition/indexing/index.rbs | 3 + .../datastore_config/customize_config_spec.rb | 144 ++++++++++++++++++ .../datastore_client_shared_examples.rb | 6 + .../datastore_client_adapter.rb | 10 ++ 22 files changed, 486 insertions(+), 45 deletions(-) create mode 100644 elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/datastore_config/customize_config_spec.rb diff --git a/elasticgraph-admin/lib/elastic_graph/admin/datastore_client_dry_run_decorator.rb b/elasticgraph-admin/lib/elastic_graph/admin/datastore_client_dry_run_decorator.rb index f1b8b78e1..03742cdb9 100644 --- a/elasticgraph-admin/lib/elastic_graph/admin/datastore_client_dry_run_decorator.rb +++ b/elasticgraph-admin/lib/elastic_graph/admin/datastore_client_dry_run_decorator.rb @@ -65,6 +65,8 @@ def put_index_mapping(*) = nil def put_index_settings(*) = nil + def update_index_aliases(*) = nil + # Document APIs def_delegators :@wrapped_client, :get, :search, :msearch diff --git a/elasticgraph-admin/lib/elastic_graph/admin/index_definition_configurator/for_index.rb b/elasticgraph-admin/lib/elastic_graph/admin/index_definition_configurator/for_index.rb index 229a41f20..a61610c13 100644 --- a/elasticgraph-admin/lib/elastic_graph/admin/index_definition_configurator/for_index.rb +++ b/elasticgraph-admin/lib/elastic_graph/admin/index_definition_configurator/for_index.rb @@ -32,12 +32,14 @@ def initialize(datastore_client, index, env_agnostic_index_config, output) # exposed by the `IndexDefinition` object. Based on the configuration of the passed index # and the state of the index in the datastore, does one of the following: # - # - If the index did not already exist: creates the index with the desired mappings and settings. + # - If the index did not already exist: creates the index with the desired aliases, mappings and settings. # - If the desired mapping has fewer fields than what is in the index: leaves the existing fields # alone, because the datastore provides no way to remove fields from a mapping. # - If the settings have desired changes: updates the settings, restoring any setting that # no longer has a desired value to its default. # - If the mapping has desired changes: updates the mappings. + # - If the aliases have desired changes: adds or updates the desired aliases, leaving aliases + # that were not declared (e.g. ones created outside of ElasticGraph) alone. # # Note that any of the writes to the index may fail. There are many things that cannot # be changed on an existing index (such as static settings, field mapping types, etc). We do not attempt @@ -57,6 +59,10 @@ def configure! update_settings if settings_updates.any? update_mapping if has_mapping_updates? + + # Update aliases after mappings, since an alias with a `filter` can reference fields that are + # only available once the mapping updates have been applied. + update_aliases if alias_updates.any? end def validate @@ -86,6 +92,15 @@ def update_settings report_action "Updated settings for index `#{@index.name}`:\n#{settings_diff}" end + def update_aliases + actions = alias_updates.map do |name, definition| + {"add" => definition.merge({"index" => @index.name, "alias" => name})} + end + + @datastore_client.update_index_aliases(body: {"actions" => actions}) + report_action "Updated aliases for index `#{@index.name}`:\n#{alias_diff}" + end + def cannot_modify_mapping_field_type_error "The datastore does not support modifying the type of a field from an existing index definition. " \ "You are attempting to update type of fields (#{mapping_type_changes.inspect}) from the #{@index.name} index definition." @@ -118,6 +133,22 @@ def settings_updates end end + # Note: aliases that exist on the index but are not desired are intentionally left alone rather + # than removed. Undeclared aliases may have been created outside of ElasticGraph (which does + # nothing with aliases itself), so their absence from our desired configuration just means + # ElasticGraph doesn't manage them. + def alias_updates + @alias_updates ||= desired_aliases.reject { |name, definition| current_aliases[name] == definition } + end + + def desired_aliases + desired_config["aliases"] || {} + end + + def current_aliases + current_config["aliases"] || {} + end + def desired_mapping_for_update @desired_mapping_for_update ||= MappingUpdate.build_mapping_update(desired: desired_mapping, current: current_mapping) end @@ -169,6 +200,10 @@ def settings_diff @settings_diff ||= Indexer::HashDiffer.diff(current_settings, desired_settings) || "(no diff)" end + def alias_diff + @alias_diff ||= Indexer::HashDiffer.diff(current_aliases, current_aliases.merge(alias_updates)) || "(no diff)" + end + def report_action(message) @reporter.report_action(message) end diff --git a/elasticgraph-admin/lib/elastic_graph/admin/index_definition_configurator/for_index_template.rb b/elasticgraph-admin/lib/elastic_graph/admin/index_definition_configurator/for_index_template.rb index db108448a..bcf63755a 100644 --- a/elasticgraph-admin/lib/elastic_graph/admin/index_definition_configurator/for_index_template.rb +++ b/elasticgraph-admin/lib/elastic_graph/admin/index_definition_configurator/for_index_template.rb @@ -36,12 +36,14 @@ def initialize(datastore_client, index_template, env_agnostic_index_config_paren # exposed by the `IndexDefinition` object. Based on the configuration of the passed index # and the state of the index in the datastore, does one of the following: # - # - If the index did not already exist: creates the index with the desired mappings and settings. + # - If the index template did not already exist: creates the index template with the desired + # aliases, mappings and settings. # - If the desired mapping has fewer fields than what is in the index template: leaves the existing # fields alone (see `put_index_template` for why). - # - If the settings have desired changes: updates the settings, restoring any setting that - # no longer has a desired value to its default. - # - If the mapping has desired changes: updates the mappings. + # - If the aliases have desired changes: adds or updates the desired aliases, leaving aliases + # that were not declared (e.g. ones created outside of ElasticGraph) alone. + # - If any other part of the template configuration (settings, mappings, etc.) has desired + # changes: updates the template. # # Note that any of the writes to the index may fail. There are many things that cannot # be changed on an existing index (such as static settings, field mapping types, etc). We do not attempt @@ -50,8 +52,10 @@ def initialize(datastore_client, index_template, env_agnostic_index_config_paren def configure! related_index_configurators.each(&:configure!) - # there is no partial update for index template config and the same API both creates and updates it - put_index_template if has_mapping_updates? || settings_updates.any? + # There is no partial update for index template config and the same API both creates and updates it. + # Note that we diff the full template config (rather than just mappings and settings) so that a + # change to any part of it--such as `customize_config` customizations--triggers an update. + put_index_template if has_config_updates? end def validate @@ -104,16 +108,8 @@ def mapping_type_changes end end - def has_mapping_updates? - current_mapping != desired_mapping_for_update - end - - def settings_updates - @settings_updates ||= begin - # Updating a setting to null will cause the datastore to restore the default value of the setting. - restore_to_defaults = (current_settings.keys - desired_settings.keys).to_h { |key| [key, nil] } - desired_settings.select { |key, value| current_settings[key] != value }.merge(restore_to_defaults) - end + def has_config_updates? + desired_config_parent_for_update.fetch("template") != (current_config_parent["template"] || {}) end def desired_mapping_for_update @@ -121,18 +117,37 @@ def desired_mapping_for_update end def desired_config_parent_for_update - @desired_config_parent_for_update ||= Support::HashUtil.deep_merge( - desired_config_parent, - {"template" => {"mappings" => desired_mapping_for_update}} - ) + @desired_config_parent_for_update ||= begin + template = DatastoreCore::IndexConfigNormalizer.normalize( + desired_config_parent.fetch("template").merge({ + "mappings" => desired_mapping_for_update, + "aliases" => aliases_for_update + }) + ) + + desired_config_parent.merge({"template" => template}) + end end - def desired_mapping - desired_config_parent.fetch("template").fetch("mappings") + # Aliases that exist on the current template but are not desired are preserved rather than removed + # (the same policy `MappingUpdate` applies to no-longer-desired mapping fields). Undeclared aliases + # may have been created outside of ElasticGraph (which does nothing with aliases itself), and since + # `put_index_template` replaces the entire template, omitting them here would silently drop them + # from all future rollover indices. + def aliases_for_update + current_aliases.merge(desired_aliases) + end + + def desired_aliases + desired_config_parent.fetch("template")["aliases"] || {} end - def desired_settings - @desired_settings ||= desired_config_parent.fetch("template").fetch("settings") + def current_aliases + current_config_parent.dig("template", "aliases") || {} + end + + def desired_mapping + desired_config_parent.fetch("template").fetch("mappings") end def desired_config_parent @@ -159,10 +174,6 @@ def current_mapping current_config_parent.dig("template", "mappings") || {} end - def current_settings - @current_settings ||= current_config_parent.dig("template", "settings") - end - def current_config_parent @current_config_parent ||= begin config = @datastore_client.get_index_template(@index_template.name) diff --git a/elasticgraph-admin/sig/elastic_graph/admin/index_definition_configurator/for_index.rbs b/elasticgraph-admin/sig/elastic_graph/admin/index_definition_configurator/for_index.rbs index a2a49fa72..f9e111c1c 100644 --- a/elasticgraph-admin/sig/elastic_graph/admin/index_definition_configurator/for_index.rbs +++ b/elasticgraph-admin/sig/elastic_graph/admin/index_definition_configurator/for_index.rbs @@ -25,6 +25,7 @@ module ElasticGraph def create_new_index: () -> void def update_mapping: () -> void def update_settings: () -> void + def update_aliases: () -> void def cannot_modify_mapping_field_type_error: () -> ::String def index_exists?: () -> bool @@ -36,6 +37,13 @@ module ElasticGraph @settings_updates: DatastoreCore::indexSettingsHash? def settings_updates: () -> DatastoreCore::indexSettingsHash + @alias_updates: DatastoreCore::indexAliasesHash? + def alias_updates: () -> DatastoreCore::indexAliasesHash + + def desired_aliases: () -> DatastoreCore::indexAliasesHash + + def current_aliases: () -> DatastoreCore::indexAliasesHash + @desired_mapping_for_update: DatastoreCore::indexMappingHash? def desired_mapping_for_update: () -> DatastoreCore::indexMappingHash @@ -61,6 +69,9 @@ module ElasticGraph @settings_diff: ::String? def settings_diff: () -> ::String + @alias_diff: ::String? + def alias_diff: () -> ::String + def report_action: (::String) -> void end end diff --git a/elasticgraph-admin/sig/elastic_graph/admin/index_definition_configurator/for_index_template.rbs b/elasticgraph-admin/sig/elastic_graph/admin/index_definition_configurator/for_index_template.rbs index 788abf967..c1d037b57 100644 --- a/elasticgraph-admin/sig/elastic_graph/admin/index_definition_configurator/for_index_template.rbs +++ b/elasticgraph-admin/sig/elastic_graph/admin/index_definition_configurator/for_index_template.rbs @@ -32,10 +32,7 @@ module ElasticGraph @mapping_type_changes: ::Array[::String]? def mapping_type_changes: () -> ::Array[::String] - def has_mapping_updates?: () -> bool - - @settings_updates: DatastoreCore::indexSettingsHash? - def settings_updates: () -> DatastoreCore::indexSettingsHash + def has_config_updates?: () -> bool @desired_mapping_for_update: DatastoreCore::indexMappingHash? def desired_mapping_for_update: () -> DatastoreCore::indexMappingHash @@ -43,19 +40,19 @@ module ElasticGraph @desired_config_parent_for_update: ::Hash[::String, untyped]? def desired_config_parent_for_update: () -> ::Hash[::String, untyped] - def desired_mapping: () -> DatastoreCore::indexMappingHash + def aliases_for_update: () -> DatastoreCore::indexAliasesHash + + def desired_aliases: () -> DatastoreCore::indexAliasesHash - @desired_settings: DatastoreCore::indexSettingsHash? - def desired_settings: () -> DatastoreCore::indexSettingsHash + def current_aliases: () -> DatastoreCore::indexAliasesHash + + def desired_mapping: () -> DatastoreCore::indexMappingHash @desired_config_parent: ::Hash[::String, untyped] def desired_config_parent: () -> ::Hash[::String, untyped] def current_mapping: () -> DatastoreCore::indexMappingHash - @current_settings: DatastoreCore::indexSettingsHash? - def current_settings: () -> DatastoreCore::indexSettingsHash - @current_config_parent: ::Hash[::String, untyped] def current_config_parent: () -> ::Hash[::String, untyped] diff --git a/elasticgraph-admin/spec/integration/elastic_graph/admin/index_definition_configurator/for_index_spec.rb b/elasticgraph-admin/spec/integration/elastic_graph/admin/index_definition_configurator/for_index_spec.rb index 113f36276..cc3db4277 100644 --- a/elasticgraph-admin/spec/integration/elastic_graph/admin/index_definition_configurator/for_index_spec.rb +++ b/elasticgraph-admin/spec/integration/elastic_graph/admin/index_definition_configurator/for_index_spec.rb @@ -39,6 +39,10 @@ def make_datastore_calls_to_configure_index_def(index_name, subresource = nil) make_datastore_write_calls("main", "PUT #{put_index_definition_url(index_name, subresource)}") end + def make_datastore_calls_to_update_aliases(_index_name) + make_datastore_write_calls("main", "POST /_aliases") + end + def fetch_artifact_configuration(schema_artifacts, index_def_name) schema_artifacts.indices.fetch(index_def_name) end diff --git a/elasticgraph-admin/spec/integration/elastic_graph/admin/index_definition_configurator/for_index_template_spec.rb b/elasticgraph-admin/spec/integration/elastic_graph/admin/index_definition_configurator/for_index_template_spec.rb index 056b30d81..0a9974a6c 100644 --- a/elasticgraph-admin/spec/integration/elastic_graph/admin/index_definition_configurator/for_index_template_spec.rb +++ b/elasticgraph-admin/spec/integration/elastic_graph/admin/index_definition_configurator/for_index_template_spec.rb @@ -68,6 +68,30 @@ def simulate_presence_of_extra_setting(admin, index_definition_name, name, value end end + def simulate_presence_of_external_alias(alias_name) + template = main_datastore_client.get_index_template(unique_index_name) + template_body = template.fetch("template") + + # Note: we only include the writable template properties here, since the datastore rejects a + # `put_index_template` payload containing system-managed properties (such as `created_date`). + main_datastore_client.put_index_template(name: unique_index_name, body: { + "index_patterns" => template.fetch("index_patterns"), + "template" => template_body.merge( + "aliases" => (template_body["aliases"] || {}).merge(alias_name => {}) + ) + }) + end + + def make_datastore_calls_to_update_aliases(index_name) + # Besides updating the template itself, the newly declared alias gets added (via the `_aliases` + # API) to the concrete rollover index that already exists from the prior configuration run. + make_datastore_write_calls( + "main", + "PUT #{put_index_definition_url(index_name)}", + "POST /_aliases" + ) + end + def fetch_artifact_configuration(schema_artifacts, index_def_name) schema_artifacts.index_templates.fetch(index_def_name) end @@ -125,6 +149,20 @@ def fetch_artifact_configuration(schema_artifacts, index_def_name) expect(index_def_creation_order).to eq([jan_2020_index_name, unique_index_name]) end + it "manages declared aliases on the concrete rollover indices created from the template" do + read_alias = "#{unique_index_name}_read" + now_index = concrete_index_name_for_now(unique_index_name) + + # A concrete index created from the template config gets the declared aliases stamped at creation. + configure_index_definition(schema_def_with_aliases({read_alias => {}})) + expect(main_datastore_client.get_index(now_index).fetch("aliases")).to eq({read_alias => {}}) + + # A newly declared alias is reconciled onto concrete indices that already exist. + active_alias = "#{unique_index_name}_active" + configure_index_definition(schema_def_with_aliases({read_alias => {}, active_alias => {}})) + expect(main_datastore_client.get_index(now_index).fetch("aliases")).to eq({read_alias => {}, active_alias => {}}) + end + context "when the settings do not force the creation of any concrete indices" do it "creates an index using the current time so that our search queries always have an index to hit" do expect { diff --git a/elasticgraph-admin/spec/integration/elastic_graph/admin/index_definition_configurator/shared_examples.rb b/elasticgraph-admin/spec/integration/elastic_graph/admin/index_definition_configurator/shared_examples.rb index d9a8f1f6a..62d3ab21a 100644 --- a/elasticgraph-admin/spec/integration/elastic_graph/admin/index_definition_configurator/shared_examples.rb +++ b/elasticgraph-admin/spec/integration/elastic_graph/admin/index_definition_configurator/shared_examples.rb @@ -36,6 +36,12 @@ def simulate_presence_of_extra_setting(admin, index_definition_name, name, value end end end + + def simulate_presence_of_external_alias(alias_name) + main_datastore_client.update_index_aliases(body: {"actions" => [ + {"add" => {"index" => unique_index_name, "alias" => alias_name}} + ]}) + end end RSpec.shared_examples_for IndexDefinitionConfigurator, :uses_datastore do @@ -271,6 +277,72 @@ def simulate_presence_of_extra_setting(admin, index_definition_name, name, value }.from([nil, nil]).to([["created_at"], ["asc"]]) end + it "applies `customize_config` customizations when creating an index or index template, and reconciles them idempotently" do + read_alias = "#{unique_index_name}_read" + active_alias = "#{unique_index_name}_active" + schema = schema_def_with_aliases({ + read_alias => {}, + active_alias => {"filter" => {"term" => {"name" => "active"}}} + }) + + expect { + configure_index_definition(schema) + }.to change { aliases_of(unique_index_name) } + .from({}) + .to({read_alias => {}, active_alias => {"filter" => {"term" => {"name" => "active"}}}}) + + expect { + configure_index_definition(schema) + }.to make_no_datastore_write_calls("main") + end + + it "adds newly declared aliases to an existing index or index template, and updates aliases whose desired definition has changed" do + read_alias = "#{unique_index_name}_read" + + configure_index_definition(schema_def) + output_io.string = +"" # use `+` so it is not a frozen string literal. + + expect { + configure_index_definition(schema_def_with_aliases({read_alias => {}})) + }.to change { aliases_of(unique_index_name) } + .from({}) + .to({read_alias => {}}) + .and make_datastore_calls_to_update_aliases(unique_index_name) + + expect(output_io.string).to include(read_alias) + + expect { + configure_index_definition(schema_def_with_aliases({read_alias => {"filter" => {"term" => {"name" => "active"}}}})) + }.to change { aliases_of(unique_index_name) } + .from({read_alias => {}}) + .to({read_alias => {"filter" => {"term" => {"name" => "active"}}}}) + end + + it "leaves aliases it did not declare alone rather than removing them" do + external_alias = "#{unique_index_name}_external" + + configure_index_definition(schema_def) + simulate_presence_of_external_alias(external_alias) + + expect { + configure_index_definition(schema_def) + }.to maintain { aliases_of(unique_index_name) } + .from(a_hash_including(external_alias)) + .and make_no_datastore_write_calls("main") + end + + def schema_def_with_aliases(aliases) + schema_def(configure_index: ->(index) { + index.customize_config do |config| + config["aliases"] = aliases + end + }) + end + + def aliases_of(index_definition_name) + get_index_definition_configuration(index_definition_name)["aliases"] || {} + end + def schema_def( configure_index: nil, configure_widget: nil, diff --git a/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/index_config_normalizer.rb b/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/index_config_normalizer.rb index 7c5d216b2..8bacfad99 100644 --- a/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/index_config_normalizer.rb +++ b/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/index_config_normalizer.rb @@ -48,7 +48,13 @@ module IndexConfigNormalizer # (e.g. `"false"` or `"7"` instead of `false` or `7`), so this matches that behavior. # - Drops `type: object` from a mapping when there are `properties` because the datastore omits it in that # situation, treating it as the default type. + # - Drops an empty `aliases` hash. Empty and absent `aliases` both mean "no declared aliases", but the datastore + # doesn't consistently echo the key back, so we normalize to the absent form for stable comparisons. def self.normalize(index_config) + if (aliases = index_config["aliases"]) && aliases.empty? + index_config = index_config.except("aliases") + end + if (settings = index_config["settings"]) index_config = index_config.merge("settings" => normalize_settings(settings)) end diff --git a/elasticgraph-datastore_core/sig/elastic_graph/datastore_core/client.rbs b/elasticgraph-datastore_core/sig/elastic_graph/datastore_core/client.rbs index e5b94e3e3..7c3776c47 100644 --- a/elasticgraph-datastore_core/sig/elastic_graph/datastore_core/client.rbs +++ b/elasticgraph-datastore_core/sig/elastic_graph/datastore_core/client.rbs @@ -2,6 +2,7 @@ module ElasticGraph class DatastoreCore type indexMappingHash = ::Hash[::String, untyped] type indexSettingsHash = ::Hash[::String, untyped] + type indexAliasesHash = ::Hash[::String, untyped] type indexConfigHash = ::Hash[::String, untyped] interface _Client @@ -25,6 +26,7 @@ module ElasticGraph def create_index: (index: ::String, body: ::Hash[::String, untyped]) -> void def put_index_mapping: (index: ::String, body: ::Hash[::String, untyped]) -> void def put_index_settings: (index: ::String, body: ::Hash[::String, untyped]) -> void + def update_index_aliases: (body: ::Hash[::String, untyped]) -> void def delete_indices: (*::String) -> void def msearch: (body: ::Array[::Hash[::String | ::Symbol, untyped]], ?headers: ::Hash[::String, untyped]?) -> ::Hash[::String, untyped] diff --git a/elasticgraph-datastore_core/spec/unit/elastic_graph/datastore_core/index_config_normalizer_spec.rb b/elasticgraph-datastore_core/spec/unit/elastic_graph/datastore_core/index_config_normalizer_spec.rb index bf8c87e5a..65bdb4709 100644 --- a/elasticgraph-datastore_core/spec/unit/elastic_graph/datastore_core/index_config_normalizer_spec.rb +++ b/elasticgraph-datastore_core/spec/unit/elastic_graph/datastore_core/index_config_normalizer_spec.rb @@ -92,6 +92,31 @@ class DatastoreCore end end + it "drops an empty `aliases` hash since it means the same thing as an absent one and the datastore doesn't consistently echo the key back" do + index_config = { + "aliases" => {}, + "settings" => {} + } + + normalized = IndexConfigNormalizer.normalize(index_config) + + expect(normalized).to eq({ + "settings" => {} + }) + end + + it "leaves a non-empty `aliases` hash unchanged" do + index_config = { + "aliases" => {"my_alias" => {}} + } + + normalized = IndexConfigNormalizer.normalize(index_config) + + expect(normalized).to eq({ + "aliases" => {"my_alias" => {}} + }) + end + it "drops `type: object` when it is alongside `properties` since the datastore treats that as the default type when `properties` are used and omits it" do index_config = { "mappings" => { diff --git a/elasticgraph-elasticsearch/lib/elastic_graph/elasticsearch/client.rb b/elasticgraph-elasticsearch/lib/elastic_graph/elasticsearch/client.rb index cfb7c87a3..455728e78 100644 --- a/elasticgraph-elasticsearch/lib/elastic_graph/elasticsearch/client.rb +++ b/elasticgraph-elasticsearch/lib/elastic_graph/elasticsearch/client.rb @@ -158,6 +158,10 @@ def put_index_settings(index:, body:) transform_errors { |c| c.indices.put_settings(index: index, body: body).body } end + def update_index_aliases(body:) + transform_errors { |c| c.indices.update_aliases(body: body).body } + end + def delete_indices(*index_names) # `allow_no_indices: true` is needed when we attempt to delete a non-existing index to avoid errors. For rollover indices, # when we delete the actual indices, we will always perform a wildcard deletion, and `allow_no_indices: true` is needed. diff --git a/elasticgraph-elasticsearch/sig/elasticsearch.rbs b/elasticgraph-elasticsearch/sig/elasticsearch.rbs index 5e64a0946..bb3f3ec06 100644 --- a/elasticgraph-elasticsearch/sig/elasticsearch.rbs +++ b/elasticgraph-elasticsearch/sig/elasticsearch.rbs @@ -25,6 +25,7 @@ module Elasticsearch def create: (index: ::String, body: stringOrSymbolHash) -> Response def put_mapping: (index: ::String, body: stringOrSymbolHash) -> Response def put_settings: (index: ::String, body: stringOrSymbolHash) -> Response + def update_aliases: (body: stringOrSymbolHash) -> Response def delete: (index: ::Array[::String], ?ignore_unavailable: bool, ?allow_no_indices: bool) -> Response end end diff --git a/elasticgraph-elasticsearch/spec/unit/elastic_graph/elasticsearch/client_spec.rb b/elasticgraph-elasticsearch/spec/unit/elastic_graph/elasticsearch/client_spec.rb index 4ad645eba..dfaad7962 100644 --- a/elasticgraph-elasticsearch/spec/unit/elastic_graph/elasticsearch/client_spec.rb +++ b/elasticgraph-elasticsearch/spec/unit/elastic_graph/elasticsearch/client_spec.rb @@ -56,6 +56,8 @@ def define_stubs(stub, requested_stubs) stub.put("/my_index/_mapping") { |env| response_for(body, env) } in :put_index_settings_my_index stub.put("/my_index/_settings") { |env| response_for(body, env) } + in :update_index_aliases + stub.post("/_aliases") { |env| response_for(body, env) } in :delete_indices_ind1_ind2 stub.delete("/ind1,ind2?allow_no_indices=true&ignore_unavailable=true") { |env| response_for(body, env) } diff --git a/elasticgraph-opensearch/lib/elastic_graph/opensearch/client.rb b/elasticgraph-opensearch/lib/elastic_graph/opensearch/client.rb index dbcfc9391..19ff2e74c 100644 --- a/elasticgraph-opensearch/lib/elastic_graph/opensearch/client.rb +++ b/elasticgraph-opensearch/lib/elastic_graph/opensearch/client.rb @@ -175,6 +175,10 @@ def put_index_settings(index:, body:) transform_errors { |c| c.indices.put_settings(index: index, body: body) } end + def update_index_aliases(body:) + transform_errors { |c| c.indices.update_aliases(body: body) } + end + def delete_indices(*index_names) # `allow_no_indices: true` is needed when we attempt to delete a non-existing index to avoid errors. For rollover indices, # when we delete the actual indices, we will always perform a wildcard deletion, and `allow_no_indices: true` is needed. diff --git a/elasticgraph-opensearch/sig/opensearch.rbs b/elasticgraph-opensearch/sig/opensearch.rbs index 4ec9baa96..3aec5b268 100644 --- a/elasticgraph-opensearch/sig/opensearch.rbs +++ b/elasticgraph-opensearch/sig/opensearch.rbs @@ -25,6 +25,7 @@ module OpenSearch def create: (index: ::String, body: stringOrSymbolHash) -> void def put_mapping: (index: ::String, body: stringOrSymbolHash) -> void def put_settings: (index: ::String, body: stringOrSymbolHash) -> void + def update_aliases: (body: stringOrSymbolHash) -> void def delete: (index: ::Array[::String], ?ignore_unavailable: bool, ?allow_no_indices: bool) -> void end end diff --git a/elasticgraph-opensearch/spec/unit/elastic_graph/opensearch/client_spec.rb b/elasticgraph-opensearch/spec/unit/elastic_graph/opensearch/client_spec.rb index da5178f5d..b030bbb9a 100644 --- a/elasticgraph-opensearch/spec/unit/elastic_graph/opensearch/client_spec.rb +++ b/elasticgraph-opensearch/spec/unit/elastic_graph/opensearch/client_spec.rb @@ -79,6 +79,8 @@ def define_stubs(stub, requested_stubs) stub.put("/my_index/_mappings") { |env| response_for(body, env) } in :put_index_settings_my_index stub.put("/my_index/_settings") { |env| response_for(body, env) } + in :update_index_aliases + stub.post("/_aliases") { |env| response_for(body, env) } in :delete_indices_ind1_ind2 stub.delete("/ind1,ind2?allow_no_indices=true&ignore_unavailable=true") { |env| response_for(body, env) } diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/index.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/index.rb index 460d46789..0c99d96ed 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/index.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/index.rb @@ -42,6 +42,8 @@ module Indexing # @return [Hash>] # map from a qualified (leaf) relationship to the path segments the painless script uses to navigate from this # root index's documents down to the nested elements that receive `sourced_from` data + # @!attribute [r] config_customizations + # @return [Array] blocks registered via {#customize_config}, applied to the datastore configuration of this index class Index < Struct.new( :name, :default_sort_pairs, @@ -51,7 +53,8 @@ class Index < Struct.new( :routing_field_path, :rollover_config, :has_had_multiple_sources_flag, - :sourced_from_nested_paths_by_qualified_relationship + :sourced_from_nested_paths_by_qualified_relationship, + :config_customizations ) include Mixins::HasReadableToSAndInspect.new { |i| i.name } @@ -69,7 +72,7 @@ def initialize(name, settings, schema_def_state, indexed_type) settings = DEFAULT_SETTINGS.merge(Support::HashUtil.flatten_and_stringify_keys(settings, prefix: "index")) - super(name, [], settings, schema_def_state, indexed_type, nil, nil, false, {}) + super(name, [], settings, schema_def_state, indexed_type, nil, nil, false, {}, []) schema_def_state.after_user_definition_complete do # `id` is the field Elasticsearch/OpenSearch use for routing by default: @@ -233,6 +236,54 @@ def has_had_multiple_sources! self.has_had_multiple_sources_flag = true end + # Customizes the datastore configuration of this index (or of the index template, when {#rollover} is used) in ways + # ElasticGraph doesn't natively model. The customization block is yielded the index configuration--a hash containing + # `aliases`, `mappings`, and `settings`--and is expected to mutate it (the return value is ignored). The customized + # configuration is included in the `datastore_config.yaml` schema artifact, and `elasticgraph-admin` applies it to + # the datastore just like the rest of the index configuration. + # + # When {#rollover} is used, the customization applies to the index template body, so every concrete index created + # from the template (including rollover indices the datastore auto-creates at indexing time) gets the customized + # configuration. + # + # @note ElasticGraph does nothing with the customized configuration besides passing it through to the datastore, + # and makes no claims about the safety or correctness of any customization. Use with care! + # + # @yield [Hash] the datastore configuration of the index, to be mutated by the block + # @return [void] + # + # @example Define index aliases and a field alias on a `campaigns` index + # ElasticGraph.define_schema do |schema| + # schema.object_type "Campaign" do |t| + # t.field "id", "ID!" + # t.field "status", "String" + # t.field "createdAt", "DateTime" + # + # t.index "campaigns" do |i| + # i.rollover :monthly, "createdAt" + # + # i.customize_config do |config| + # # Index aliases, so clients querying the datastore directly can use a stable name + # # (`campaigns_read` fans out across all the rollover indices). + # config["aliases"] = { + # "campaigns_read" => {}, + # "campaigns_active" => {"filter" => {"term" => {"status" => "ACTIVE"}}} + # } + # + # # A field alias, so direct datastore queries can reference `created` as an alias of `createdAt`. + # config["mappings"]["properties"]["created"] = {"type" => "alias", "path" => "createdAt"} + # end + # end + # end + # end + def customize_config(&customization_block) + if customization_block.nil? + raise Errors::SchemaError, "`customize_config` was called on the `#{name}` index without a block, but a block is required." + end + + config_customizations << customization_block + end + # @see #route_with # @return [Boolean] whether or not this index uses custom shard routing def uses_custom_routing? @@ -241,22 +292,22 @@ def uses_custom_routing? # @return [Hash] datastore configuration for this index for when it does not use rollover def to_index_config - { + customized_config({ "aliases" => {}, "mappings" => mappings, "settings" => settings - }.compact + }.compact) end # @return [Hash] datastore configuration for the index template that will be defined if rollover is used def to_index_template_config { "index_patterns" => ["#{name}#{ROLLOVER_INDEX_INFIX_MARKER}*"], - "template" => { + "template" => customized_config({ "aliases" => {}, "mappings" => mappings, "settings" => settings - } + }) } end @@ -296,6 +347,16 @@ def register_sourced_from_nested_paths(qualified_relationship, nested_paths) "index.max_result_window" => 10000 } + def customized_config(config) + return config if config_customizations.empty? + + # Yield a defensive deep copy so that mutations made by customization blocks can't corrupt + # ElasticGraph's internal data structures (parts of `mappings` are shared across indices). + config = ::Marshal.load(::Marshal.dump(config)) + config_customizations.each { |customization| customization.call(config) } + config + end + def mappings field_mappings = indexed_type .to_indexing_field_type diff --git a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/index.rbs b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/index.rbs index 430f4fbb6..b7ad65e9a 100644 --- a/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/index.rbs +++ b/elasticgraph-schema_definition/sig/elastic_graph/schema_definition/indexing/index.rbs @@ -10,9 +10,11 @@ module ElasticGraph attr_reader indexed_type: indexableType attr_reader sourced_from_nested_paths_by_qualified_relationship: ::Hash[::String, ::Array[SchemaArtifacts::RuntimeMetadata::sourcedFromNestedPathSegment]] attr_reader schema_def_state: State + attr_reader config_customizations: ::Array[^(::Hash[::String, untyped]) -> void] def rollover: (::Symbol, ::String) -> void def route_with: (::String) -> void + def customize_config: () { (::Hash[::String, untyped]) -> void } -> void def uses_custom_routing?: () -> bool def to_index_config: () -> ::Hash[::String, untyped] def to_index_template_config: () -> ::Hash[::String, untyped] @@ -21,6 +23,7 @@ module ElasticGraph private + def customized_config: (::Hash[::String, untyped]) -> ::Hash[::String, untyped] def public_field_path: (::String, explanation: ::String) -> SchemaElements::FieldPath def date_and_datetime_types: () -> ::Array[::String] end diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/datastore_config/customize_config_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/datastore_config/customize_config_spec.rb new file mode 100644 index 000000000..f99733f52 --- /dev/null +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/datastore_config/customize_config_spec.rb @@ -0,0 +1,144 @@ +# Copyright 2024 - 2026 Block, Inc. +# +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +# +# frozen_string_literal: true + +require_relative "index_definition_spec_support" + +module ElasticGraph + module SchemaDefinition + RSpec.describe "Datastore config -- customize_config" do + include_context "IndexDefinitionSpecSupport" + + it "applies customizations to the config of an index" do + campaigns = index_configs_for "campaigns" do |s| + s.object_type "Campaign" do |t| + t.field "id", "ID!" + t.field "status", "String" + + t.index "campaigns" do |i| + i.customize_config do |config| + config["aliases"] = { + "campaigns_read" => {}, + "campaigns_active" => {"filter" => {"term" => {"status" => "ACTIVE"}}} + } + end + end + end + end.first + + expect(campaigns).to match( + "aliases" => { + "campaigns_read" => {}, + "campaigns_active" => {"filter" => {"term" => {"status" => "ACTIVE"}}} + }, + "mappings" => an_instance_of(::Hash), + "settings" => an_instance_of(::Hash) + ) + end + + it "applies customizations to the template body of a rollover index" do + campaigns = index_template_configs_for "campaigns" do |s| + s.object_type "Campaign" do |t| + t.field "id", "ID!" + t.field "created_at", "DateTime" + + t.index "campaigns" do |i| + i.rollover :monthly, "created_at" + + i.customize_config do |config| + config["aliases"] = {"campaigns_read" => {}} + end + end + end + end.first + + expect(campaigns).to match( + "index_patterns" => ["campaigns_rollover__*"], + "template" => { + "aliases" => {"campaigns_read" => {}}, + "mappings" => an_instance_of(::Hash), + "settings" => an_instance_of(::Hash) + } + ) + end + + it "applies multiple customization blocks in the order they were registered, ignoring their return values" do + campaigns = index_configs_for "campaigns" do |s| + s.object_type "Campaign" do |t| + t.field "id", "ID!" + + t.index "campaigns" do |i| + i.customize_config do |config| + config["aliases"] = {"campaigns_alias1" => {}} + :ignored_return_value + end + + i.customize_config do |config| + config["aliases"] = config["aliases"].merge("campaigns_alias2" => {}) + end + end + end + end.first + + expect(campaigns.fetch("aliases")).to eq({"campaigns_alias1" => {}, "campaigns_alias2" => {}}) + end + + it "supports nested customizations such as field aliases" do + campaigns = index_configs_for "campaigns" do |s| + s.object_type "Campaign" do |t| + t.field "id", "ID!" + t.field "created_at", "DateTime" + + t.index "campaigns" do |i| + i.customize_config do |config| + config["mappings"]["properties"]["created"] = {"type" => "alias", "path" => "created_at"} + end + end + end + end.first + + expect(campaigns.dig("mappings", "properties", "created")).to eq({"type" => "alias", "path" => "created_at"}) + end + + it "yields a defensive copy so that customization mutations cannot corrupt the config of other indices" do + campaigns, promotions = index_configs_for "campaigns", "promotions" do |s| + s.object_type "Campaign" do |t| + t.field "id", "ID!" + + t.index "campaigns" do |i| + i.customize_config do |config| + config["mappings"]["properties"]["id"]["type"] = "text" + end + end + end + + s.object_type "Promotion" do |t| + t.field "id", "ID!" + t.index "promotions" + end + end + + expect(campaigns.dig("mappings", "properties", "id", "type")).to eq("text") + expect(promotions.dig("mappings", "properties", "id", "type")).to eq("keyword") + end + + it "raises a clear error when `customize_config` is called without a block" do + expect { + index_configs_for "campaigns" do |s| + s.object_type "Campaign" do |t| + t.field "id", "ID!" + + t.index "campaigns" do |i| + i.customize_config + end + end + end + }.to raise_error(Errors::SchemaError, a_string_including("customize_config", "campaigns", "block")) + end + end + end +end diff --git a/spec_support/lib/elastic_graph/spec_support/datastore_client_shared_examples.rb b/spec_support/lib/elastic_graph/spec_support/datastore_client_shared_examples.rb index 027338340..5673ec5e1 100644 --- a/spec_support/lib/elastic_graph/spec_support/datastore_client_shared_examples.rb +++ b/spec_support/lib/elastic_graph/spec_support/datastore_client_shared_examples.rb @@ -164,6 +164,12 @@ module ElasticGraph expect(client.put_index_settings(index: "my_index", body: {"settings" => "config"})).to eq("ok") end + it "supports `update_index_aliases`" do + client = build_client({update_index_aliases: "ok"}) + + expect(client.update_index_aliases(body: {"actions" => [{"add" => {"index" => "my_index", "alias" => "my_alias"}}]})).to eq("ok") + end + it "supports `delete_indices`" do client = build_client({delete_indices_ind1_ind2: "ok"}) diff --git a/spec_support/lib/elastic_graph/spec_support/parallel_spec_runner/datastore_client_adapter.rb b/spec_support/lib/elastic_graph/spec_support/parallel_spec_runner/datastore_client_adapter.rb index bf7b5a34f..651d7cbf3 100644 --- a/spec_support/lib/elastic_graph/spec_support/parallel_spec_runner/datastore_client_adapter.rb +++ b/spec_support/lib/elastic_graph/spec_support/parallel_spec_runner/datastore_client_adapter.rb @@ -83,6 +83,16 @@ def put_index_settings(index:, body:) super(index: index_expression_for_test_env(index), body: body) end + def update_index_aliases(body:) + actions = body.fetch("actions").map do |action| + action.transform_values do |action_body| + action_body.merge("index" => index_expression_for_test_env(action_body.fetch("index"))) + end + end + + super(body: body.merge("actions" => actions)) + end + def delete_indices(*index_names) super(*index_names.map { |index_name| index_expression_for_test_env(index_name) }) end From e1e6910f1852579238a067558a0686dd56b3be50 Mon Sep 17 00:00:00 2001 From: Mark Yang Date: Tue, 25 Aug 2026 08:53:24 +0100 Subject: [PATCH 2/7] Add integration coverage for field alias config customizations Field aliases declared via `customize_config` (mapping properties with `type: alias`) ride the existing mappings reconciliation. This covers that end-to-end for both `ForIndex` and `ForIndexTemplate`: they are included at creation, reconciled onto already-existing indices via the mapping update path, and idempotent on reconvergence. Generated with Claude Code --- .../shared_examples.rb | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/elasticgraph-admin/spec/integration/elastic_graph/admin/index_definition_configurator/shared_examples.rb b/elasticgraph-admin/spec/integration/elastic_graph/admin/index_definition_configurator/shared_examples.rb index 62d3ab21a..0b3832038 100644 --- a/elasticgraph-admin/spec/integration/elastic_graph/admin/index_definition_configurator/shared_examples.rb +++ b/elasticgraph-admin/spec/integration/elastic_graph/admin/index_definition_configurator/shared_examples.rb @@ -331,6 +331,24 @@ def simulate_presence_of_external_alias(alias_name) .and make_no_datastore_write_calls("main") end + it "applies `customize_config` mapping customizations such as field aliases when creating an index or index template, and reconciles them onto existing ones" do + field_alias_mapping = {"type" => "alias", "path" => "created_at"} + + configure_index_definition(schema_def_with_field_aliases("created")) + expect(mapping_properties_of(unique_index_name)).to include("created" => field_alias_mapping) + + expect { + configure_index_definition(schema_def_with_field_aliases("created", "creation_time")) + }.to change { mapping_properties_of(unique_index_name)["creation_time"] } + .from(nil) + .to(field_alias_mapping) + .and make_datastore_calls_to_configure_index_def(unique_index_name, :mappings) + + expect { + configure_index_definition(schema_def_with_field_aliases("created", "creation_time")) + }.to make_no_datastore_write_calls("main") + end + def schema_def_with_aliases(aliases) schema_def(configure_index: ->(index) { index.customize_config do |config| @@ -339,10 +357,24 @@ def schema_def_with_aliases(aliases) }) end + def schema_def_with_field_aliases(*field_alias_names) + schema_def(configure_index: ->(index) { + index.customize_config do |config| + field_alias_names.each do |name| + config["mappings"]["properties"][name] = {"type" => "alias", "path" => "created_at"} + end + end + }) + end + def aliases_of(index_definition_name) get_index_definition_configuration(index_definition_name)["aliases"] || {} end + def mapping_properties_of(index_definition_name) + get_index_definition_configuration(index_definition_name).dig("mappings", "properties") || {} + end + def schema_def( configure_index: nil, configure_widget: nil, From b5717b61f4cc661beb3ec448bd4181ef99876427 Mon Sep 17 00:00:00 2001 From: Mark Yang Date: Wed, 26 Aug 2026 15:26:39 +0100 Subject: [PATCH 3/7] Update elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/index.rb Co-authored-by: Myron Marston --- .../lib/elastic_graph/schema_definition/indexing/index.rb | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/index.rb b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/index.rb index 0c99d96ed..4b814c12d 100644 --- a/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/index.rb +++ b/elasticgraph-schema_definition/lib/elastic_graph/schema_definition/indexing/index.rb @@ -247,7 +247,13 @@ def has_had_multiple_sources! # configuration. # # @note ElasticGraph does nothing with the customized configuration besides passing it through to the datastore, - # and makes no claims about the safety or correctness of any customization. Use with care! + # and makes no claims about the safety or correctness of any customization. A customization the datastore + # rejects surfaces as an error when `elasticgraph-admin` configures the cluster--after earlier configuration + # steps have been applied--rather than when the schema is defined. Also note that {#rollover} affects what the + # datastore accepts: the customization goes into the index template body, which the datastore validates + # differently than a concrete index, so a customization that works without {#rollover} may be rejected with it. + # **When you add or change a customization, apply it locally against the same datastore version you run in + # production--via `bundle exec rake boot_locally`--to confirm it works as intended.** # # @yield [Hash] the datastore configuration of the index, to be mutated by the block # @return [void] From 2615aa45cf56199f911a2ad61944afdcea0ad38f Mon Sep 17 00:00:00 2001 From: Mark Yang Date: Wed, 26 Aug 2026 12:01:11 +0100 Subject: [PATCH 4/7] Address review feedback on customize_config - Replace the defensive-copy spec with one that actually fails without the deep copy: it asserts the index's own `settings` (and the very array the caller passed to `sort:`) survive customization mutations. The prior spec mutated per-type mapping hashes that are never shared across indexed types, so it passed even with the copy removed. - Add an integration spec proving mapping updates are applied before alias updates, using a `nested` alias filter that the datastore rejects while its path is unmapped. Verified it fails (with `BadDatastoreRequest`) when the order is swapped, under `NO_VCR=1` (VCR playback masks the ordering). - Prove and document `_aliases` `add` semantics: it is an upsert, so a re-declared definition fully replaces the existing one (a dropped `filter` is removed, not merged). - Simplify the empty-`aliases` check in `IndexConfigNormalizer`. The suggested `&.empty?` form trips `Lint/SafeNavigationWithEmpty`, so standardrb's corrected form is used instead. - Rename `simulate_presence_of_external_alias` to `create_external_alias`. - Use `customize_config` in the test/local schema (`teams` index): index aliases (plain and filtered) plus a field alias, so the feature is exercised whenever the schema is applied to a datastore. Generated with Claude Code --- config/schema/artifacts/datastore_config.yaml | 10 +++++- .../datastore_config.yaml | 10 +++++- config/schema/teams.rb | 9 +++++ .../for_index.rb | 2 ++ .../for_index_spec.rb | 25 ++++++++++++++ .../for_index_template_spec.rb | 2 +- .../shared_examples.rb | 11 +++++-- .../datastore_core/index_config_normalizer.rb | 2 +- .../datastore_config/customize_config_spec.rb | 33 ++++++++++++------- 9 files changed, 87 insertions(+), 17 deletions(-) diff --git a/config/schema/artifacts/datastore_config.yaml b/config/schema/artifacts/datastore_config.yaml index 9405a5d51..9a559bf94 100644 --- a/config/schema/artifacts/datastore_config.yaml +++ b/config/schema/artifacts/datastore_config.yaml @@ -6,7 +6,12 @@ index_templates: index_patterns: - teams_rollover__* template: - aliases: {} + aliases: + teams_all: {} + teams_nfl: + filter: + term: + league: NFL mappings: dynamic: strict properties: @@ -1268,6 +1273,9 @@ index_templates: __typename: type: constant_keyword value: Team + formed: + type: alias + path: formed_on _routing: required: true _size: diff --git a/config/schema/artifacts_with_apollo/datastore_config.yaml b/config/schema/artifacts_with_apollo/datastore_config.yaml index 9405a5d51..9a559bf94 100644 --- a/config/schema/artifacts_with_apollo/datastore_config.yaml +++ b/config/schema/artifacts_with_apollo/datastore_config.yaml @@ -6,7 +6,12 @@ index_templates: index_patterns: - teams_rollover__* template: - aliases: {} + aliases: + teams_all: {} + teams_nfl: + filter: + term: + league: NFL mappings: dynamic: strict properties: @@ -1268,6 +1273,9 @@ index_templates: __typename: type: constant_keyword value: Team + formed: + type: alias + path: formed_on _routing: required: true _size: diff --git a/config/schema/teams.rb b/config/schema/teams.rb index f5b511cd8..393264b49 100644 --- a/config/schema/teams.rb +++ b/config/schema/teams.rb @@ -100,6 +100,15 @@ i.route_with "league" i.rollover :yearly, "formed_on" i.has_had_multiple_sources! + # Exercises `customize_config` with both index aliases (plain and filtered) and a field alias. + i.customize_config do |config| + config["aliases"] = { + "teams_all" => {}, + "teams_nfl" => {"filter" => {"term" => {"league" => "NFL"}}} + } + + config["mappings"]["properties"]["formed"] = {"type" => "alias", "path" => "formed_on"} + end end end diff --git a/elasticgraph-admin/lib/elastic_graph/admin/index_definition_configurator/for_index.rb b/elasticgraph-admin/lib/elastic_graph/admin/index_definition_configurator/for_index.rb index a61610c13..e0ee123a9 100644 --- a/elasticgraph-admin/lib/elastic_graph/admin/index_definition_configurator/for_index.rb +++ b/elasticgraph-admin/lib/elastic_graph/admin/index_definition_configurator/for_index.rb @@ -93,6 +93,8 @@ def update_settings end def update_aliases + # An `add` action is an upsert: when the named alias already exists on the index, the + # datastore replaces its definition with the one provided here (rather than merging). actions = alias_updates.map do |name, definition| {"add" => definition.merge({"index" => @index.name, "alias" => name})} end diff --git a/elasticgraph-admin/spec/integration/elastic_graph/admin/index_definition_configurator/for_index_spec.rb b/elasticgraph-admin/spec/integration/elastic_graph/admin/index_definition_configurator/for_index_spec.rb index cc3db4277..564e2f57f 100644 --- a/elasticgraph-admin/spec/integration/elastic_graph/admin/index_definition_configurator/for_index_spec.rb +++ b/elasticgraph-admin/spec/integration/elastic_graph/admin/index_definition_configurator/for_index_spec.rb @@ -25,6 +25,31 @@ module IndexDefinitionConfigurator .and log_warning(/Can't update non dynamic setting/) end + it "applies mapping updates before alias updates so that an alias filter can reference a newly declared field" do + nested_alias = "#{unique_index_name}_nested" + # The datastore rejects a `nested` alias filter when the path is not yet in the index mapping. + nested_filter = {"nested" => {"path" => "nested_options", "query" => {"term" => {"nested_options.size" => "large"}}}} + + configure_index_definition(schema_def) + + expect { + configure_index_definition(schema_def( + configure_widget: ->(t) { + t.field "nested_options", "[WidgetOptions!]!" do |f| + f.mapping type: "nested" + end + }, + configure_index: ->(index) { + index.customize_config do |config| + config["aliases"] = {nested_alias => {"filter" => nested_filter}} + end + } + )) + }.to change { aliases_of(unique_index_name) } + .from({}) + .to({nested_alias => {"filter" => nested_filter}}) + end + it "handles empty indexed types" do schema = schema_def(define_no_widget_fields: true) diff --git a/elasticgraph-admin/spec/integration/elastic_graph/admin/index_definition_configurator/for_index_template_spec.rb b/elasticgraph-admin/spec/integration/elastic_graph/admin/index_definition_configurator/for_index_template_spec.rb index 0a9974a6c..91af72fd8 100644 --- a/elasticgraph-admin/spec/integration/elastic_graph/admin/index_definition_configurator/for_index_template_spec.rb +++ b/elasticgraph-admin/spec/integration/elastic_graph/admin/index_definition_configurator/for_index_template_spec.rb @@ -68,7 +68,7 @@ def simulate_presence_of_extra_setting(admin, index_definition_name, name, value end end - def simulate_presence_of_external_alias(alias_name) + def create_external_alias(alias_name) template = main_datastore_client.get_index_template(unique_index_name) template_body = template.fetch("template") diff --git a/elasticgraph-admin/spec/integration/elastic_graph/admin/index_definition_configurator/shared_examples.rb b/elasticgraph-admin/spec/integration/elastic_graph/admin/index_definition_configurator/shared_examples.rb index 0b3832038..a530d2b15 100644 --- a/elasticgraph-admin/spec/integration/elastic_graph/admin/index_definition_configurator/shared_examples.rb +++ b/elasticgraph-admin/spec/integration/elastic_graph/admin/index_definition_configurator/shared_examples.rb @@ -37,7 +37,7 @@ def simulate_presence_of_extra_setting(admin, index_definition_name, name, value end end - def simulate_presence_of_external_alias(alias_name) + def create_external_alias(alias_name) main_datastore_client.update_index_aliases(body: {"actions" => [ {"add" => {"index" => unique_index_name, "alias" => alias_name}} ]}) @@ -316,13 +316,20 @@ def simulate_presence_of_external_alias(alias_name) }.to change { aliases_of(unique_index_name) } .from({read_alias => {}}) .to({read_alias => {"filter" => {"term" => {"name" => "active"}}}}) + + # A changed definition fully replaces the existing one (the `filter` is dropped, not merged). + expect { + configure_index_definition(schema_def_with_aliases({read_alias => {}})) + }.to change { aliases_of(unique_index_name) } + .from({read_alias => {"filter" => {"term" => {"name" => "active"}}}}) + .to({read_alias => {}}) end it "leaves aliases it did not declare alone rather than removing them" do external_alias = "#{unique_index_name}_external" configure_index_definition(schema_def) - simulate_presence_of_external_alias(external_alias) + create_external_alias(external_alias) expect { configure_index_definition(schema_def) diff --git a/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/index_config_normalizer.rb b/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/index_config_normalizer.rb index 8bacfad99..aab8d286c 100644 --- a/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/index_config_normalizer.rb +++ b/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/index_config_normalizer.rb @@ -51,7 +51,7 @@ module IndexConfigNormalizer # - Drops an empty `aliases` hash. Empty and absent `aliases` both mean "no declared aliases", but the datastore # doesn't consistently echo the key back, so we normalize to the absent form for stable comparisons. def self.normalize(index_config) - if (aliases = index_config["aliases"]) && aliases.empty? + if index_config["aliases"] && index_config["aliases"].empty? index_config = index_config.except("aliases") end diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/datastore_config/customize_config_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/datastore_config/customize_config_spec.rb index f99733f52..5067ac22e 100644 --- a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/datastore_config/customize_config_spec.rb +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/datastore_config/customize_config_spec.rb @@ -104,26 +104,37 @@ module SchemaDefinition expect(campaigns.dig("mappings", "properties", "created")).to eq({"type" => "alias", "path" => "created_at"}) end - it "yields a defensive copy so that customization mutations cannot corrupt the config of other indices" do - campaigns, promotions = index_configs_for "campaigns", "promotions" do |s| + it "yields a defensive deep copy so that customization mutations cannot corrupt the index's own internal state" do + index = nil # : Indexing::Index? + sort_fields = ["created_at"] + + campaigns = index_configs_for "campaigns" do |s| s.object_type "Campaign" do |t| t.field "id", "ID!" + t.field "created_at", "DateTime" + + t.index "campaigns", sort: {field: sort_fields, order: ["asc"]} do |i| + index = i - t.index "campaigns" do |i| i.customize_config do |config| - config["mappings"]["properties"]["id"]["type"] = "text" + # Replacing a value one level down requires the copy to not be a bare `dup` of the config hash... + config["settings"]["index.number_of_shards"] = 17 + # ...while mutating this array in place requires the copy to be recursive, since the array is the + # very one the caller passed to `sort:` above. + config["settings"]["index.sort.field"] << "id" end end end + end.first - s.object_type "Promotion" do |t| - t.field "id", "ID!" - t.index "promotions" - end - end + expect(campaigns.dig("settings", "index.number_of_shards")).to eq(17) + expect(campaigns.dig("settings", "index.sort.field")).to eq(["created_at", "id"]) - expect(campaigns.dig("mappings", "properties", "id", "type")).to eq("text") - expect(promotions.dig("mappings", "properties", "id", "type")).to eq("keyword") + expect(index&.settings).to include( + "index.number_of_shards" => 1, + "index.sort.field" => ["created_at"] + ) + expect(sort_fields).to eq(["created_at"]) end it "raises a clear error when `customize_config` is called without a block" do From 666ff8cb3e7366678c2e8a36535115efc0775aae Mon Sep 17 00:00:00 2001 From: Mark Yang Date: Wed, 26 Aug 2026 18:24:34 +0100 Subject: [PATCH 5/7] Restore assignment-in-condition form of the empty aliases check Reverts the review round-trip on this line: the suggested `&.empty?` form trips `Lint/SafeNavigationWithEmpty`, so per review we are sticking with the original form. Generated with Claude Code --- .../lib/elastic_graph/datastore_core/index_config_normalizer.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/index_config_normalizer.rb b/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/index_config_normalizer.rb index aab8d286c..8bacfad99 100644 --- a/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/index_config_normalizer.rb +++ b/elasticgraph-datastore_core/lib/elastic_graph/datastore_core/index_config_normalizer.rb @@ -51,7 +51,7 @@ module IndexConfigNormalizer # - Drops an empty `aliases` hash. Empty and absent `aliases` both mean "no declared aliases", but the datastore # doesn't consistently echo the key back, so we normalize to the absent form for stable comparisons. def self.normalize(index_config) - if index_config["aliases"] && index_config["aliases"].empty? + if (aliases = index_config["aliases"]) && aliases.empty? index_config = index_config.except("aliases") end From bf24cd91ee29b2dc7487e1e3b2554f70eee354c4 Mon Sep 17 00:00:00 2001 From: Mark Yang Date: Wed, 26 Aug 2026 23:24:24 +0100 Subject: [PATCH 6/7] Update elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/datastore_config/customize_config_spec.rb Co-authored-by: Myron Marston --- .../schema_definition/datastore_config/customize_config_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/datastore_config/customize_config_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/datastore_config/customize_config_spec.rb index 5067ac22e..7aa66b4e7 100644 --- a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/datastore_config/customize_config_spec.rb +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/datastore_config/customize_config_spec.rb @@ -130,7 +130,7 @@ module SchemaDefinition expect(campaigns.dig("settings", "index.number_of_shards")).to eq(17) expect(campaigns.dig("settings", "index.sort.field")).to eq(["created_at", "id"]) - expect(index&.settings).to include( + expect(index.settings).to include( "index.number_of_shards" => 1, "index.sort.field" => ["created_at"] ) From 7017eae351f619d5f5486de7819df62645861e57 Mon Sep 17 00:00:00 2001 From: Mark Yang Date: Wed, 26 Aug 2026 23:24:57 +0100 Subject: [PATCH 7/7] Update elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/datastore_config/customize_config_spec.rb Co-authored-by: Myron Marston --- .../schema_definition/datastore_config/customize_config_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/datastore_config/customize_config_spec.rb b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/datastore_config/customize_config_spec.rb index 7aa66b4e7..e9ed7273d 100644 --- a/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/datastore_config/customize_config_spec.rb +++ b/elasticgraph-schema_definition/spec/unit/elastic_graph/schema_definition/datastore_config/customize_config_spec.rb @@ -105,7 +105,7 @@ module SchemaDefinition end it "yields a defensive deep copy so that customization mutations cannot corrupt the index's own internal state" do - index = nil # : Indexing::Index? + index = nil sort_fields = ["created_at"] campaigns = index_configs_for "campaigns" do |s|