From 2b11a4fab4e44200871b20f1e9b75e7aa5b84033 Mon Sep 17 00:00:00 2001 From: sivabalan Date: Mon, 6 Jul 2026 12:08:54 -0700 Subject: [PATCH 01/13] feat(writer): add hoodie.meta.fields.mode for selective meta-field population on CoW tables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduces `hoodie.meta.fields.mode` — a comma-separated list of meta columns to populate when `hoodie.populate.meta.fields=false`. Allowed tokens are `_hoodie_commit_time` and `_hoodie_file_name`; any other token is rejected up-front. The mode is immutable at runtime (settable only at table creation, via hudi-cli, or during upgrade). Motivation: the existing all-or-nothing `populate.meta.fields=false` disables incremental queries (needs `_hoodie_commit_time`) and file-level pruning / debugging lookups (needs `_hoodie_file_name`). This lets a table opt into either or both without paying for the remaining three meta columns. Resulting modes: - populate.meta.fields=true → ALL (default; mode list ignored) - populate.meta.fields=false, mode="" → NONE - populate.meta.fields=false, mode= → selective (COMMIT_TIME_ONLY / FILE_NAME_ONLY / COMMIT_TIME_AND_FILE_NAME) - populate.meta.fields=true, mode= → rejected at writer init Scope in this patch: - Spark writer path (Avro + Row): HoodieAvroParquetWriter, HoodieSparkParquetWriter, HoodieRowCreateHandle take a Set meta-fields mode and populate the columns selectively; bloom filter / record-key index registration stays skipped when record key is not populated. - Incremental read path: CoW IncrementalRelationV1/V2 accept the new mode via isCommitTimePopulated(); MoR incremental relations keep the strict populate=true guard until the MoR log-write path is updated in a follow-up. Fail-fast at writer init: - Unknown token in the mode list (parser throws). - populate.meta.fields=true combined with a non-empty mode (ambiguous). - MERGE_ON_READ + non-empty mode (log-write path not yet wired; follow-up). - Non-Spark engine + non-empty mode (Flink RowData / Java-client not yet wired; follow-up). Runtime immutability: - HoodieWriterUtils.validateTableConfig now explicitly rejects a null-on-disk → non-empty transition on the mode property (existing loop only flagged the mismatch when the on-disk value was non-null, which let this silent-drop path slip through). Tests: - TestHoodieMetaFieldsMode (hudi-hadoop-common) — accessor coverage for every combination plus unknown-token rejection and whitespace tolerance. - TestHoodieWriteConfigMetaFieldsMode (hudi-client-common) — builder + validate() coverage. - TestMetaFieldsMode (hudi-spark) — end-to-end write / re-read for every mode plus the three rejection paths (populate=true+mode, unknown token, MoR+mode). --- .../apache/hudi/config/HoodieWriteConfig.java | 64 +++++ .../TestHoodieWriteConfigMetaFieldsMode.java | 132 ++++++++++ .../storage/HoodieSparkFileWriterFactory.java | 5 +- .../io/storage/HoodieSparkParquetWriter.java | 31 +++ .../io/storage/row/HoodieRowCreateHandle.java | 44 ++++ .../hudi/common/table/HoodieTableConfig.java | 79 ++++++ .../common/table/HoodieTableMetaClient.java | 15 ++ .../hadoop/HoodieAvroFileWriterFactory.java | 5 +- .../hadoop/HoodieAvroParquetWriter.java | 37 +++ .../table/TestHoodieMetaFieldsMode.java | 145 +++++++++++ .../common/table/TestHoodieTableConfig.java | 2 +- .../apache/hudi/HoodieSparkSqlWriter.scala | 7 + .../org/apache/hudi/HoodieWriterUtils.scala | 15 ++ .../apache/hudi/IncrementalRelationV1.scala | 5 +- .../apache/hudi/IncrementalRelationV2.scala | 5 +- .../MergeOnReadIncrementalRelationV1.scala | 8 +- .../MergeOnReadIncrementalRelationV2.scala | 8 +- .../hudi/functional/TestMetaFieldsMode.java | 240 ++++++++++++++++++ 18 files changed, 838 insertions(+), 9 deletions(-) create mode 100644 hudi-client/hudi-client-common/src/test/java/org/apache/hudi/config/TestHoodieWriteConfigMetaFieldsMode.java create mode 100644 hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestHoodieMetaFieldsMode.java create mode 100644 hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestMetaFieldsMode.java diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java index 7aadac3d34079..b4dfa4d8bc88b 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java @@ -47,6 +47,7 @@ import org.apache.hudi.common.model.HoodieFailedWritesCleaningPolicy; import org.apache.hudi.common.model.HoodieFileFormat; import org.apache.hudi.common.model.HoodiePreWriteCleanerPolicy; +import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieRecordMerger; import org.apache.hudi.common.model.HoodieRecordPayload; import org.apache.hudi.common.model.HoodieTableType; @@ -1776,6 +1777,39 @@ public boolean populateMetaFields() { return getBooleanOrDefault(HoodieTableConfig.POPULATE_META_FIELDS); } + /** + * @return the raw comma-separated value of {@link HoodieTableConfig#META_FIELDS_MODE}, or an + * empty string when unset. Callers that need parsed access should use {@link #getMetaFieldsMode()}. + */ + public String getMetaFieldsModeRaw() { + return getStringOrDefault(HoodieTableConfig.META_FIELDS_MODE); + } + + /** + * @return the parsed set of meta columns opted in via {@link HoodieTableConfig#META_FIELDS_MODE}. + * Throws when an unrecognized token is present. + */ + public java.util.Set getMetaFieldsMode() { + return HoodieTableConfig.parseMetaFieldsMode(getMetaFieldsModeRaw()); + } + + /** + * @return true when {@code _hoodie_commit_time} is physically populated on every row — either + * because all meta fields are populated, or because the mode set includes {@code _hoodie_commit_time}. + */ + public boolean isCommitTimePopulated() { + return populateMetaFields() + || getMetaFieldsMode().contains(HoodieRecord.COMMIT_TIME_METADATA_FIELD); + } + + /** + * @return true when {@code _hoodie_file_name} is physically populated on every row. + */ + public boolean isFileNamePopulated() { + return populateMetaFields() + || getMetaFieldsMode().contains(HoodieRecord.FILENAME_METADATA_FIELD); + } + /** * compaction properties. */ @@ -3591,6 +3625,11 @@ public Builder withPopulateMetaFields(boolean populateMetaFields) { return this; } + public Builder withMetaFieldsMode(String metaFieldsMode) { + writeConfig.setValue(HoodieTableConfig.META_FIELDS_MODE, metaFieldsMode == null ? "" : metaFieldsMode); + return this; + } + public Builder withAllowOperationMetadataField(boolean allowOperationMetadataField) { writeConfig.setValue(ALLOW_OPERATION_METADATA_FIELD, Boolean.toString(allowOperationMetadataField)); return this; @@ -3883,6 +3922,31 @@ private void validate() { checkArgument(ttlStatsMaxParallelism > 0, String.format("%s must be positive, but was %d", HoodieTTLConfig.STATS_MAX_PARALLELISM.key(), ttlStatsMaxParallelism)); + + // hoodie.meta.fields.mode is an additive opt-in on top of populate.meta.fields=false. Setting + // populate.meta.fields=true together with a non-empty mode list is ambiguous (the mode list + // has no effect when all meta fields are already populated) so reject it explicitly rather + // than silently ignore. Also reject unknown tokens up-front — parseMetaFieldsMode throws. + boolean populateMetaFields = writeConfig.populateMetaFields(); + java.util.Set metaFieldsMode = writeConfig.getMetaFieldsMode(); + checkArgument(!(populateMetaFields && !metaFieldsMode.isEmpty()), + String.format("%s must be empty when %s=true. Disable populate.meta.fields or clear the mode list.", + HoodieTableConfig.META_FIELDS_MODE.key(), + HoodieTableConfig.POPULATE_META_FIELDS.key())); + // Selective meta-field modes are CoW-only in this release. MoR log-write path does not yet + // respect the mode set, which would silently produce log records with null meta columns. + checkArgument(!(writeConfig.getTableType() == HoodieTableType.MERGE_ON_READ && !metaFieldsMode.isEmpty()), + String.format("%s is currently supported for COPY_ON_WRITE tables only. MoR support is a follow-up. " + + "For MoR either keep %s=true or leave the mode list empty.", + HoodieTableConfig.META_FIELDS_MODE.key(), + HoodieTableConfig.POPULATE_META_FIELDS.key())); + // Selective meta-field modes are wired only for the Spark writer path in this release. Flink + // RowData / Java-client writers ignore the mode and would silently produce NONE-mode output. + checkArgument(!(engineType != EngineType.SPARK && !metaFieldsMode.isEmpty()), + String.format("%s is currently supported for the Spark writer only. Support for engine=%s is a follow-up. " + + "Either keep %s=true or leave the mode list empty.", + HoodieTableConfig.META_FIELDS_MODE.key(), engineType, + HoodieTableConfig.POPULATE_META_FIELDS.key())); } public HoodieWriteConfig build() { diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/config/TestHoodieWriteConfigMetaFieldsMode.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/config/TestHoodieWriteConfigMetaFieldsMode.java new file mode 100644 index 0000000000000..d08fcbad4d924 --- /dev/null +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/config/TestHoodieWriteConfigMetaFieldsMode.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.config; + +import org.apache.hudi.common.model.HoodieRecord; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Validates the writer-side accessors and validation guards for the meta-field-population modes + * on {@link HoodieWriteConfig}. Companion test for the {@link + * org.apache.hudi.common.table.HoodieTableConfig} accessors lives in {@code TestHoodieMetaFieldsMode}; + * this test covers the writer-builder surface and the cross-flag validation that runs at + * {@code build()} time. + */ +class TestHoodieWriteConfigMetaFieldsMode { + + private static HoodieWriteConfig.Builder baseBuilder() { + return HoodieWriteConfig.newBuilder().withPath("file:///tmp/test_hudi_meta_fields_mode"); + } + + @Test + void defaultsToAllMode() { + HoodieWriteConfig cfg = baseBuilder().build(); + assertTrue(cfg.populateMetaFields()); + assertTrue(cfg.getMetaFieldsMode().isEmpty(), + "mode list must be ignored when populate.meta.fields=true"); + assertTrue(cfg.isCommitTimePopulated()); + assertTrue(cfg.isFileNamePopulated()); + } + + @Test + void explicitNoneModeBuilds() { + HoodieWriteConfig cfg = baseBuilder().withPopulateMetaFields(false).build(); + assertFalse(cfg.populateMetaFields()); + assertTrue(cfg.getMetaFieldsMode().isEmpty()); + assertFalse(cfg.isCommitTimePopulated()); + assertFalse(cfg.isFileNamePopulated()); + } + + @Test + void commitTimeOnlyModeBuildsAndIsAdditiveOverNone() { + HoodieWriteConfig cfg = baseBuilder() + .withPopulateMetaFields(false) + .withMetaFieldsMode(HoodieRecord.COMMIT_TIME_METADATA_FIELD) + .build(); + assertFalse(cfg.populateMetaFields()); + assertTrue(cfg.isCommitTimePopulated(), + "incremental query semantics depend on _hoodie_commit_time being populated in this mode"); + assertFalse(cfg.isFileNamePopulated()); + } + + @Test + void fileNameOnlyModeBuilds() { + HoodieWriteConfig cfg = baseBuilder() + .withPopulateMetaFields(false) + .withMetaFieldsMode(HoodieRecord.FILENAME_METADATA_FIELD) + .build(); + assertFalse(cfg.populateMetaFields()); + assertFalse(cfg.isCommitTimePopulated()); + assertTrue(cfg.isFileNamePopulated()); + } + + @Test + void commitTimeAndFileNameCombinationBuilds() { + HoodieWriteConfig cfg = baseBuilder() + .withPopulateMetaFields(false) + .withMetaFieldsMode(HoodieRecord.COMMIT_TIME_METADATA_FIELD + "," + HoodieRecord.FILENAME_METADATA_FIELD) + .build(); + assertFalse(cfg.populateMetaFields()); + assertTrue(cfg.isCommitTimePopulated()); + assertTrue(cfg.isFileNamePopulated()); + } + + @Test + void rejectsIncompatibleCombination() { + // populate.meta.fields=true together with a non-empty mode is ambiguous (mode has no effect + // when all meta fields are already populated); reject loudly. + HoodieWriteConfig.Builder builder = baseBuilder() + .withPopulateMetaFields(true) + .withMetaFieldsMode(HoodieRecord.COMMIT_TIME_METADATA_FIELD); + IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, builder::build); + assertTrue(ex.getMessage().contains("hoodie.meta.fields.mode"), + "exception must name the mode property: " + ex.getMessage()); + assertTrue(ex.getMessage().contains("hoodie.populate.meta.fields"), + "exception must name the legacy property too: " + ex.getMessage()); + } + + @Test + void rejectsUnknownTokenInMode() { + HoodieWriteConfig.Builder builder = baseBuilder() + .withPopulateMetaFields(false) + .withMetaFieldsMode(HoodieRecord.RECORD_KEY_METADATA_FIELD); + IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, builder::build); + assertTrue(ex.getMessage().contains(HoodieRecord.RECORD_KEY_METADATA_FIELD), + "exception must name the rejected token: " + ex.getMessage()); + assertTrue(ex.getMessage().contains("populate.meta.fields"), + "exception must recommend populate.meta.fields=true for other columns: " + ex.getMessage()); + } + + @Test + void noneModeWithEmptyModeExplicitIsStillNone() { + HoodieWriteConfig cfg = baseBuilder() + .withPopulateMetaFields(false) + .withMetaFieldsMode("") + .build(); + assertFalse(cfg.populateMetaFields()); + assertTrue(cfg.getMetaFieldsMode().isEmpty()); + assertFalse(cfg.isCommitTimePopulated()); + assertFalse(cfg.isFileNamePopulated()); + } +} diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkFileWriterFactory.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkFileWriterFactory.java index 67949fee472e6..eb4458e134e2d 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkFileWriterFactory.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkFileWriterFactory.java @@ -57,6 +57,9 @@ protected HoodieFileWriter newParquetFileWriter( String instantTime, StoragePath path, HoodieConfig config, HoodieSchema schema, TaskContextSupplier taskContextSupplier) throws IOException { boolean populateMetaFields = config.getBooleanOrDefault(HoodieTableConfig.POPULATE_META_FIELDS); + java.util.Set metaFieldsMode = populateMetaFields + ? java.util.Collections.emptySet() + : HoodieTableConfig.parseMetaFieldsMode(config.getStringOrDefault(HoodieTableConfig.META_FIELDS_MODE)); Pair injectedConfigs = HoodieParquetConfigInjector.applyConfigInjector(path, storage.getConf(), config); StorageConfiguration storageConfiguration = injectedConfigs.getLeft(); @@ -80,7 +83,7 @@ protected HoodieFileWriter newParquetFileWriter( hoodieConfig.getBooleanOrDefault(HoodieStorageConfig.PARQUET_DICTIONARY_ENABLED)); parquetConfig.getHadoopConf().addResource(writeSupport.getHadoopConf()); - return new HoodieSparkParquetWriter(path, parquetConfig, instantTime, taskContextSupplier, populateMetaFields); + return new HoodieSparkParquetWriter(path, parquetConfig, instantTime, taskContextSupplier, populateMetaFields, metaFieldsMode); } protected HoodieFileWriter newParquetFileWriter(OutputStream outputStream, HoodieConfig config, diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkParquetWriter.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkParquetWriter.java index f2ed3831d492a..3c2c35bf11fd1 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkParquetWriter.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkParquetWriter.java @@ -30,7 +30,9 @@ import org.apache.spark.unsafe.types.UTF8String; import java.io.IOException; +import java.util.Collections; import java.util.Map; +import java.util.Set; import java.util.function.Function; import static org.apache.hudi.common.model.HoodieRecord.HoodieMetadataField.COMMIT_SEQNO_METADATA_FIELD; @@ -45,6 +47,13 @@ public class HoodieSparkParquetWriter extends HoodieBaseParquetWriter metaFieldsMode) throws IOException { super(file, parquetConfig); this.writeSupport = parquetConfig.getWriteSupport(); this.fileName = UTF8String.fromString(file.getName()); this.instantTime = UTF8String.fromString(instantTime); this.populateMetaFields = populateMetaFields; + Set mode = metaFieldsMode == null ? Collections.emptySet() : metaFieldsMode; + this.populateCommitTime = !populateMetaFields && mode.contains(HoodieRecord.COMMIT_TIME_METADATA_FIELD); + this.populateFileName = !populateMetaFields && mode.contains(HoodieRecord.FILENAME_METADATA_FIELD); this.seqIdGenerator = recordIndex -> { Integer partitionId = taskContextSupplier.getPartitionIdSupplier().get(); return HoodieRecord.generateSequenceId(instantTime, partitionId, recordIndex); @@ -74,6 +95,16 @@ public void writeRowWithMetadata(HoodieKey key, InternalRow row) throws IOExcept super.write(row); writeSupport.add(recordKey); + } else if (populateCommitTime || populateFileName) { + if (populateCommitTime) { + row.update(COMMIT_TIME_METADATA_FIELD.ordinal(), instantTime); + row.update(COMMIT_SEQNO_METADATA_FIELD.ordinal(), + UTF8String.fromString(seqIdGenerator.apply(getWrittenRecordCount()))); + } + if (populateFileName) { + row.update(FILENAME_METADATA_FIELD.ordinal(), fileName); + } + super.write(row); } else { super.write(row); } diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowCreateHandle.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowCreateHandle.java index c6d20eefb5823..db107cf2627d0 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowCreateHandle.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowCreateHandle.java @@ -67,6 +67,11 @@ public class HoodieRowCreateHandle implements Serializable { private final String fileId; private final boolean populateMetaFields; + // Selective meta-field population when populateMetaFields is false. commit_time enables + // incremental queries; file_name enables file-level pruning / investigation lookups. Other meta + // columns stay null. Record-key is never populated in these modes. + private final boolean populateCommitTime; + private final boolean populateFileName; private final UTF8String fileName; private final UTF8String commitTime; @@ -119,6 +124,11 @@ public HoodieRowCreateHandle(HoodieTable table, this.path = makeNewPath(storage, partitionPath, fileName, writeConfig); this.populateMetaFields = writeConfig.populateMetaFields(); + java.util.Set metaFieldsMode = this.populateMetaFields + ? java.util.Collections.emptySet() + : writeConfig.getMetaFieldsMode(); + this.populateCommitTime = !this.populateMetaFields && metaFieldsMode.contains(HoodieRecord.COMMIT_TIME_METADATA_FIELD); + this.populateFileName = !this.populateMetaFields && metaFieldsMode.contains(HoodieRecord.FILENAME_METADATA_FIELD); this.fileName = UTF8String.fromString(path.getName()); this.commitTime = UTF8String.fromString(instantTime); this.seqIdGenerator = (id) -> HoodieRecord.generateSequenceId(instantTime, taskPartitionId, id); @@ -160,11 +170,45 @@ public HoodieRowCreateHandle(HoodieTable table, public void write(InternalRow row) throws IOException { if (populateMetaFields) { writeRow(row); + } else if (populateCommitTime || populateFileName) { + writeRowSelectiveMetaFields(row); } else { writeRowNoMetaFields(row); } } + /** + * Selective meta-field write path: populate only the meta columns opted in via + * {@code hoodie.meta.fields.mode} — {@code _hoodie_commit_time} (with derived seq id) and/or + * {@code _hoodie_file_name}. The other meta columns stay null on disk. Record key is never + * populated in this path, so the record key is not registered with the write support (bloom + * filter / RLI hooks are meaningless without the record-key column). + */ + private void writeRowSelectiveMetaFields(InternalRow row) { + try { + UTF8String[] metaFields = new UTF8String[5]; + if (populateCommitTime) { + metaFields[HoodieRecord.COMMIT_TIME_METADATA_FIELD_ORD] = shouldPreserveHoodieMetadata + ? row.getUTF8String(HoodieRecord.COMMIT_TIME_METADATA_FIELD_ORD) : commitTime; + metaFields[HoodieRecord.COMMIT_SEQNO_METADATA_FIELD_ORD] = shouldPreserveHoodieMetadata + ? row.getUTF8String(HoodieRecord.COMMIT_SEQNO_METADATA_FIELD_ORD) + : UTF8String.fromString(seqIdGenerator.apply(GLOBAL_SEQ_NO.getAndIncrement())); + } + if (populateFileName) { + metaFields[HoodieRecord.FILENAME_META_FIELD_ORD] = shouldPreserveHoodieMetadata + ? row.getUTF8String(HoodieRecord.FILENAME_META_FIELD_ORD) : fileName; + } + // The remaining meta columns stay null — Parquet stores nulls as definition-level flags + // (zero data bytes). + InternalRow updatedRow = SparkAdapterSupport$.MODULE$.sparkAdapter().createInternalRow(metaFields, row, true); + fileWriter.writeRow(updatedRow); + writeStatus.markSuccess((HoodieRecordDelegate) null, Option.empty()); + } catch (Exception e) { + writeStatus.setGlobalError(e); + throw new HoodieException("Exception thrown while writing spark InternalRows to file ", e); + } + } + private void writeRow(InternalRow row) { try { // NOTE: PLEASE READ THIS CAREFULLY BEFORE MODIFYING diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java index 99f4cfdfd2893..eeb1f671f7a60 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java @@ -333,6 +333,15 @@ public static final String getDefaultPayloadClassName() { .withDocumentation("When enabled, populates all meta fields. When disabled, no meta fields are populated " + "and incremental queries will not be functional. This is only meant to be used for append only/immutable data for batch processing"); + public static final ConfigProperty META_FIELDS_MODE = ConfigProperty + .key("hoodie.meta.fields.mode") + .defaultValue("") + .withDocumentation("Comma-separated list of meta columns to populate when hoodie.populate.meta.fields=false. " + + "Allowed values: _hoodie_commit_time, _hoodie_file_name. Any other value is rejected — to populate the " + + "remaining meta columns, set hoodie.populate.meta.fields=true. Ignored when hoodie.populate.meta.fields=true " + + "(all meta fields are populated in that case). Set only at table creation, via the hudi-cli, or during " + + "table upgrade — the property is immutable at runtime."); + public static final ConfigProperty KEY_GENERATOR_CLASS_NAME = ConfigProperty .key("hoodie.table.keygenerator.class") .noDefaultValue() @@ -1235,6 +1244,76 @@ public boolean populateMetaFields() { return Boolean.parseBoolean(getStringOrDefault(POPULATE_META_FIELDS)); } + /** + * @return the set of meta columns explicitly opted in via {@link #META_FIELDS_MODE}. Empty when + * the mode is unset or when {@link #POPULATE_META_FIELDS} is {@code true} (in which case all meta + * columns are populated and the mode list is ignored). Throws when an unrecognized token is + * present — only {@code _hoodie_commit_time} and {@code _hoodie_file_name} are allowed here; to + * populate other meta columns, set {@code hoodie.populate.meta.fields=true}. + */ + public Set getMetaFieldsMode() { + return parseMetaFieldsMode(getStringOrDefault(META_FIELDS_MODE)); + } + + /** + * Package-visible helper so writer-side validation and CLI can parse without going through a + * {@link HoodieTableConfig} instance. + */ + public static Set parseMetaFieldsMode(String raw) { + if (raw == null || raw.trim().isEmpty()) { + return Collections.emptySet(); + } + Set result = new HashSet<>(); + for (String token : raw.split(",")) { + String field = token.trim(); + if (field.isEmpty()) { + continue; + } + if (!ALLOWED_META_FIELDS_MODE_VALUES.contains(field)) { + throw new IllegalArgumentException(String.format( + "Unsupported value '%s' for %s. Allowed values are %s. To populate other meta columns, set %s=true.", + field, META_FIELDS_MODE.key(), ALLOWED_META_FIELDS_MODE_VALUES, POPULATE_META_FIELDS.key())); + } + result.add(field); + } + return Collections.unmodifiableSet(result); + } + + /** + * Allowed tokens for {@link #META_FIELDS_MODE}. Any other meta-field name is rejected at parse + * time to preserve the "either all meta fields, none, or the two selectable ones" invariant. + */ + public static final Set ALLOWED_META_FIELDS_MODE_VALUES = Collections.unmodifiableSet( + new HashSet<>(Arrays.asList( + HoodieRecord.COMMIT_TIME_METADATA_FIELD, + HoodieRecord.FILENAME_METADATA_FIELD))); + + /** + * @return true when the {@code _hoodie_commit_time} meta column is physically populated on disk — + * i.e., either all meta fields are populated, or the mode set includes {@code _hoodie_commit_time}. + */ + public boolean isCommitTimePopulated() { + return populateMetaFields() || getMetaFieldsMode().contains(HoodieRecord.COMMIT_TIME_METADATA_FIELD); + } + + /** + * @return true when the {@code _hoodie_file_name} meta column is physically populated on disk — + * i.e., either all meta fields are populated, or the mode set includes {@code _hoodie_file_name}. + */ + public boolean isFileNamePopulated() { + return populateMetaFields() || getMetaFieldsMode().contains(HoodieRecord.FILENAME_METADATA_FIELD); + } + + /** + * @return true when the {@code _hoodie_record_key} meta column is physically populated on disk. + * The selective mode never populates the record-key column, so this exactly mirrors + * {@link #populateMetaFields()} — exposed separately so callers that specifically care about the + * record-key column do not need to be updated again if the mode set is extended. + */ + public boolean isRecordKeyPopulated() { + return populateMetaFields(); + } + /** * @returns the record key field prop. */ diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java index 0f3f8e7bf64b0..0778b549c25a6 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java @@ -1013,6 +1013,7 @@ public static class TableBuilder { private String bootstrapBasePath; private Boolean bootstrapIndexEnable; private Boolean populateMetaFields; + private String metaFieldsMode; private String keyGeneratorClassProp; private String partitionValueExtractorClass; private String keyGeneratorType; @@ -1170,6 +1171,11 @@ public TableBuilder setPopulateMetaFields(boolean populateMetaFields) { return this; } + public TableBuilder setMetaFieldsMode(String metaFieldsMode) { + this.metaFieldsMode = metaFieldsMode; + return this; + } + public TableBuilder setKeyGeneratorClassProp(String keyGeneratorClassProp) { this.keyGeneratorClassProp = keyGeneratorClassProp; return this; @@ -1384,6 +1390,9 @@ public TableBuilder fromProperties(Properties properties) { if (hoodieConfig.contains(HoodieTableConfig.POPULATE_META_FIELDS)) { setPopulateMetaFields(hoodieConfig.getBoolean(HoodieTableConfig.POPULATE_META_FIELDS)); } + if (hoodieConfig.contains(HoodieTableConfig.META_FIELDS_MODE)) { + setMetaFieldsMode(hoodieConfig.getString(HoodieTableConfig.META_FIELDS_MODE)); + } if (hoodieConfig.contains(HoodieTableConfig.KEY_GENERATOR_CLASS_NAME)) { setKeyGeneratorClassProp(hoodieConfig.getString(HoodieTableConfig.KEY_GENERATOR_CLASS_NAME)); } else if (hoodieConfig.contains(HoodieTableConfig.KEY_GENERATOR_TYPE)) { @@ -1522,6 +1531,12 @@ public Properties build() { if (null != populateMetaFields) { tableConfig.setValue(HoodieTableConfig.POPULATE_META_FIELDS, Boolean.toString(populateMetaFields)); } + if (null != metaFieldsMode) { + // Validate at table-init time; parseMetaFieldsMode throws on unknown tokens so the bad + // property never makes it onto disk. + HoodieTableConfig.parseMetaFieldsMode(metaFieldsMode); + tableConfig.setValue(HoodieTableConfig.META_FIELDS_MODE, metaFieldsMode); + } if (null != keyGeneratorClassProp) { KeyGeneratorType type = KeyGeneratorType.fromClassName(keyGeneratorClassProp); tableConfig.setValue(HoodieTableConfig.KEY_GENERATOR_TYPE, type.name()); diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroFileWriterFactory.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroFileWriterFactory.java index 272de5d4a3103..e70d84af58c6a 100644 --- a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroFileWriterFactory.java +++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroFileWriterFactory.java @@ -70,6 +70,9 @@ protected HoodieFileWriter newParquetFileWriter( String instantTime, StoragePath path, HoodieConfig config, HoodieSchema schema, TaskContextSupplier taskContextSupplier) throws IOException { boolean populateMetaFields = config.getBooleanOrDefault(HoodieTableConfig.POPULATE_META_FIELDS); + java.util.Set metaFieldsMode = populateMetaFields + ? java.util.Collections.emptySet() + : HoodieTableConfig.parseMetaFieldsMode(config.getStringOrDefault(HoodieTableConfig.META_FIELDS_MODE)); Pair injectedConfigs = HoodieParquetConfigInjector.applyConfigInjector(path, storage.getConf(), config); StorageConfiguration storageConfiguration = injectedConfigs.getLeft(); @@ -89,7 +92,7 @@ protected HoodieFileWriter newParquetFileWriter( hoodieConfig.getLongOrDefault(HoodieStorageConfig.PARQUET_MAX_FILE_SIZE), storageConfiguration, hoodieConfig.getDoubleOrDefault(HoodieStorageConfig.PARQUET_COMPRESSION_RATIO_FRACTION), hoodieConfig.getBooleanOrDefault(HoodieStorageConfig.PARQUET_DICTIONARY_ENABLED)); - return new HoodieAvroParquetWriter(path, parquetConfig, instantTime, taskContextSupplier, populateMetaFields); + return new HoodieAvroParquetWriter(path, parquetConfig, instantTime, taskContextSupplier, populateMetaFields, metaFieldsMode); } protected HoodieFileWriter newParquetFileWriter( diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroParquetWriter.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroParquetWriter.java index 1fb381b26b0ef..6564c48558bd2 100644 --- a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroParquetWriter.java +++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroParquetWriter.java @@ -19,20 +19,25 @@ package org.apache.hudi.io.storage.hadoop; +import org.apache.hudi.avro.HoodieAvroUtils; import org.apache.hudi.avro.HoodieAvroWriteSupport; import org.apache.hudi.common.config.HoodieParquetConfig; import org.apache.hudi.common.engine.TaskContextSupplier; import org.apache.hudi.common.model.HoodieKey; +import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.core.io.storage.HoodieAvroFileWriter; import org.apache.hudi.io.hadoop.HoodieBaseParquetWriter; import org.apache.hudi.storage.StoragePath; +import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.IndexedRecord; import javax.annotation.concurrent.NotThreadSafe; import java.io.IOException; +import java.util.Collections; import java.util.Map; +import java.util.Set; /** * HoodieParquetWriter extends the ParquetWriter to help limit the size of underlying file. Provides a way to check if @@ -49,6 +54,11 @@ public class HoodieAvroParquetWriter private final String instantTime; private final TaskContextSupplier taskContextSupplier; private final boolean populateMetaFields; + // Meta fields to populate when populateMetaFields is false. Selectively enables + // _hoodie_commit_time / _hoodie_file_name so incremental queries and file-level lookups keep + // working on otherwise-minimal-meta-field tables. Ignored when populateMetaFields is true. + private final boolean populateCommitTime; + private final boolean populateFileName; private final HoodieAvroWriteSupport writeSupport; @SuppressWarnings({"unchecked", "rawtypes"}) @@ -57,12 +67,25 @@ public HoodieAvroParquetWriter(StoragePath file, String instantTime, TaskContextSupplier taskContextSupplier, boolean populateMetaFields) throws IOException { + this(file, parquetConfig, instantTime, taskContextSupplier, populateMetaFields, Collections.emptySet()); + } + + @SuppressWarnings({"unchecked", "rawtypes"}) + public HoodieAvroParquetWriter(StoragePath file, + HoodieParquetConfig parquetConfig, + String instantTime, + TaskContextSupplier taskContextSupplier, + boolean populateMetaFields, + Set metaFieldsMode) throws IOException { super(file, (HoodieParquetConfig) parquetConfig); this.fileName = file.getName(); this.writeSupport = parquetConfig.getWriteSupport(); this.instantTime = instantTime; this.taskContextSupplier = taskContextSupplier; this.populateMetaFields = populateMetaFields; + Set mode = metaFieldsMode == null ? Collections.emptySet() : metaFieldsMode; + this.populateCommitTime = !populateMetaFields && mode.contains(HoodieRecord.COMMIT_TIME_METADATA_FIELD); + this.populateFileName = !populateMetaFields && mode.contains(HoodieRecord.FILENAME_METADATA_FIELD); } @Override @@ -72,6 +95,20 @@ public void writeAvroWithMetadata(HoodieKey key, IndexedRecord avroRecord) throw taskContextSupplier.getPartitionIdSupplier().get(), getWrittenRecordCount(), fileName); super.write(avroRecord); writeSupport.add(key.getRecordKey()); + } else if (populateCommitTime || populateFileName) { + // Selective meta-field population. The other meta columns stay null on disk, which Parquet + // stores as definition-level flags (zero data bytes). Bloom filter / record-key index + // population is intentionally skipped — that requires the record-key column. + GenericRecord genericRecord = (GenericRecord) avroRecord; + if (populateCommitTime) { + String seqId = HoodieRecord.generateSequenceId(instantTime, + taskContextSupplier.getPartitionIdSupplier().get(), getWrittenRecordCount()); + HoodieAvroUtils.addCommitMetadataToRecord(genericRecord, instantTime, seqId); + } + if (populateFileName) { + genericRecord.put(HoodieRecord.FILENAME_METADATA_FIELD, fileName); + } + super.write(avroRecord); } else { super.write(avroRecord); } diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestHoodieMetaFieldsMode.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestHoodieMetaFieldsMode.java new file mode 100644 index 0000000000000..f343786dfbdf4 --- /dev/null +++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestHoodieMetaFieldsMode.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.common.table; + +import org.apache.hudi.common.model.HoodieRecord; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests the meta-field-population modes exposed by {@link HoodieTableConfig} via the + * {@code hoodie.meta.fields.mode} property: + * + *
    + *
  • ALL — {@code populate.meta.fields=true} (default). All five meta columns populated.
    • + *
  • NONE — {@code populate.meta.fields=false} and mode empty. No meta columns populated.
    • + *
  • COMMIT_TIME_ONLY — {@code populate.meta.fields=false}, + * {@code meta.fields.mode=_hoodie_commit_time}. Only commit-time populated.
    • + *
  • FILE_NAME_ONLY — same but with {@code _hoodie_file_name}.
    • + *
  • COMMIT_TIME_AND_FILE_NAME — both tokens in the mode list.
    • + *
+ * + *

Tokens other than the two allowed ones are rejected up-front by the parser. This test + * exercises {@link HoodieTableConfig} accessors directly without touching the storage layer. + */ +class TestHoodieMetaFieldsMode { + + private static HoodieTableConfig configOf(Boolean populate, String mode) { + HoodieTableConfig cfg = new HoodieTableConfig(); + if (populate != null) { + cfg.setValue(HoodieTableConfig.POPULATE_META_FIELDS, String.valueOf(populate)); + } + if (mode != null) { + cfg.setValue(HoodieTableConfig.META_FIELDS_MODE, mode); + } + return cfg; + } + + @Test + void defaultsResolveToAllMode() { + HoodieTableConfig cfg = configOf(null, null); + assertTrue(cfg.populateMetaFields(), "populateMetaFields default must remain true"); + assertTrue(cfg.getMetaFieldsMode().isEmpty(), "mode list defaults to empty"); + assertTrue(cfg.isCommitTimePopulated(), "commit time must be populated in ALL mode"); + assertTrue(cfg.isFileNamePopulated(), "file name must be populated in ALL mode"); + assertTrue(cfg.isRecordKeyPopulated(), "record key must be populated in ALL mode"); + } + + @Test + void noneModeWhenPopulateFalseAndModeEmpty() { + HoodieTableConfig cfg = configOf(false, ""); + assertFalse(cfg.populateMetaFields()); + assertTrue(cfg.getMetaFieldsMode().isEmpty()); + assertFalse(cfg.isCommitTimePopulated()); + assertFalse(cfg.isFileNamePopulated()); + assertFalse(cfg.isRecordKeyPopulated()); + } + + @Test + void noneModeWhenPopulateFalseAndModeUnset() { + // Existing populate.meta.fields=false table without the mode property must still resolve to NONE. + HoodieTableConfig cfg = configOf(false, null); + assertFalse(cfg.populateMetaFields()); + assertTrue(cfg.getMetaFieldsMode().isEmpty()); + assertFalse(cfg.isCommitTimePopulated()); + assertFalse(cfg.isFileNamePopulated()); + assertFalse(cfg.isRecordKeyPopulated()); + } + + @Test + void commitTimeOnlyMode() { + HoodieTableConfig cfg = configOf(false, HoodieRecord.COMMIT_TIME_METADATA_FIELD); + assertFalse(cfg.populateMetaFields()); + assertTrue(cfg.isCommitTimePopulated()); + assertFalse(cfg.isFileNamePopulated()); + assertFalse(cfg.isRecordKeyPopulated()); + } + + @Test + void fileNameOnlyMode() { + HoodieTableConfig cfg = configOf(false, HoodieRecord.FILENAME_METADATA_FIELD); + assertFalse(cfg.populateMetaFields()); + assertFalse(cfg.isCommitTimePopulated()); + assertTrue(cfg.isFileNamePopulated()); + assertFalse(cfg.isRecordKeyPopulated()); + } + + @Test + void commitTimeAndFileNameMode() { + HoodieTableConfig cfg = configOf(false, + HoodieRecord.COMMIT_TIME_METADATA_FIELD + "," + HoodieRecord.FILENAME_METADATA_FIELD); + assertFalse(cfg.populateMetaFields()); + assertTrue(cfg.isCommitTimePopulated()); + assertTrue(cfg.isFileNamePopulated()); + assertFalse(cfg.isRecordKeyPopulated()); + } + + @Test + void modeIsIgnoredWhenPopulateMetaFieldsIsTrue() { + // Note: writer-side validate() rejects this combination, but the accessor must still report + // ALL semantics defensively if a bad combo ever leaks through (mode has no effect when all + // meta fields are already populated). + HoodieTableConfig cfg = configOf(true, HoodieRecord.COMMIT_TIME_METADATA_FIELD); + assertTrue(cfg.populateMetaFields()); + assertTrue(cfg.isCommitTimePopulated()); + assertTrue(cfg.isFileNamePopulated()); + assertTrue(cfg.isRecordKeyPopulated()); + } + + @Test + void unknownTokenIsRejected() { + HoodieTableConfig cfg = configOf(false, HoodieRecord.RECORD_KEY_METADATA_FIELD); + IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, cfg::getMetaFieldsMode); + assertTrue(ex.getMessage().contains(HoodieRecord.RECORD_KEY_METADATA_FIELD), + "message must name the rejected token: " + ex.getMessage()); + assertTrue(ex.getMessage().contains("populate.meta.fields"), + "message must recommend populate.meta.fields=true for other columns: " + ex.getMessage()); + } + + @Test + void whitespaceAndEmptyTokensAreTolerated() { + HoodieTableConfig cfg = configOf(false, " " + HoodieRecord.COMMIT_TIME_METADATA_FIELD + " , "); + assertTrue(cfg.isCommitTimePopulated()); + assertFalse(cfg.isFileNamePopulated()); + } +} diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableConfig.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableConfig.java index ff21ec822f821..273abd7a31e51 100644 --- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableConfig.java +++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableConfig.java @@ -386,7 +386,7 @@ void testDropInvalidConfigs() { @Test void testDefinedTableConfigs() { List> configProperties = HoodieTableConfig.definedTableConfigs(); - assertEquals(45, configProperties.size()); + assertEquals(46, configProperties.size()); configProperties.forEach(c -> { assertNotNull(c); assertFalse(c.doc().isEmpty()); diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala index 37f8e569a0e7a..c2580a0e37dd3 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala @@ -284,6 +284,7 @@ class HoodieSparkSqlWriterInternal { val baseFileFormat = hoodieConfig.getStringOrDefault(HoodieTableConfig.BASE_FILE_FORMAT) val archiveLogFolder = hoodieConfig.getStringOrDefault(HoodieTableConfig.TIMELINE_HISTORY_PATH) val populateMetaFields = hoodieConfig.getBooleanOrDefault(HoodieTableConfig.POPULATE_META_FIELDS) + val metaFieldsMode = hoodieConfig.getStringOrDefault(HoodieTableConfig.META_FIELDS_MODE) val useBaseFormatMetaFile = hoodieConfig.getBooleanOrDefault(HoodieTableConfig.PARTITION_METAFILE_USE_BASE_FORMAT); val payloadClass = hoodieConfig.getString(DataSourceWriteOptions.PAYLOAD_CLASS_NAME) val recordMergeStrategyId = hoodieConfig.getString(DataSourceWriteOptions.RECORD_MERGE_STRATEGY_ID) @@ -308,6 +309,7 @@ class HoodieSparkSqlWriterInternal { .setOrderingFields(ConfigUtils.getOrderingFieldsStrDuringWrite(optParams.asJava)) .setPartitionFields(partitionColumnsForKeyGenerator) .setPopulateMetaFields(populateMetaFields) + .setMetaFieldsMode(metaFieldsMode) .setRecordKeyFields(hoodieConfig.getString(RECORDKEY_FIELD)) .setSecondaryKeyFields(hoodieConfig.getString(SECONDARYKEY_COLUMN_NAME)) .setCDCEnabled(hoodieConfig.getBooleanOrDefault(HoodieTableConfig.CDC_ENABLED)) @@ -754,6 +756,10 @@ class HoodieSparkSqlWriterInternal { HoodieTableConfig.POPULATE_META_FIELDS.key(), String.valueOf(HoodieTableConfig.POPULATE_META_FIELDS.defaultValue()) )) + val metaFieldsMode = parameters.getOrElse( + HoodieTableConfig.META_FIELDS_MODE.key(), + HoodieTableConfig.META_FIELDS_MODE.defaultValue() + ) val baseFileFormat = hoodieConfig.getStringOrDefault(HoodieTableConfig.BASE_FILE_FORMAT) val useBaseFormatMetaFile = java.lang.Boolean.parseBoolean(parameters.getOrElse( HoodieTableConfig.PARTITION_METAFILE_USE_BASE_FORMAT.key(), @@ -780,6 +786,7 @@ class HoodieSparkSqlWriterInternal { .setCDCEnabled(hoodieConfig.getBooleanOrDefault(HoodieTableConfig.CDC_ENABLED)) .setCDCSupplementalLoggingMode(hoodieConfig.getStringOrDefault(HoodieTableConfig.CDC_SUPPLEMENTAL_LOGGING_MODE)) .setPopulateMetaFields(populateMetaFields) + .setMetaFieldsMode(metaFieldsMode) .setKeyGeneratorClassProp(keyGenProp) .setPartitionValueExtractorClass(partitionValueExtractorClassName) .set(timestampKeyGeneratorConfigs.asJava.asInstanceOf[java.util.Map[String, Object]]) diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieWriterUtils.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieWriterUtils.scala index 54459ff61e00a..bab13c98a4156 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieWriterUtils.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieWriterUtils.scala @@ -352,6 +352,21 @@ object HoodieWriterUtils { diffConfigs.append(s"${HoodieTableConfig.RECORD_MERGE_STRATEGY_ID}:\t$mergeStrategyId\tnull\n") } } + + // hoodie.meta.fields.mode is a physical-storage decision baked into files at write time. + // Changing it at runtime would silently produce mixed-mode files whose incremental / file + // pruning behavior differs between old and new commits. The default loop above only flags + // the mismatch when the on-disk value is non-null, so an older table with the property + // absent from hoodie.properties would let a null → non-empty transition slip through + // (silent-drop risk on pre-enablement commits). Guard the null → non-empty case explicitly + // here. Set it only at table creation, via the hudi-cli, or during table upgrade. + val paramsMetaFieldsMode = params.getOrElse(HoodieTableConfig.META_FIELDS_MODE.key(), "") + val onDiskMetaFieldsMode = tableConfig.getString(HoodieTableConfig.META_FIELDS_MODE) + if (paramsMetaFieldsMode.nonEmpty && (onDiskMetaFieldsMode == null || onDiskMetaFieldsMode.isEmpty)) { + diffConfigs.append( + s"${HoodieTableConfig.META_FIELDS_MODE.key()}:\t$paramsMetaFieldsMode\t${if (onDiskMetaFieldsMode == null) "null" else "\"\""}" + + " (immutable at runtime; set only at table creation / hudi-cli / upgrade)\n") + } } if (diffConfigs.nonEmpty) { diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/IncrementalRelationV1.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/IncrementalRelationV1.scala index 50db4c678c8f8..db33f735a6508 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/IncrementalRelationV1.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/IncrementalRelationV1.scala @@ -89,8 +89,9 @@ class IncrementalRelationV1(val sqlContext: SQLContext, s"option ${DataSourceReadOptions.START_COMMIT.key}") } - if (!metaClient.getTableConfig.populateMetaFields()) { - throw new HoodieException("Incremental queries are not supported when meta fields are disabled") + if (!metaClient.getTableConfig.isCommitTimePopulated()) { + throw new HoodieException("Incremental queries are not supported when _hoodie_commit_time is not populated. " + + "Either keep hoodie.populate.meta.fields=true or include _hoodie_commit_time in hoodie.meta.fields.mode.") } private val useEndInstantSchema = optParams.getOrElse(INCREMENTAL_READ_SCHEMA_USE_END_INSTANTTIME.key, diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/IncrementalRelationV2.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/IncrementalRelationV2.scala index 308ca87a47592..311e24baad25d 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/IncrementalRelationV2.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/IncrementalRelationV2.scala @@ -78,8 +78,9 @@ class IncrementalRelationV2(val sqlContext: SQLContext, s"option ${DataSourceReadOptions.START_COMMIT.key}") } - if (!metaClient.getTableConfig.populateMetaFields()) { - throw new HoodieException("Incremental queries are not supported when meta fields are disabled") + if (!metaClient.getTableConfig.isCommitTimePopulated()) { + throw new HoodieException("Incremental queries are not supported when _hoodie_commit_time is not populated. " + + "Either keep hoodie.populate.meta.fields=true or include _hoodie_commit_time in hoodie.meta.fields.mode.") } private val queryContext: IncrementalQueryAnalyzer.QueryContext = diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/MergeOnReadIncrementalRelationV1.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/MergeOnReadIncrementalRelationV1.scala index e0fc7ceb6824b..f43071ca2ee89 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/MergeOnReadIncrementalRelationV1.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/MergeOnReadIncrementalRelationV1.scala @@ -267,8 +267,14 @@ trait HoodieIncrementalRelationV1Trait extends HoodieBaseRelation { s"option ${DataSourceReadOptions.START_COMMIT.key}") } + // MoR incremental relies on _hoodie_commit_time being present in BOTH base files AND log + // records. The base-file writer respects hoodie.meta.fields.mode, but the log-write path + // (HoodieAppendHandle) does not yet — until that gap is closed, MoR incremental must require + // populate.meta.fields=true to avoid silently dropping log-file rows whose commit_time is null. if (!this.tableConfig.populateMetaFields()) { - throw new HoodieException("Incremental queries are not supported when meta fields are disabled") + throw new HoodieException("Incremental queries on MoR tables are not supported when " + + "hoodie.populate.meta.fields=false. Selective meta-field modes (hoodie.meta.fields.mode) " + + "are supported for CoW only in this release; MoR support is tracked as a follow-up.") } if (hollowCommitHandling == USE_TRANSITION_TIME && fullTableScan) { diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/MergeOnReadIncrementalRelationV2.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/MergeOnReadIncrementalRelationV2.scala index aea594d9157ca..5e9a611bc6637 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/MergeOnReadIncrementalRelationV2.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/MergeOnReadIncrementalRelationV2.scala @@ -257,8 +257,14 @@ trait HoodieIncrementalRelationV2Trait extends HoodieBaseRelation { s"option ${DataSourceReadOptions.START_COMMIT.key}") } + // MoR incremental relies on _hoodie_commit_time being present in BOTH base files AND log + // records. The base-file writer respects hoodie.meta.fields.mode, but the log-write path + // (HoodieAppendHandle) does not yet — until that gap is closed, MoR incremental must require + // populate.meta.fields=true to avoid silently dropping log-file rows whose commit_time is null. if (!this.tableConfig.populateMetaFields()) { - throw new HoodieException("Incremental queries are not supported when meta fields are disabled") + throw new HoodieException("Incremental queries on MoR tables are not supported when " + + "hoodie.populate.meta.fields=false. Selective meta-field modes (hoodie.meta.fields.mode) " + + "are supported for CoW only in this release; MoR support is tracked as a follow-up.") } } diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestMetaFieldsMode.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestMetaFieldsMode.java new file mode 100644 index 0000000000000..12c01e303708d --- /dev/null +++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestMetaFieldsMode.java @@ -0,0 +1,240 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.functional; + +import org.apache.hudi.DataSourceWriteOptions; +import org.apache.hudi.SparkAdapterSupport$; +import org.apache.hudi.common.config.HoodieMetadataConfig; +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.table.HoodieTableConfig; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.testutils.SparkClientFunctionalTestHarness; + +import org.apache.spark.sql.Row; +import org.apache.spark.sql.RowFactory; +import org.apache.spark.sql.SaveMode; +import org.apache.spark.sql.types.DataTypes; +import org.apache.spark.sql.types.StructField; +import org.apache.spark.sql.types.StructType; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Spark-datasource tests for the {@code hoodie.meta.fields.mode} property on CoW tables. + * + *

Modes covered: + *

    + *
  • ALL — {@code populate.meta.fields=true} (default).
    • + *
  • NONE — {@code populate.meta.fields=false} and mode empty.
    • + *
  • COMMIT_TIME_ONLY — {@code mode=_hoodie_commit_time}.
    • + *
  • FILE_NAME_ONLY — {@code mode=_hoodie_file_name}.
    • + *
  • COMMIT_TIME_AND_FILE_NAME — both tokens in the mode list.
    • + *
+ * + *

Rejection paths: an unknown token or the ambiguous + * {@code populate.meta.fields=true} + non-empty mode combination must fail at writer init. MoR + + * non-empty mode is likewise rejected until log-write support lands (tracked as follow-up). + */ +class TestMetaFieldsMode extends SparkClientFunctionalTestHarness { + + private static final String COMMIT_TIME_TOKEN = HoodieRecord.COMMIT_TIME_METADATA_FIELD; + private static final String FILE_NAME_TOKEN = HoodieRecord.FILENAME_METADATA_FIELD; + + private static StructType simpleSchema() { + return DataTypes.createStructType(new StructField[]{ + DataTypes.createStructField("column1", DataTypes.StringType, true), + DataTypes.createStructField("column2", DataTypes.StringType, true), + DataTypes.createStructField("column3", DataTypes.StringType, true) + }).asNullable(); + } + + private Map baseOptions() { + Map opts = new HashMap<>(); + opts.put(DataSourceWriteOptions.RECORDKEY_FIELD().key(), "column1"); + opts.put(DataSourceWriteOptions.PARTITIONPATH_FIELD().key(), "column2"); + opts.put(DataSourceWriteOptions.ORDERING_FIELDS().key(), "column3"); + opts.put(HoodieTableConfig.NAME.key(), "test_meta_fields_mode"); + opts.put(DataSourceWriteOptions.TABLE_TYPE().key(), "COPY_ON_WRITE"); + opts.put(HoodieMetadataConfig.ENABLE.key(), "false"); + return opts; + } + + private void writeRows(List records, StructType schema, Map options, String path, SaveMode mode) { + spark().createDataset(records, + SparkAdapterSupport$.MODULE$.sparkAdapter().getCatalystExpressionUtils().getEncoder(schema)) + .write() + .format("hudi") + .options(options) + .mode(mode) + .save(path); + } + + private HoodieTableConfig writeAndReadTableConfig(Map options, String path) { + writeRows(Collections.singletonList(RowFactory.create("k1", "p1", "v1")), + simpleSchema(), options, path, SaveMode.Overwrite); + HoodieTableMetaClient metaClient = + HoodieTableMetaClient.builder().setBasePath(path).setConf(storageConf()).build(); + return metaClient.getTableConfig(); + } + + @Test + void commitTimeOnlyModePersistsPropertyAndReportsMode() { + Map options = baseOptions(); + options.put(HoodieTableConfig.POPULATE_META_FIELDS.key(), "false"); + options.put(HoodieTableConfig.META_FIELDS_MODE.key(), COMMIT_TIME_TOKEN); + options.put(DataSourceWriteOptions.OPERATION().key(), DataSourceWriteOptions.BULK_INSERT_OPERATION_OPT_VAL()); + + HoodieTableConfig tc = writeAndReadTableConfig(options, basePath()); + + assertEquals("false", tc.getProps().getProperty(HoodieTableConfig.POPULATE_META_FIELDS.key())); + assertEquals(COMMIT_TIME_TOKEN, tc.getProps().getProperty(HoodieTableConfig.META_FIELDS_MODE.key())); + assertFalse(tc.populateMetaFields()); + assertTrue(tc.isCommitTimePopulated()); + assertFalse(tc.isFileNamePopulated()); + assertFalse(tc.isRecordKeyPopulated()); + } + + @Test + void fileNameOnlyModePersistsPropertyAndReportsMode() { + Map options = baseOptions(); + options.put(HoodieTableConfig.POPULATE_META_FIELDS.key(), "false"); + options.put(HoodieTableConfig.META_FIELDS_MODE.key(), FILE_NAME_TOKEN); + options.put(DataSourceWriteOptions.OPERATION().key(), DataSourceWriteOptions.BULK_INSERT_OPERATION_OPT_VAL()); + + HoodieTableConfig tc = writeAndReadTableConfig(options, basePath()); + + assertEquals(FILE_NAME_TOKEN, tc.getProps().getProperty(HoodieTableConfig.META_FIELDS_MODE.key())); + assertFalse(tc.populateMetaFields()); + assertFalse(tc.isCommitTimePopulated()); + assertTrue(tc.isFileNamePopulated()); + assertFalse(tc.isRecordKeyPopulated()); + } + + @Test + void commitTimeAndFileNameModePersistsAndReports() { + String combined = COMMIT_TIME_TOKEN + "," + FILE_NAME_TOKEN; + Map options = baseOptions(); + options.put(HoodieTableConfig.POPULATE_META_FIELDS.key(), "false"); + options.put(HoodieTableConfig.META_FIELDS_MODE.key(), combined); + options.put(DataSourceWriteOptions.OPERATION().key(), DataSourceWriteOptions.BULK_INSERT_OPERATION_OPT_VAL()); + + HoodieTableConfig tc = writeAndReadTableConfig(options, basePath()); + + assertEquals(combined, tc.getProps().getProperty(HoodieTableConfig.META_FIELDS_MODE.key())); + assertTrue(tc.isCommitTimePopulated()); + assertTrue(tc.isFileNamePopulated()); + assertFalse(tc.isRecordKeyPopulated()); + } + + @Test + void noneModePersistsAndReportsCorrectly() { + Map options = baseOptions(); + options.put(HoodieTableConfig.POPULATE_META_FIELDS.key(), "false"); + options.put(DataSourceWriteOptions.OPERATION().key(), DataSourceWriteOptions.BULK_INSERT_OPERATION_OPT_VAL()); + + HoodieTableConfig tc = writeAndReadTableConfig(options, basePath()); + + assertFalse(tc.populateMetaFields()); + assertTrue(tc.getMetaFieldsMode().isEmpty()); + assertFalse(tc.isCommitTimePopulated()); + assertFalse(tc.isFileNamePopulated()); + assertFalse(tc.isRecordKeyPopulated()); + } + + @Test + void defaultAllModeReportsPopulated() { + Map options = baseOptions(); + options.put(DataSourceWriteOptions.OPERATION().key(), DataSourceWriteOptions.BULK_INSERT_OPERATION_OPT_VAL()); + + HoodieTableConfig tc = writeAndReadTableConfig(options, basePath()); + + assertTrue(tc.populateMetaFields()); + assertTrue(tc.isCommitTimePopulated()); + assertTrue(tc.isFileNamePopulated()); + assertTrue(tc.isRecordKeyPopulated()); + } + + @Test + void populateTrueWithNonEmptyModeIsRejected() { + Map options = baseOptions(); + options.put(HoodieTableConfig.POPULATE_META_FIELDS.key(), "true"); + options.put(HoodieTableConfig.META_FIELDS_MODE.key(), COMMIT_TIME_TOKEN); + options.put(DataSourceWriteOptions.OPERATION().key(), DataSourceWriteOptions.BULK_INSERT_OPERATION_OPT_VAL()); + + Throwable thrown = assertThrows(Throwable.class, () -> + writeRows(Collections.singletonList(RowFactory.create("k1", "p1", "v1")), + simpleSchema(), options, basePath(), SaveMode.Overwrite)); + + String rootMessage = rootMessageOf(thrown); + assertTrue(rootMessage.contains(HoodieTableConfig.META_FIELDS_MODE.key()) + || rootMessage.contains(HoodieTableConfig.POPULATE_META_FIELDS.key()), + "Expected validation error to name one of the conflicting properties, got: " + rootMessage); + } + + @Test + void unknownTokenInModeIsRejected() { + Map options = baseOptions(); + options.put(HoodieTableConfig.POPULATE_META_FIELDS.key(), "false"); + options.put(HoodieTableConfig.META_FIELDS_MODE.key(), HoodieRecord.RECORD_KEY_METADATA_FIELD); + options.put(DataSourceWriteOptions.OPERATION().key(), DataSourceWriteOptions.BULK_INSERT_OPERATION_OPT_VAL()); + + Throwable thrown = assertThrows(Throwable.class, () -> + writeRows(Collections.singletonList(RowFactory.create("k1", "p1", "v1")), + simpleSchema(), options, basePath(), SaveMode.Overwrite)); + + String rootMessage = rootMessageOf(thrown); + assertTrue(rootMessage.contains(HoodieRecord.RECORD_KEY_METADATA_FIELD), + "Expected error to name the rejected token, got: " + rootMessage); + } + + @Test + void morWithNonEmptyModeIsRejected() { + Map options = baseOptions(); + options.put(DataSourceWriteOptions.TABLE_TYPE().key(), "MERGE_ON_READ"); + options.put(HoodieTableConfig.POPULATE_META_FIELDS.key(), "false"); + options.put(HoodieTableConfig.META_FIELDS_MODE.key(), COMMIT_TIME_TOKEN); + options.put(DataSourceWriteOptions.OPERATION().key(), DataSourceWriteOptions.BULK_INSERT_OPERATION_OPT_VAL()); + + Throwable thrown = assertThrows(Throwable.class, () -> + writeRows(Collections.singletonList(RowFactory.create("k1", "p1", "v1")), + simpleSchema(), options, basePath(), SaveMode.Overwrite)); + + String rootMessage = rootMessageOf(thrown); + assertTrue(rootMessage.contains("COPY_ON_WRITE") || rootMessage.contains("MoR") || rootMessage.contains("MERGE_ON_READ"), + "Expected MoR-restriction error, got: " + rootMessage); + } + + private static String rootMessageOf(Throwable thrown) { + Throwable root = thrown; + while (root.getCause() != null) { + root = root.getCause(); + } + return root.getMessage() == null ? "" : root.getMessage(); + } +} From 0b259186c19b7ae3d6dd51fd2dfb82ffbde4a1fd Mon Sep 17 00:00:00 2001 From: sivabalan Date: Mon, 6 Jul 2026 13:22:08 -0700 Subject: [PATCH 02/13] refactor(writer): fold populateMetaFields + mode into MetaFieldsMode enum MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses review feedback on #19205: - Fold (populateMetaFields boolean, comma-separated mode) into a single MetaFieldsMode enum (ALL / NONE / COMMIT_TIME_ONLY / FILE_NAME_ONLY / COMMIT_TIME_AND_FILE_NAME). The enum is the sole argument passed through every writer factory and writer constructor. populate.meta.fields stays on the config surface for backward compat and is auto-derived from the enum. - On-disk representation is now the enum name (e.g. "COMMIT_TIME_ONLY") instead of a comma-separated token list. Existing tables without the property continue to fall back to ALL or NONE based on the legacy hoodie.populate.meta.fields boolean. - Drop _hoodie_commit_seqno population from the selective write path — only _hoodie_commit_time is needed for incremental queries. (Review comments on HoodieRowCreateHandle:193, HoodieSparkParquetWriter:100, HoodieAvroParquetWriter:103.) - Extend the IncrementalRelation error message to state that hoodie.meta.fields.mode is physical-storage and cannot be flipped by changing write options — existing tables must be recreated to change it. - StreamSync.initializeEmptyTable() was missing the mode pass-through — wired via setMetaFieldsModeFromString. All other non-test initTable call sites either use fromProperties() (which now routes the mode through) or intentionally don't touch meta fields (CLI table create, MDT, sample writes). Test coverage additions: - TestMetaFieldsMode (hudi-spark): end-to-end assertions read parquet back and verify which meta columns are populated / null per mode. Covers row-writer path (bulk_insert with row.writer.enable=true, default) AND non-row-writer path (insert with row.writer.enable=false). Adds inline-clustering coverage for all 5 modes verifying that clustered files preserve the mode's column population semantics. - TestHoodieStreamerMetaFieldsMode (hudi-utilities): parameterized end-to-end test through the HoodieStreamer entrypoint covering all 5 modes plus MoR+selective rejection. --- .../apache/hudi/config/HoodieWriteConfig.java | 71 ++-- .../TestHoodieWriteConfigMetaFieldsMode.java | 64 ++-- .../storage/HoodieSparkFileWriterFactory.java | 8 +- .../io/storage/HoodieSparkParquetWriter.java | 65 ++-- .../io/storage/row/HoodieRowCreateHandle.java | 46 +-- .../hudi/common/model/MetaFieldsMode.java | 134 +++++++ .../hudi/common/table/HoodieTableConfig.java | 73 +--- .../common/table/HoodieTableMetaClient.java | 45 ++- .../hadoop/HoodieAvroFileWriterFactory.java | 8 +- .../hadoop/HoodieAvroParquetWriter.java | 69 ++-- .../table/TestHoodieMetaFieldsMode.java | 74 ++-- .../apache/hudi/HoodieSparkSqlWriter.scala | 4 +- .../org/apache/hudi/HoodieWriterUtils.scala | 8 +- .../apache/hudi/IncrementalRelationV1.scala | 5 +- .../apache/hudi/IncrementalRelationV2.scala | 5 +- .../hudi/functional/TestMetaFieldsMode.java | 326 ++++++++++++++---- .../hudi/utilities/streamer/StreamSync.java | 2 + .../TestHoodieStreamerMetaFieldsMode.java | 139 ++++++++ 18 files changed, 792 insertions(+), 354 deletions(-) create mode 100644 hudi-common/src/main/java/org/apache/hudi/common/model/MetaFieldsMode.java create mode 100644 hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieStreamerMetaFieldsMode.java diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java index b4dfa4d8bc88b..3365694f56367 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java @@ -47,10 +47,10 @@ import org.apache.hudi.common.model.HoodieFailedWritesCleaningPolicy; import org.apache.hudi.common.model.HoodieFileFormat; import org.apache.hudi.common.model.HoodiePreWriteCleanerPolicy; -import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieRecordMerger; import org.apache.hudi.common.model.HoodieRecordPayload; import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.common.model.MetaFieldsMode; import org.apache.hudi.common.model.WriteConcurrencyMode; import org.apache.hudi.common.model.WriteOperationType; import org.apache.hudi.common.table.HoodieTableConfig; @@ -1778,36 +1778,27 @@ public boolean populateMetaFields() { } /** - * @return the raw comma-separated value of {@link HoodieTableConfig#META_FIELDS_MODE}, or an - * empty string when unset. Callers that need parsed access should use {@link #getMetaFieldsMode()}. + * @return the {@link MetaFieldsMode} resolved from the write config. Older tables without the + * mode property fall back to {@link MetaFieldsMode#ALL} or {@link MetaFieldsMode#NONE} based on + * the legacy {@code hoodie.populate.meta.fields} boolean. */ - public String getMetaFieldsModeRaw() { - return getStringOrDefault(HoodieTableConfig.META_FIELDS_MODE); + public MetaFieldsMode getMetaFieldsMode() { + return MetaFieldsMode.fromConfig(populateMetaFields(), + getStringOrDefault(HoodieTableConfig.META_FIELDS_MODE)); } /** - * @return the parsed set of meta columns opted in via {@link HoodieTableConfig#META_FIELDS_MODE}. - * Throws when an unrecognized token is present. - */ - public java.util.Set getMetaFieldsMode() { - return HoodieTableConfig.parseMetaFieldsMode(getMetaFieldsModeRaw()); - } - - /** - * @return true when {@code _hoodie_commit_time} is physically populated on every row — either - * because all meta fields are populated, or because the mode set includes {@code _hoodie_commit_time}. + * @return true when {@code _hoodie_commit_time} is physically populated on every row. */ public boolean isCommitTimePopulated() { - return populateMetaFields() - || getMetaFieldsMode().contains(HoodieRecord.COMMIT_TIME_METADATA_FIELD); + return getMetaFieldsMode().isCommitTimePopulated(); } /** * @return true when {@code _hoodie_file_name} is physically populated on every row. */ public boolean isFileNamePopulated() { - return populateMetaFields() - || getMetaFieldsMode().contains(HoodieRecord.FILENAME_METADATA_FIELD); + return getMetaFieldsMode().isFileNamePopulated(); } /** @@ -3625,8 +3616,14 @@ public Builder withPopulateMetaFields(boolean populateMetaFields) { return this; } - public Builder withMetaFieldsMode(String metaFieldsMode) { - writeConfig.setValue(HoodieTableConfig.META_FIELDS_MODE, metaFieldsMode == null ? "" : metaFieldsMode); + public Builder withMetaFieldsMode(MetaFieldsMode metaFieldsMode) { + // ALL is the default (implicit) mode — persist the enum name only for selective modes to + // keep hoodie.properties clean for tables that don't opt in. + if (metaFieldsMode == null || metaFieldsMode == MetaFieldsMode.ALL) { + writeConfig.setValue(HoodieTableConfig.META_FIELDS_MODE, ""); + } else { + writeConfig.setValue(HoodieTableConfig.META_FIELDS_MODE, metaFieldsMode.name()); + } return this; } @@ -3924,28 +3921,30 @@ private void validate() { HoodieTTLConfig.STATS_MAX_PARALLELISM.key(), ttlStatsMaxParallelism)); // hoodie.meta.fields.mode is an additive opt-in on top of populate.meta.fields=false. Setting - // populate.meta.fields=true together with a non-empty mode list is ambiguous (the mode list - // has no effect when all meta fields are already populated) so reject it explicitly rather - // than silently ignore. Also reject unknown tokens up-front — parseMetaFieldsMode throws. + // populate.meta.fields=true together with a non-ALL mode is ambiguous (the mode has no effect + // when all meta fields are already populated) so reject it explicitly rather than silently + // ignore. MetaFieldsMode.fromConfig also throws on unrecognized on-disk values. + MetaFieldsMode metaFieldsMode = writeConfig.getMetaFieldsMode(); boolean populateMetaFields = writeConfig.populateMetaFields(); - java.util.Set metaFieldsMode = writeConfig.getMetaFieldsMode(); - checkArgument(!(populateMetaFields && !metaFieldsMode.isEmpty()), - String.format("%s must be empty when %s=true. Disable populate.meta.fields or clear the mode list.", + String rawMode = writeConfig.getStringOrDefault(HoodieTableConfig.META_FIELDS_MODE); + checkArgument(!(populateMetaFields && rawMode != null && !rawMode.isEmpty()), + String.format("%s must be empty when %s=true. Disable populate.meta.fields or clear the mode.", HoodieTableConfig.META_FIELDS_MODE.key(), HoodieTableConfig.POPULATE_META_FIELDS.key())); // Selective meta-field modes are CoW-only in this release. MoR log-write path does not yet - // respect the mode set, which would silently produce log records with null meta columns. - checkArgument(!(writeConfig.getTableType() == HoodieTableType.MERGE_ON_READ && !metaFieldsMode.isEmpty()), - String.format("%s is currently supported for COPY_ON_WRITE tables only. MoR support is a follow-up. " - + "For MoR either keep %s=true or leave the mode list empty.", - HoodieTableConfig.META_FIELDS_MODE.key(), + // respect the mode, which would silently produce log records with null meta columns. + boolean isSelective = metaFieldsMode != MetaFieldsMode.ALL && metaFieldsMode != MetaFieldsMode.NONE; + checkArgument(!(writeConfig.getTableType() == HoodieTableType.MERGE_ON_READ && isSelective), + String.format("%s=%s is currently supported for COPY_ON_WRITE tables only. MoR support is a follow-up. " + + "For MoR either keep %s=true or use NONE mode.", + HoodieTableConfig.META_FIELDS_MODE.key(), metaFieldsMode, HoodieTableConfig.POPULATE_META_FIELDS.key())); // Selective meta-field modes are wired only for the Spark writer path in this release. Flink // RowData / Java-client writers ignore the mode and would silently produce NONE-mode output. - checkArgument(!(engineType != EngineType.SPARK && !metaFieldsMode.isEmpty()), - String.format("%s is currently supported for the Spark writer only. Support for engine=%s is a follow-up. " - + "Either keep %s=true or leave the mode list empty.", - HoodieTableConfig.META_FIELDS_MODE.key(), engineType, + checkArgument(!(engineType != EngineType.SPARK && isSelective), + String.format("%s=%s is currently supported for the Spark writer only. Support for engine=%s is a follow-up. " + + "Either keep %s=true or use NONE mode.", + HoodieTableConfig.META_FIELDS_MODE.key(), metaFieldsMode, engineType, HoodieTableConfig.POPULATE_META_FIELDS.key())); } diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/config/TestHoodieWriteConfigMetaFieldsMode.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/config/TestHoodieWriteConfigMetaFieldsMode.java index d08fcbad4d924..c578489feb26f 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/config/TestHoodieWriteConfigMetaFieldsMode.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/config/TestHoodieWriteConfigMetaFieldsMode.java @@ -18,20 +18,21 @@ package org.apache.hudi.config; -import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.MetaFieldsMode; +import org.apache.hudi.common.table.HoodieTableConfig; import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; /** * Validates the writer-side accessors and validation guards for the meta-field-population modes - * on {@link HoodieWriteConfig}. Companion test for the {@link - * org.apache.hudi.common.table.HoodieTableConfig} accessors lives in {@code TestHoodieMetaFieldsMode}; - * this test covers the writer-builder surface and the cross-flag validation that runs at - * {@code build()} time. + * on {@link HoodieWriteConfig}. Companion test for the {@link HoodieTableConfig} accessors lives + * in {@code TestHoodieMetaFieldsMode}; this test covers the writer-builder surface and the + * cross-flag validation that runs at {@code build()} time. */ class TestHoodieWriteConfigMetaFieldsMode { @@ -43,8 +44,7 @@ private static HoodieWriteConfig.Builder baseBuilder() { void defaultsToAllMode() { HoodieWriteConfig cfg = baseBuilder().build(); assertTrue(cfg.populateMetaFields()); - assertTrue(cfg.getMetaFieldsMode().isEmpty(), - "mode list must be ignored when populate.meta.fields=true"); + assertEquals(MetaFieldsMode.ALL, cfg.getMetaFieldsMode()); assertTrue(cfg.isCommitTimePopulated()); assertTrue(cfg.isFileNamePopulated()); } @@ -53,20 +53,19 @@ void defaultsToAllMode() { void explicitNoneModeBuilds() { HoodieWriteConfig cfg = baseBuilder().withPopulateMetaFields(false).build(); assertFalse(cfg.populateMetaFields()); - assertTrue(cfg.getMetaFieldsMode().isEmpty()); + assertEquals(MetaFieldsMode.NONE, cfg.getMetaFieldsMode()); assertFalse(cfg.isCommitTimePopulated()); assertFalse(cfg.isFileNamePopulated()); } @Test - void commitTimeOnlyModeBuildsAndIsAdditiveOverNone() { + void commitTimeOnlyModeBuilds() { HoodieWriteConfig cfg = baseBuilder() .withPopulateMetaFields(false) - .withMetaFieldsMode(HoodieRecord.COMMIT_TIME_METADATA_FIELD) + .withMetaFieldsMode(MetaFieldsMode.COMMIT_TIME_ONLY) .build(); - assertFalse(cfg.populateMetaFields()); - assertTrue(cfg.isCommitTimePopulated(), - "incremental query semantics depend on _hoodie_commit_time being populated in this mode"); + assertEquals(MetaFieldsMode.COMMIT_TIME_ONLY, cfg.getMetaFieldsMode()); + assertTrue(cfg.isCommitTimePopulated()); assertFalse(cfg.isFileNamePopulated()); } @@ -74,9 +73,9 @@ void commitTimeOnlyModeBuildsAndIsAdditiveOverNone() { void fileNameOnlyModeBuilds() { HoodieWriteConfig cfg = baseBuilder() .withPopulateMetaFields(false) - .withMetaFieldsMode(HoodieRecord.FILENAME_METADATA_FIELD) + .withMetaFieldsMode(MetaFieldsMode.FILE_NAME_ONLY) .build(); - assertFalse(cfg.populateMetaFields()); + assertEquals(MetaFieldsMode.FILE_NAME_ONLY, cfg.getMetaFieldsMode()); assertFalse(cfg.isCommitTimePopulated()); assertTrue(cfg.isFileNamePopulated()); } @@ -85,20 +84,19 @@ void fileNameOnlyModeBuilds() { void commitTimeAndFileNameCombinationBuilds() { HoodieWriteConfig cfg = baseBuilder() .withPopulateMetaFields(false) - .withMetaFieldsMode(HoodieRecord.COMMIT_TIME_METADATA_FIELD + "," + HoodieRecord.FILENAME_METADATA_FIELD) + .withMetaFieldsMode(MetaFieldsMode.COMMIT_TIME_AND_FILE_NAME) .build(); - assertFalse(cfg.populateMetaFields()); + assertEquals(MetaFieldsMode.COMMIT_TIME_AND_FILE_NAME, cfg.getMetaFieldsMode()); assertTrue(cfg.isCommitTimePopulated()); assertTrue(cfg.isFileNamePopulated()); } @Test void rejectsIncompatibleCombination() { - // populate.meta.fields=true together with a non-empty mode is ambiguous (mode has no effect - // when all meta fields are already populated); reject loudly. + // populate.meta.fields=true together with a selective mode is ambiguous — reject. HoodieWriteConfig.Builder builder = baseBuilder() .withPopulateMetaFields(true) - .withMetaFieldsMode(HoodieRecord.COMMIT_TIME_METADATA_FIELD); + .withMetaFieldsMode(MetaFieldsMode.COMMIT_TIME_ONLY); IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, builder::build); assertTrue(ex.getMessage().contains("hoodie.meta.fields.mode"), "exception must name the mode property: " + ex.getMessage()); @@ -107,25 +105,25 @@ void rejectsIncompatibleCombination() { } @Test - void rejectsUnknownTokenInMode() { - HoodieWriteConfig.Builder builder = baseBuilder() - .withPopulateMetaFields(false) - .withMetaFieldsMode(HoodieRecord.RECORD_KEY_METADATA_FIELD); - IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, builder::build); - assertTrue(ex.getMessage().contains(HoodieRecord.RECORD_KEY_METADATA_FIELD), - "exception must name the rejected token: " + ex.getMessage()); - assertTrue(ex.getMessage().contains("populate.meta.fields"), - "exception must recommend populate.meta.fields=true for other columns: " + ex.getMessage()); + void allModeWithPopulateFalseIsAlsoRejectedByBuilder() { + // Explicitly setting ALL is a no-op — the builder normalizes it to empty. Passing ALL directly + // is fine; ensuring populateMetaFields agrees is the caller's responsibility (validate() runs + // the cross-check at build time). + HoodieWriteConfig cfg = baseBuilder() + .withPopulateMetaFields(true) + .withMetaFieldsMode(MetaFieldsMode.ALL) + .build(); + assertEquals(MetaFieldsMode.ALL, cfg.getMetaFieldsMode()); } @Test - void noneModeWithEmptyModeExplicitIsStillNone() { + void noneModeWithExplicitBuildIsStillNone() { HoodieWriteConfig cfg = baseBuilder() .withPopulateMetaFields(false) - .withMetaFieldsMode("") + .withMetaFieldsMode(MetaFieldsMode.NONE) .build(); - assertFalse(cfg.populateMetaFields()); - assertTrue(cfg.getMetaFieldsMode().isEmpty()); + // NONE is normalized to empty on-disk (implicit from populate=false). + assertEquals(MetaFieldsMode.NONE, cfg.getMetaFieldsMode()); assertFalse(cfg.isCommitTimePopulated()); assertFalse(cfg.isFileNamePopulated()); } diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkFileWriterFactory.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkFileWriterFactory.java index eb4458e134e2d..762dc027643da 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkFileWriterFactory.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkFileWriterFactory.java @@ -57,9 +57,9 @@ protected HoodieFileWriter newParquetFileWriter( String instantTime, StoragePath path, HoodieConfig config, HoodieSchema schema, TaskContextSupplier taskContextSupplier) throws IOException { boolean populateMetaFields = config.getBooleanOrDefault(HoodieTableConfig.POPULATE_META_FIELDS); - java.util.Set metaFieldsMode = populateMetaFields - ? java.util.Collections.emptySet() - : HoodieTableConfig.parseMetaFieldsMode(config.getStringOrDefault(HoodieTableConfig.META_FIELDS_MODE)); + org.apache.hudi.common.model.MetaFieldsMode metaFieldsMode = + org.apache.hudi.common.model.MetaFieldsMode.fromConfig(populateMetaFields, + config.getStringOrDefault(HoodieTableConfig.META_FIELDS_MODE)); Pair injectedConfigs = HoodieParquetConfigInjector.applyConfigInjector(path, storage.getConf(), config); StorageConfiguration storageConfiguration = injectedConfigs.getLeft(); @@ -83,7 +83,7 @@ protected HoodieFileWriter newParquetFileWriter( hoodieConfig.getBooleanOrDefault(HoodieStorageConfig.PARQUET_DICTIONARY_ENABLED)); parquetConfig.getHadoopConf().addResource(writeSupport.getHadoopConf()); - return new HoodieSparkParquetWriter(path, parquetConfig, instantTime, taskContextSupplier, populateMetaFields, metaFieldsMode); + return new HoodieSparkParquetWriter(path, parquetConfig, instantTime, taskContextSupplier, metaFieldsMode); } protected HoodieFileWriter newParquetFileWriter(OutputStream outputStream, HoodieConfig config, diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkParquetWriter.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkParquetWriter.java index 3c2c35bf11fd1..02e637a52fee9 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkParquetWriter.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkParquetWriter.java @@ -21,6 +21,7 @@ import org.apache.hudi.common.engine.TaskContextSupplier; import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.MetaFieldsMode; import org.apache.hudi.io.hadoop.HoodieBaseParquetWriter; import org.apache.hudi.io.storage.row.HoodieRowParquetConfig; import org.apache.hudi.io.storage.row.HoodieRowParquetWriteSupport; @@ -30,9 +31,7 @@ import org.apache.spark.unsafe.types.UTF8String; import java.io.IOException; -import java.util.Collections; import java.util.Map; -import java.util.Set; import java.util.function.Function; import static org.apache.hudi.common.model.HoodieRecord.HoodieMetadataField.COMMIT_SEQNO_METADATA_FIELD; @@ -46,14 +45,7 @@ public class HoodieSparkParquetWriter extends HoodieBaseParquetWriter metaFieldsMode) throws IOException { + MetaFieldsMode metaFieldsMode) throws IOException { super(file, parquetConfig); this.writeSupport = parquetConfig.getWriteSupport(); this.fileName = UTF8String.fromString(file.getName()); this.instantTime = UTF8String.fromString(instantTime); - this.populateMetaFields = populateMetaFields; - Set mode = metaFieldsMode == null ? Collections.emptySet() : metaFieldsMode; - this.populateCommitTime = !populateMetaFields && mode.contains(HoodieRecord.COMMIT_TIME_METADATA_FIELD); - this.populateFileName = !populateMetaFields && mode.contains(HoodieRecord.FILENAME_METADATA_FIELD); + this.metaFieldsMode = metaFieldsMode == null ? MetaFieldsMode.NONE : metaFieldsMode; this.seqIdGenerator = recordIndex -> { Integer partitionId = taskContextSupplier.getPartitionIdSupplier().get(); return HoodieRecord.generateSequenceId(instantTime, partitionId, recordIndex); @@ -89,31 +78,35 @@ public HoodieSparkParquetWriter(StoragePath file, @Override public void writeRowWithMetadata(HoodieKey key, InternalRow row) throws IOException { - if (populateMetaFields) { - UTF8String recordKey = UTF8String.fromString(key.getRecordKey()); - updateRecordMetadata(row, recordKey, key.getPartitionPath(), getWrittenRecordCount()); - - super.write(row); - writeSupport.add(recordKey); - } else if (populateCommitTime || populateFileName) { - if (populateCommitTime) { - row.update(COMMIT_TIME_METADATA_FIELD.ordinal(), instantTime); - row.update(COMMIT_SEQNO_METADATA_FIELD.ordinal(), - UTF8String.fromString(seqIdGenerator.apply(getWrittenRecordCount()))); - } - if (populateFileName) { - row.update(FILENAME_METADATA_FIELD.ordinal(), fileName); - } - super.write(row); - } else { - super.write(row); + switch (metaFieldsMode) { + case ALL: + UTF8String recordKey = UTF8String.fromString(key.getRecordKey()); + updateRecordMetadata(row, recordKey, key.getPartitionPath(), getWrittenRecordCount()); + super.write(row); + writeSupport.add(recordKey); + break; + case NONE: + super.write(row); + break; + default: + // Selective mode — populate only the opted-in columns. Record-key column stays null, so + // we do NOT register the record key with the write support (bloom filter / RLI hooks are + // meaningless without the record-key column). + if (metaFieldsMode.isCommitTimePopulated()) { + row.update(COMMIT_TIME_METADATA_FIELD.ordinal(), instantTime); + } + if (metaFieldsMode.isFileNamePopulated()) { + row.update(FILENAME_METADATA_FIELD.ordinal(), fileName); + } + super.write(row); + break; } } @Override public void writeRow(String recordKey, InternalRow row) throws IOException { super.write(row); - if (populateMetaFields) { + if (metaFieldsMode == MetaFieldsMode.ALL) { writeSupport.add(UTF8String.fromString(recordKey)); } } diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowCreateHandle.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowCreateHandle.java index db107cf2627d0..50e4e9d514539 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowCreateHandle.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowCreateHandle.java @@ -28,6 +28,7 @@ import org.apache.hudi.common.model.HoodieRecordLocation; import org.apache.hudi.common.model.HoodieWriteStat; import org.apache.hudi.common.model.IOType; +import org.apache.hudi.common.model.MetaFieldsMode; import org.apache.hudi.common.util.HoodieTimer; import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.collection.Pair; @@ -66,12 +67,7 @@ public class HoodieRowCreateHandle implements Serializable { private final StoragePath path; private final String fileId; - private final boolean populateMetaFields; - // Selective meta-field population when populateMetaFields is false. commit_time enables - // incremental queries; file_name enables file-level pruning / investigation lookups. Other meta - // columns stay null. Record-key is never populated in these modes. - private final boolean populateCommitTime; - private final boolean populateFileName; + private final MetaFieldsMode metaFieldsMode; private final UTF8String fileName; private final UTF8String commitTime; @@ -123,12 +119,7 @@ public HoodieRowCreateHandle(HoodieTable table, table.getBaseFileExtension()); this.path = makeNewPath(storage, partitionPath, fileName, writeConfig); - this.populateMetaFields = writeConfig.populateMetaFields(); - java.util.Set metaFieldsMode = this.populateMetaFields - ? java.util.Collections.emptySet() - : writeConfig.getMetaFieldsMode(); - this.populateCommitTime = !this.populateMetaFields && metaFieldsMode.contains(HoodieRecord.COMMIT_TIME_METADATA_FIELD); - this.populateFileName = !this.populateMetaFields && metaFieldsMode.contains(HoodieRecord.FILENAME_METADATA_FIELD); + this.metaFieldsMode = writeConfig.getMetaFieldsMode(); this.fileName = UTF8String.fromString(path.getName()); this.commitTime = UTF8String.fromString(instantTime); this.seqIdGenerator = (id) -> HoodieRecord.generateSequenceId(instantTime, taskPartitionId, id); @@ -168,33 +159,34 @@ public HoodieRowCreateHandle(HoodieTable table, * @throws IOException */ public void write(InternalRow row) throws IOException { - if (populateMetaFields) { - writeRow(row); - } else if (populateCommitTime || populateFileName) { - writeRowSelectiveMetaFields(row); - } else { - writeRowNoMetaFields(row); + switch (metaFieldsMode) { + case ALL: + writeRow(row); + break; + case NONE: + writeRowNoMetaFields(row); + break; + default: + writeRowSelectiveMetaFields(row); + break; } } /** * Selective meta-field write path: populate only the meta columns opted in via - * {@code hoodie.meta.fields.mode} — {@code _hoodie_commit_time} (with derived seq id) and/or - * {@code _hoodie_file_name}. The other meta columns stay null on disk. Record key is never - * populated in this path, so the record key is not registered with the write support (bloom - * filter / RLI hooks are meaningless without the record-key column). + * {@code hoodie.meta.fields.mode} — {@code _hoodie_commit_time} and/or {@code _hoodie_file_name}. + * The other meta columns stay null on disk. Record key is never populated in this path, so the + * record key is not registered with the write support (bloom filter / RLI hooks are meaningless + * without the record-key column). */ private void writeRowSelectiveMetaFields(InternalRow row) { try { UTF8String[] metaFields = new UTF8String[5]; - if (populateCommitTime) { + if (metaFieldsMode.isCommitTimePopulated()) { metaFields[HoodieRecord.COMMIT_TIME_METADATA_FIELD_ORD] = shouldPreserveHoodieMetadata ? row.getUTF8String(HoodieRecord.COMMIT_TIME_METADATA_FIELD_ORD) : commitTime; - metaFields[HoodieRecord.COMMIT_SEQNO_METADATA_FIELD_ORD] = shouldPreserveHoodieMetadata - ? row.getUTF8String(HoodieRecord.COMMIT_SEQNO_METADATA_FIELD_ORD) - : UTF8String.fromString(seqIdGenerator.apply(GLOBAL_SEQ_NO.getAndIncrement())); } - if (populateFileName) { + if (metaFieldsMode.isFileNamePopulated()) { metaFields[HoodieRecord.FILENAME_META_FIELD_ORD] = shouldPreserveHoodieMetadata ? row.getUTF8String(HoodieRecord.FILENAME_META_FIELD_ORD) : fileName; } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/MetaFieldsMode.java b/hudi-common/src/main/java/org/apache/hudi/common/model/MetaFieldsMode.java new file mode 100644 index 0000000000000..6766a9edf6836 --- /dev/null +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/MetaFieldsMode.java @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.common.model; + +import org.apache.hudi.common.util.StringUtils; + +/** + * Which of Hudi's meta columns are physically populated on disk. + * + *

Selective modes exist so that tables that opt out of the default {@code populate.meta.fields=true} + * can still keep the two columns that matter for downstream operations without paying for the other + * three: + * + *

    + *
  • {@code _hoodie_commit_time} — required for incremental queries.
    • + *
  • {@code _hoodie_file_name} — useful for file-level pruning / investigation lookups.
    • + *
+ * + *

The remaining three meta columns ({@code _hoodie_commit_seqno}, {@code _hoodie_record_key}, + * {@code _hoodie_partition_path}) are all-or-nothing — either populate every meta column ({@link #ALL}) + * or none of them beyond the two selectable ones. If you need any of the remaining columns, set + * {@code hoodie.populate.meta.fields=true}. + * + *

Mapping to the legacy {@code hoodie.populate.meta.fields} boolean: + * + *

    + *
  • {@link #ALL} corresponds to {@code populate.meta.fields=true} — today's default.
    • + *
  • Every other value corresponds to {@code populate.meta.fields=false} plus a selective opt-in.
    • + *
+ * + *

On-disk representation: the enum {@link #name()} is persisted in {@code hoodie.properties} + * under the property {@code hoodie.meta.fields.mode}. For backward compatibility, older tables that + * predate this property fall back to {@link #ALL} or {@link #NONE} based on the legacy boolean. + */ +public enum MetaFieldsMode { + /** + * All five Hudi meta columns are populated — today's default. + */ + ALL(true, true), + + /** + * No Hudi meta columns are populated. Incremental queries are unsupported. File-level pruning + * that depends on {@code _hoodie_file_name} is unsupported. + */ + NONE(false, false), + + /** + * Only {@code _hoodie_commit_time} is populated. Incremental queries remain functional; other + * meta columns stay null on disk. + */ + COMMIT_TIME_ONLY(true, false), + + /** + * Only {@code _hoodie_file_name} is populated. Useful for file-level lookups and debugging; + * incremental queries are unsupported. + */ + FILE_NAME_ONLY(false, true), + + /** + * Both {@code _hoodie_commit_time} and {@code _hoodie_file_name} are populated. + */ + COMMIT_TIME_AND_FILE_NAME(true, true); + + private final boolean commitTimePopulated; + private final boolean fileNamePopulated; + + MetaFieldsMode(boolean commitTimePopulated, boolean fileNamePopulated) { + this.commitTimePopulated = commitTimePopulated; + this.fileNamePopulated = fileNamePopulated; + } + + public boolean isCommitTimePopulated() { + return commitTimePopulated; + } + + public boolean isFileNamePopulated() { + return fileNamePopulated; + } + + /** + * @return true when all five meta columns are populated (i.e. this is {@link #ALL}). Selective + * modes never populate {@code _hoodie_record_key}, {@code _hoodie_partition_path}, or + * {@code _hoodie_commit_seqno}. + */ + public boolean isRecordKeyPopulated() { + return this == ALL; + } + + /** + * Auto-derive a mode from the legacy {@code hoodie.populate.meta.fields} boolean and the raw + * {@code hoodie.meta.fields.mode} property value. Precedence: + * + *

    + *
  • {@code populateMetaFields=true} → {@link #ALL} (the mode property is ignored).
    • + *
  • {@code populateMetaFields=false} + null/empty mode → {@link #NONE}.
    • + *
  • {@code populateMetaFields=false} + non-empty mode → the parsed enum value.
    • + *
+ * + *

Throws {@link IllegalArgumentException} when the raw mode value does not match any enum + * value. This includes the pre-enum comma-separated format — callers that upgrade an old table + * must migrate the value through the hudi-cli. + */ + public static MetaFieldsMode fromConfig(boolean populateMetaFields, String rawMode) { + if (populateMetaFields) { + return ALL; + } + if (StringUtils.isNullOrEmpty(rawMode)) { + return NONE; + } + try { + return MetaFieldsMode.valueOf(rawMode.trim()); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException(String.format( + "Unsupported value '%s' for hoodie.meta.fields.mode. Allowed values: %s, %s, %s, %s.", + rawMode, COMMIT_TIME_ONLY, FILE_NAME_ONLY, COMMIT_TIME_AND_FILE_NAME, NONE), e); + } + } +} diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java index eeb1f671f7a60..cef932e49ba3d 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java @@ -39,6 +39,7 @@ import org.apache.hudi.common.model.HoodieRecordPayload; import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.model.HoodieTimelineTimeZone; +import org.apache.hudi.common.model.MetaFieldsMode; import org.apache.hudi.common.model.OverwriteNonDefaultsWithLatestAvroPayload; import org.apache.hudi.common.model.OverwriteWithLatestAvroPayload; import org.apache.hudi.common.model.PartialUpdateAvroPayload; @@ -336,11 +337,11 @@ public static final String getDefaultPayloadClassName() { public static final ConfigProperty META_FIELDS_MODE = ConfigProperty .key("hoodie.meta.fields.mode") .defaultValue("") - .withDocumentation("Comma-separated list of meta columns to populate when hoodie.populate.meta.fields=false. " - + "Allowed values: _hoodie_commit_time, _hoodie_file_name. Any other value is rejected — to populate the " - + "remaining meta columns, set hoodie.populate.meta.fields=true. Ignored when hoodie.populate.meta.fields=true " - + "(all meta fields are populated in that case). Set only at table creation, via the hudi-cli, or during " - + "table upgrade — the property is immutable at runtime."); + .withDocumentation("Which Hudi meta columns are physically populated on disk. Allowed values are " + + "COMMIT_TIME_ONLY, FILE_NAME_ONLY, COMMIT_TIME_AND_FILE_NAME (or NONE / ALL, though those are " + + "derived automatically from hoodie.populate.meta.fields when unset). Set only at table creation, " + + "via the hudi-cli, or during table upgrade — the property is immutable at runtime because it is a " + + "physical-storage decision baked into files at write time."); public static final ConfigProperty KEY_GENERATOR_CLASS_NAME = ConfigProperty .key("hoodie.table.keygenerator.class") @@ -1245,73 +1246,35 @@ public boolean populateMetaFields() { } /** - * @return the set of meta columns explicitly opted in via {@link #META_FIELDS_MODE}. Empty when - * the mode is unset or when {@link #POPULATE_META_FIELDS} is {@code true} (in which case all meta - * columns are populated and the mode list is ignored). Throws when an unrecognized token is - * present — only {@code _hoodie_commit_time} and {@code _hoodie_file_name} are allowed here; to - * populate other meta columns, set {@code hoodie.populate.meta.fields=true}. + * @return the {@link MetaFieldsMode} resolved from the on-disk properties. Older tables without + * the mode property fall back to {@link MetaFieldsMode#ALL} or {@link MetaFieldsMode#NONE} based + * on the legacy {@link #POPULATE_META_FIELDS} boolean. */ - public Set getMetaFieldsMode() { - return parseMetaFieldsMode(getStringOrDefault(META_FIELDS_MODE)); + public MetaFieldsMode getMetaFieldsMode() { + return MetaFieldsMode.fromConfig(populateMetaFields(), getStringOrDefault(META_FIELDS_MODE)); } /** - * Package-visible helper so writer-side validation and CLI can parse without going through a - * {@link HoodieTableConfig} instance. - */ - public static Set parseMetaFieldsMode(String raw) { - if (raw == null || raw.trim().isEmpty()) { - return Collections.emptySet(); - } - Set result = new HashSet<>(); - for (String token : raw.split(",")) { - String field = token.trim(); - if (field.isEmpty()) { - continue; - } - if (!ALLOWED_META_FIELDS_MODE_VALUES.contains(field)) { - throw new IllegalArgumentException(String.format( - "Unsupported value '%s' for %s. Allowed values are %s. To populate other meta columns, set %s=true.", - field, META_FIELDS_MODE.key(), ALLOWED_META_FIELDS_MODE_VALUES, POPULATE_META_FIELDS.key())); - } - result.add(field); - } - return Collections.unmodifiableSet(result); - } - - /** - * Allowed tokens for {@link #META_FIELDS_MODE}. Any other meta-field name is rejected at parse - * time to preserve the "either all meta fields, none, or the two selectable ones" invariant. - */ - public static final Set ALLOWED_META_FIELDS_MODE_VALUES = Collections.unmodifiableSet( - new HashSet<>(Arrays.asList( - HoodieRecord.COMMIT_TIME_METADATA_FIELD, - HoodieRecord.FILENAME_METADATA_FIELD))); - - /** - * @return true when the {@code _hoodie_commit_time} meta column is physically populated on disk — - * i.e., either all meta fields are populated, or the mode set includes {@code _hoodie_commit_time}. + * @return true when the {@code _hoodie_commit_time} meta column is physically populated on disk. */ public boolean isCommitTimePopulated() { - return populateMetaFields() || getMetaFieldsMode().contains(HoodieRecord.COMMIT_TIME_METADATA_FIELD); + return getMetaFieldsMode().isCommitTimePopulated(); } /** - * @return true when the {@code _hoodie_file_name} meta column is physically populated on disk — - * i.e., either all meta fields are populated, or the mode set includes {@code _hoodie_file_name}. + * @return true when the {@code _hoodie_file_name} meta column is physically populated on disk. */ public boolean isFileNamePopulated() { - return populateMetaFields() || getMetaFieldsMode().contains(HoodieRecord.FILENAME_METADATA_FIELD); + return getMetaFieldsMode().isFileNamePopulated(); } /** * @return true when the {@code _hoodie_record_key} meta column is physically populated on disk. - * The selective mode never populates the record-key column, so this exactly mirrors - * {@link #populateMetaFields()} — exposed separately so callers that specifically care about the - * record-key column do not need to be updated again if the mode set is extended. + * Only {@link MetaFieldsMode#ALL} populates the record-key column; every selective mode leaves + * it null. */ public boolean isRecordKeyPopulated() { - return populateMetaFields(); + return getMetaFieldsMode().isRecordKeyPopulated(); } /** diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java index 0778b549c25a6..71cd0e6b1faa0 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java @@ -38,6 +38,7 @@ import org.apache.hudi.common.model.HoodieRecordPayload; import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.model.HoodieTimelineTimeZone; +import org.apache.hudi.common.model.MetaFieldsMode; import org.apache.hudi.common.table.timeline.CommitMetadataSerDe; import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; import org.apache.hudi.common.table.timeline.HoodieArchivedTimeline; @@ -1013,7 +1014,7 @@ public static class TableBuilder { private String bootstrapBasePath; private Boolean bootstrapIndexEnable; private Boolean populateMetaFields; - private String metaFieldsMode; + private MetaFieldsMode metaFieldsMode; private String keyGeneratorClassProp; private String partitionValueExtractorClass; private String keyGeneratorType; @@ -1171,11 +1172,25 @@ public TableBuilder setPopulateMetaFields(boolean populateMetaFields) { return this; } - public TableBuilder setMetaFieldsMode(String metaFieldsMode) { + public TableBuilder setMetaFieldsMode(MetaFieldsMode metaFieldsMode) { this.metaFieldsMode = metaFieldsMode; return this; } + /** + * Convenience overload that accepts the raw on-disk string (e.g. from properties files). + * Empty or null values leave the mode unset — the caller-provided populateMetaFields boolean + * determines whether the table is ALL or NONE. + */ + public TableBuilder setMetaFieldsModeFromString(String rawMode) { + if (rawMode == null || rawMode.trim().isEmpty()) { + this.metaFieldsMode = null; + return this; + } + this.metaFieldsMode = MetaFieldsMode.valueOf(rawMode.trim()); + return this; + } + public TableBuilder setKeyGeneratorClassProp(String keyGeneratorClassProp) { this.keyGeneratorClassProp = keyGeneratorClassProp; return this; @@ -1391,7 +1406,7 @@ public TableBuilder fromProperties(Properties properties) { setPopulateMetaFields(hoodieConfig.getBoolean(HoodieTableConfig.POPULATE_META_FIELDS)); } if (hoodieConfig.contains(HoodieTableConfig.META_FIELDS_MODE)) { - setMetaFieldsMode(hoodieConfig.getString(HoodieTableConfig.META_FIELDS_MODE)); + setMetaFieldsModeFromString(hoodieConfig.getString(HoodieTableConfig.META_FIELDS_MODE)); } if (hoodieConfig.contains(HoodieTableConfig.KEY_GENERATOR_CLASS_NAME)) { setKeyGeneratorClassProp(hoodieConfig.getString(HoodieTableConfig.KEY_GENERATOR_CLASS_NAME)); @@ -1531,11 +1546,27 @@ public Properties build() { if (null != populateMetaFields) { tableConfig.setValue(HoodieTableConfig.POPULATE_META_FIELDS, Boolean.toString(populateMetaFields)); } + // Persist the mode in one place. Rules: + // - Explicit selective mode wins → validate compatibility with populateMetaFields and write. + // - populateMetaFields=false + no explicit mode → NONE (leave property empty for backward compat). + // - populateMetaFields=true + no explicit mode → ALL (implicit; leave property empty). if (null != metaFieldsMode) { - // Validate at table-init time; parseMetaFieldsMode throws on unknown tokens so the bad - // property never makes it onto disk. - HoodieTableConfig.parseMetaFieldsMode(metaFieldsMode); - tableConfig.setValue(HoodieTableConfig.META_FIELDS_MODE, metaFieldsMode); + if (Boolean.TRUE.equals(populateMetaFields) && metaFieldsMode != MetaFieldsMode.ALL) { + throw new IllegalArgumentException(String.format( + "%s=%s is incompatible with %s=true. Set populate.meta.fields=false or use MetaFieldsMode.ALL.", + HoodieTableConfig.META_FIELDS_MODE.key(), metaFieldsMode, + HoodieTableConfig.POPULATE_META_FIELDS.key())); + } + if (Boolean.FALSE.equals(populateMetaFields) && metaFieldsMode == MetaFieldsMode.ALL) { + throw new IllegalArgumentException(String.format( + "%s=ALL is incompatible with %s=false. Set populate.meta.fields=true or pick a selective mode.", + HoodieTableConfig.META_FIELDS_MODE.key(), + HoodieTableConfig.POPULATE_META_FIELDS.key())); + } + // Persist only the selective modes; ALL/NONE are implicit from populate.meta.fields. + if (metaFieldsMode != MetaFieldsMode.ALL && metaFieldsMode != MetaFieldsMode.NONE) { + tableConfig.setValue(HoodieTableConfig.META_FIELDS_MODE, metaFieldsMode.name()); + } } if (null != keyGeneratorClassProp) { KeyGeneratorType type = KeyGeneratorType.fromClassName(keyGeneratorClassProp); diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroFileWriterFactory.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroFileWriterFactory.java index e70d84af58c6a..7c4b85aa58e87 100644 --- a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroFileWriterFactory.java +++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroFileWriterFactory.java @@ -70,9 +70,9 @@ protected HoodieFileWriter newParquetFileWriter( String instantTime, StoragePath path, HoodieConfig config, HoodieSchema schema, TaskContextSupplier taskContextSupplier) throws IOException { boolean populateMetaFields = config.getBooleanOrDefault(HoodieTableConfig.POPULATE_META_FIELDS); - java.util.Set metaFieldsMode = populateMetaFields - ? java.util.Collections.emptySet() - : HoodieTableConfig.parseMetaFieldsMode(config.getStringOrDefault(HoodieTableConfig.META_FIELDS_MODE)); + org.apache.hudi.common.model.MetaFieldsMode metaFieldsMode = + org.apache.hudi.common.model.MetaFieldsMode.fromConfig(populateMetaFields, + config.getStringOrDefault(HoodieTableConfig.META_FIELDS_MODE)); Pair injectedConfigs = HoodieParquetConfigInjector.applyConfigInjector(path, storage.getConf(), config); StorageConfiguration storageConfiguration = injectedConfigs.getLeft(); @@ -92,7 +92,7 @@ protected HoodieFileWriter newParquetFileWriter( hoodieConfig.getLongOrDefault(HoodieStorageConfig.PARQUET_MAX_FILE_SIZE), storageConfiguration, hoodieConfig.getDoubleOrDefault(HoodieStorageConfig.PARQUET_COMPRESSION_RATIO_FRACTION), hoodieConfig.getBooleanOrDefault(HoodieStorageConfig.PARQUET_DICTIONARY_ENABLED)); - return new HoodieAvroParquetWriter(path, parquetConfig, instantTime, taskContextSupplier, populateMetaFields, metaFieldsMode); + return new HoodieAvroParquetWriter(path, parquetConfig, instantTime, taskContextSupplier, metaFieldsMode); } protected HoodieFileWriter newParquetFileWriter( diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroParquetWriter.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroParquetWriter.java index 6564c48558bd2..c2357a7165374 100644 --- a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroParquetWriter.java +++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroParquetWriter.java @@ -19,12 +19,12 @@ package org.apache.hudi.io.storage.hadoop; -import org.apache.hudi.avro.HoodieAvroUtils; import org.apache.hudi.avro.HoodieAvroWriteSupport; import org.apache.hudi.common.config.HoodieParquetConfig; import org.apache.hudi.common.engine.TaskContextSupplier; import org.apache.hudi.common.model.HoodieKey; import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.MetaFieldsMode; import org.apache.hudi.core.io.storage.HoodieAvroFileWriter; import org.apache.hudi.io.hadoop.HoodieBaseParquetWriter; import org.apache.hudi.storage.StoragePath; @@ -35,9 +35,7 @@ import javax.annotation.concurrent.NotThreadSafe; import java.io.IOException; -import java.util.Collections; import java.util.Map; -import java.util.Set; /** * HoodieParquetWriter extends the ParquetWriter to help limit the size of underlying file. Provides a way to check if @@ -53,12 +51,7 @@ public class HoodieAvroParquetWriter private final String fileName; private final String instantTime; private final TaskContextSupplier taskContextSupplier; - private final boolean populateMetaFields; - // Meta fields to populate when populateMetaFields is false. Selectively enables - // _hoodie_commit_time / _hoodie_file_name so incremental queries and file-level lookups keep - // working on otherwise-minimal-meta-field tables. Ignored when populateMetaFields is true. - private final boolean populateCommitTime; - private final boolean populateFileName; + private final MetaFieldsMode metaFieldsMode; private final HoodieAvroWriteSupport writeSupport; @SuppressWarnings({"unchecked", "rawtypes"}) @@ -67,7 +60,8 @@ public HoodieAvroParquetWriter(StoragePath file, String instantTime, TaskContextSupplier taskContextSupplier, boolean populateMetaFields) throws IOException { - this(file, parquetConfig, instantTime, taskContextSupplier, populateMetaFields, Collections.emptySet()); + this(file, parquetConfig, instantTime, taskContextSupplier, + populateMetaFields ? MetaFieldsMode.ALL : MetaFieldsMode.NONE); } @SuppressWarnings({"unchecked", "rawtypes"}) @@ -75,49 +69,48 @@ public HoodieAvroParquetWriter(StoragePath file, HoodieParquetConfig parquetConfig, String instantTime, TaskContextSupplier taskContextSupplier, - boolean populateMetaFields, - Set metaFieldsMode) throws IOException { + MetaFieldsMode metaFieldsMode) throws IOException { super(file, (HoodieParquetConfig) parquetConfig); this.fileName = file.getName(); this.writeSupport = parquetConfig.getWriteSupport(); this.instantTime = instantTime; this.taskContextSupplier = taskContextSupplier; - this.populateMetaFields = populateMetaFields; - Set mode = metaFieldsMode == null ? Collections.emptySet() : metaFieldsMode; - this.populateCommitTime = !populateMetaFields && mode.contains(HoodieRecord.COMMIT_TIME_METADATA_FIELD); - this.populateFileName = !populateMetaFields && mode.contains(HoodieRecord.FILENAME_METADATA_FIELD); + this.metaFieldsMode = metaFieldsMode == null ? MetaFieldsMode.NONE : metaFieldsMode; } @Override public void writeAvroWithMetadata(HoodieKey key, IndexedRecord avroRecord) throws IOException { - if (populateMetaFields) { - prepRecordWithMetadata(key, avroRecord, instantTime, - taskContextSupplier.getPartitionIdSupplier().get(), getWrittenRecordCount(), fileName); - super.write(avroRecord); - writeSupport.add(key.getRecordKey()); - } else if (populateCommitTime || populateFileName) { - // Selective meta-field population. The other meta columns stay null on disk, which Parquet - // stores as definition-level flags (zero data bytes). Bloom filter / record-key index - // population is intentionally skipped — that requires the record-key column. - GenericRecord genericRecord = (GenericRecord) avroRecord; - if (populateCommitTime) { - String seqId = HoodieRecord.generateSequenceId(instantTime, - taskContextSupplier.getPartitionIdSupplier().get(), getWrittenRecordCount()); - HoodieAvroUtils.addCommitMetadataToRecord(genericRecord, instantTime, seqId); - } - if (populateFileName) { - genericRecord.put(HoodieRecord.FILENAME_METADATA_FIELD, fileName); - } - super.write(avroRecord); - } else { - super.write(avroRecord); + switch (metaFieldsMode) { + case ALL: + prepRecordWithMetadata(key, avroRecord, instantTime, + taskContextSupplier.getPartitionIdSupplier().get(), getWrittenRecordCount(), fileName); + super.write(avroRecord); + writeSupport.add(key.getRecordKey()); + break; + case NONE: + super.write(avroRecord); + break; + default: + // Selective mode — populate only the opted-in columns. The other meta columns stay null, + // which Parquet stores as definition-level flags (zero data bytes). Bloom filter / + // record-key index population is intentionally skipped — that requires the record-key + // column, which is never populated in selective modes. + GenericRecord genericRecord = (GenericRecord) avroRecord; + if (metaFieldsMode.isCommitTimePopulated()) { + genericRecord.put(HoodieRecord.COMMIT_TIME_METADATA_FIELD, instantTime); + } + if (metaFieldsMode.isFileNamePopulated()) { + genericRecord.put(HoodieRecord.FILENAME_METADATA_FIELD, fileName); + } + super.write(avroRecord); + break; } } @Override public void writeAvro(String key, IndexedRecord object) throws IOException { super.write(object); - if (populateMetaFields) { + if (metaFieldsMode == MetaFieldsMode.ALL) { writeSupport.add(key); } } diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestHoodieMetaFieldsMode.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestHoodieMetaFieldsMode.java index f343786dfbdf4..b11cedf0a57ed 100644 --- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestHoodieMetaFieldsMode.java +++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestHoodieMetaFieldsMode.java @@ -18,29 +18,20 @@ package org.apache.hudi.common.table; -import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.MetaFieldsMode; import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; /** * Tests the meta-field-population modes exposed by {@link HoodieTableConfig} via the - * {@code hoodie.meta.fields.mode} property: - * - *

    - *
  • ALL — {@code populate.meta.fields=true} (default). All five meta columns populated.
    • - *
  • NONE — {@code populate.meta.fields=false} and mode empty. No meta columns populated.
    • - *
  • COMMIT_TIME_ONLY — {@code populate.meta.fields=false}, - * {@code meta.fields.mode=_hoodie_commit_time}. Only commit-time populated.
    • - *
  • FILE_NAME_ONLY — same but with {@code _hoodie_file_name}.
    • - *
  • COMMIT_TIME_AND_FILE_NAME — both tokens in the mode list.
    • - *
- * - *

Tokens other than the two allowed ones are rejected up-front by the parser. This test - * exercises {@link HoodieTableConfig} accessors directly without touching the storage layer. + * {@code hoodie.meta.fields.mode} property. The mode is materialized as a {@link MetaFieldsMode} + * enum resolved from both the legacy {@code hoodie.populate.meta.fields} boolean and the on-disk + * {@code hoodie.meta.fields.mode} property. */ class TestHoodieMetaFieldsMode { @@ -59,17 +50,17 @@ private static HoodieTableConfig configOf(Boolean populate, String mode) { void defaultsResolveToAllMode() { HoodieTableConfig cfg = configOf(null, null); assertTrue(cfg.populateMetaFields(), "populateMetaFields default must remain true"); - assertTrue(cfg.getMetaFieldsMode().isEmpty(), "mode list defaults to empty"); - assertTrue(cfg.isCommitTimePopulated(), "commit time must be populated in ALL mode"); - assertTrue(cfg.isFileNamePopulated(), "file name must be populated in ALL mode"); - assertTrue(cfg.isRecordKeyPopulated(), "record key must be populated in ALL mode"); + assertEquals(MetaFieldsMode.ALL, cfg.getMetaFieldsMode()); + assertTrue(cfg.isCommitTimePopulated()); + assertTrue(cfg.isFileNamePopulated()); + assertTrue(cfg.isRecordKeyPopulated()); } @Test void noneModeWhenPopulateFalseAndModeEmpty() { HoodieTableConfig cfg = configOf(false, ""); assertFalse(cfg.populateMetaFields()); - assertTrue(cfg.getMetaFieldsMode().isEmpty()); + assertEquals(MetaFieldsMode.NONE, cfg.getMetaFieldsMode()); assertFalse(cfg.isCommitTimePopulated()); assertFalse(cfg.isFileNamePopulated()); assertFalse(cfg.isRecordKeyPopulated()); @@ -77,10 +68,10 @@ void noneModeWhenPopulateFalseAndModeEmpty() { @Test void noneModeWhenPopulateFalseAndModeUnset() { - // Existing populate.meta.fields=false table without the mode property must still resolve to NONE. + // Existing populate.meta.fields=false table without the mode property must resolve to NONE. HoodieTableConfig cfg = configOf(false, null); assertFalse(cfg.populateMetaFields()); - assertTrue(cfg.getMetaFieldsMode().isEmpty()); + assertEquals(MetaFieldsMode.NONE, cfg.getMetaFieldsMode()); assertFalse(cfg.isCommitTimePopulated()); assertFalse(cfg.isFileNamePopulated()); assertFalse(cfg.isRecordKeyPopulated()); @@ -88,8 +79,9 @@ void noneModeWhenPopulateFalseAndModeUnset() { @Test void commitTimeOnlyMode() { - HoodieTableConfig cfg = configOf(false, HoodieRecord.COMMIT_TIME_METADATA_FIELD); + HoodieTableConfig cfg = configOf(false, MetaFieldsMode.COMMIT_TIME_ONLY.name()); assertFalse(cfg.populateMetaFields()); + assertEquals(MetaFieldsMode.COMMIT_TIME_ONLY, cfg.getMetaFieldsMode()); assertTrue(cfg.isCommitTimePopulated()); assertFalse(cfg.isFileNamePopulated()); assertFalse(cfg.isRecordKeyPopulated()); @@ -97,8 +89,9 @@ void commitTimeOnlyMode() { @Test void fileNameOnlyMode() { - HoodieTableConfig cfg = configOf(false, HoodieRecord.FILENAME_METADATA_FIELD); + HoodieTableConfig cfg = configOf(false, MetaFieldsMode.FILE_NAME_ONLY.name()); assertFalse(cfg.populateMetaFields()); + assertEquals(MetaFieldsMode.FILE_NAME_ONLY, cfg.getMetaFieldsMode()); assertFalse(cfg.isCommitTimePopulated()); assertTrue(cfg.isFileNamePopulated()); assertFalse(cfg.isRecordKeyPopulated()); @@ -106,9 +99,9 @@ void fileNameOnlyMode() { @Test void commitTimeAndFileNameMode() { - HoodieTableConfig cfg = configOf(false, - HoodieRecord.COMMIT_TIME_METADATA_FIELD + "," + HoodieRecord.FILENAME_METADATA_FIELD); + HoodieTableConfig cfg = configOf(false, MetaFieldsMode.COMMIT_TIME_AND_FILE_NAME.name()); assertFalse(cfg.populateMetaFields()); + assertEquals(MetaFieldsMode.COMMIT_TIME_AND_FILE_NAME, cfg.getMetaFieldsMode()); assertTrue(cfg.isCommitTimePopulated()); assertTrue(cfg.isFileNamePopulated()); assertFalse(cfg.isRecordKeyPopulated()); @@ -116,30 +109,27 @@ void commitTimeAndFileNameMode() { @Test void modeIsIgnoredWhenPopulateMetaFieldsIsTrue() { - // Note: writer-side validate() rejects this combination, but the accessor must still report - // ALL semantics defensively if a bad combo ever leaks through (mode has no effect when all - // meta fields are already populated). - HoodieTableConfig cfg = configOf(true, HoodieRecord.COMMIT_TIME_METADATA_FIELD); - assertTrue(cfg.populateMetaFields()); - assertTrue(cfg.isCommitTimePopulated()); - assertTrue(cfg.isFileNamePopulated()); - assertTrue(cfg.isRecordKeyPopulated()); + // populate.meta.fields=true always resolves to ALL — the raw mode on disk is not consulted. + // Writer-side validate() rejects this combination up-front, but the accessor must still report + // ALL semantics defensively. + HoodieTableConfig cfg = configOf(true, MetaFieldsMode.COMMIT_TIME_ONLY.name()); + assertEquals(MetaFieldsMode.ALL, cfg.getMetaFieldsMode()); } @Test void unknownTokenIsRejected() { - HoodieTableConfig cfg = configOf(false, HoodieRecord.RECORD_KEY_METADATA_FIELD); + HoodieTableConfig cfg = configOf(false, "GARBAGE_VALUE"); IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, cfg::getMetaFieldsMode); - assertTrue(ex.getMessage().contains(HoodieRecord.RECORD_KEY_METADATA_FIELD), - "message must name the rejected token: " + ex.getMessage()); - assertTrue(ex.getMessage().contains("populate.meta.fields"), - "message must recommend populate.meta.fields=true for other columns: " + ex.getMessage()); + assertTrue(ex.getMessage().contains("GARBAGE_VALUE"), + "message must name the rejected value: " + ex.getMessage()); + assertTrue(ex.getMessage().contains("hoodie.meta.fields.mode"), + "message must name the property: " + ex.getMessage()); } @Test - void whitespaceAndEmptyTokensAreTolerated() { - HoodieTableConfig cfg = configOf(false, " " + HoodieRecord.COMMIT_TIME_METADATA_FIELD + " , "); - assertTrue(cfg.isCommitTimePopulated()); - assertFalse(cfg.isFileNamePopulated()); + void modeStringIsCaseSensitiveAndTrimmed() { + // Enum-name form is uppercase-only; whitespace around the value is tolerated. + HoodieTableConfig cfg = configOf(false, " COMMIT_TIME_ONLY "); + assertEquals(MetaFieldsMode.COMMIT_TIME_ONLY, cfg.getMetaFieldsMode()); } } diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala index c2580a0e37dd3..5d66069733589 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala @@ -309,7 +309,7 @@ class HoodieSparkSqlWriterInternal { .setOrderingFields(ConfigUtils.getOrderingFieldsStrDuringWrite(optParams.asJava)) .setPartitionFields(partitionColumnsForKeyGenerator) .setPopulateMetaFields(populateMetaFields) - .setMetaFieldsMode(metaFieldsMode) + .setMetaFieldsModeFromString(metaFieldsMode) .setRecordKeyFields(hoodieConfig.getString(RECORDKEY_FIELD)) .setSecondaryKeyFields(hoodieConfig.getString(SECONDARYKEY_COLUMN_NAME)) .setCDCEnabled(hoodieConfig.getBooleanOrDefault(HoodieTableConfig.CDC_ENABLED)) @@ -786,7 +786,7 @@ class HoodieSparkSqlWriterInternal { .setCDCEnabled(hoodieConfig.getBooleanOrDefault(HoodieTableConfig.CDC_ENABLED)) .setCDCSupplementalLoggingMode(hoodieConfig.getStringOrDefault(HoodieTableConfig.CDC_SUPPLEMENTAL_LOGGING_MODE)) .setPopulateMetaFields(populateMetaFields) - .setMetaFieldsMode(metaFieldsMode) + .setMetaFieldsModeFromString(metaFieldsMode) .setKeyGeneratorClassProp(keyGenProp) .setPartitionValueExtractorClass(partitionValueExtractorClassName) .set(timestampKeyGeneratorConfigs.asJava.asInstanceOf[java.util.Map[String, Object]]) diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieWriterUtils.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieWriterUtils.scala index bab13c98a4156..e0cbe5858849d 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieWriterUtils.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieWriterUtils.scala @@ -358,14 +358,16 @@ object HoodieWriterUtils { // pruning behavior differs between old and new commits. The default loop above only flags // the mismatch when the on-disk value is non-null, so an older table with the property // absent from hoodie.properties would let a null → non-empty transition slip through - // (silent-drop risk on pre-enablement commits). Guard the null → non-empty case explicitly - // here. Set it only at table creation, via the hudi-cli, or during table upgrade. + // (silent-drop risk on pre-enablement commits). Guard the null → selective-mode case + // explicitly here. Set it only at table creation, via the hudi-cli, or during table upgrade; + // otherwise the only way to change it is to recreate the table. val paramsMetaFieldsMode = params.getOrElse(HoodieTableConfig.META_FIELDS_MODE.key(), "") val onDiskMetaFieldsMode = tableConfig.getString(HoodieTableConfig.META_FIELDS_MODE) if (paramsMetaFieldsMode.nonEmpty && (onDiskMetaFieldsMode == null || onDiskMetaFieldsMode.isEmpty)) { diffConfigs.append( s"${HoodieTableConfig.META_FIELDS_MODE.key()}:\t$paramsMetaFieldsMode\t${if (onDiskMetaFieldsMode == null) "null" else "\"\""}" - + " (immutable at runtime; set only at table creation / hudi-cli / upgrade)\n") + + " (immutable at runtime; set only at table creation / hudi-cli / upgrade; " + + "existing tables must be recreated to change this)\n") } } diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/IncrementalRelationV1.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/IncrementalRelationV1.scala index db33f735a6508..31c9620a7b0b6 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/IncrementalRelationV1.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/IncrementalRelationV1.scala @@ -91,7 +91,10 @@ class IncrementalRelationV1(val sqlContext: SQLContext, if (!metaClient.getTableConfig.isCommitTimePopulated()) { throw new HoodieException("Incremental queries are not supported when _hoodie_commit_time is not populated. " - + "Either keep hoodie.populate.meta.fields=true or include _hoodie_commit_time in hoodie.meta.fields.mode.") + + "hoodie.meta.fields.mode is a physical-storage decision baked into files at write time and cannot be " + + "changed by flipping write options — setting it only takes effect at table creation. To enable incremental " + + "queries on this table, recreate it with hoodie.populate.meta.fields=true or hoodie.meta.fields.mode=COMMIT_TIME_ONLY " + + "(or COMMIT_TIME_AND_FILE_NAME).") } private val useEndInstantSchema = optParams.getOrElse(INCREMENTAL_READ_SCHEMA_USE_END_INSTANTTIME.key, diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/IncrementalRelationV2.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/IncrementalRelationV2.scala index 311e24baad25d..47fbd3799ffee 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/IncrementalRelationV2.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/IncrementalRelationV2.scala @@ -80,7 +80,10 @@ class IncrementalRelationV2(val sqlContext: SQLContext, if (!metaClient.getTableConfig.isCommitTimePopulated()) { throw new HoodieException("Incremental queries are not supported when _hoodie_commit_time is not populated. " - + "Either keep hoodie.populate.meta.fields=true or include _hoodie_commit_time in hoodie.meta.fields.mode.") + + "hoodie.meta.fields.mode is a physical-storage decision baked into files at write time and cannot be " + + "changed by flipping write options — setting it only takes effect at table creation. To enable incremental " + + "queries on this table, recreate it with hoodie.populate.meta.fields=true or hoodie.meta.fields.mode=COMMIT_TIME_ONLY " + + "(or COMMIT_TIME_AND_FILE_NAME).") } private val queryContext: IncrementalQueryAnalyzer.QueryContext = diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestMetaFieldsMode.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestMetaFieldsMode.java index 12c01e303708d..f05d3f3804ab3 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestMetaFieldsMode.java +++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestMetaFieldsMode.java @@ -23,10 +23,12 @@ import org.apache.hudi.SparkAdapterSupport$; import org.apache.hudi.common.config.HoodieMetadataConfig; import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.MetaFieldsMode; import org.apache.hudi.common.table.HoodieTableConfig; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.testutils.SparkClientFunctionalTestHarness; +import org.apache.spark.sql.Dataset; import org.apache.spark.sql.Row; import org.apache.spark.sql.RowFactory; import org.apache.spark.sql.SaveMode; @@ -35,6 +37,7 @@ import org.apache.spark.sql.types.StructType; import org.junit.jupiter.api.Test; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -42,30 +45,18 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; /** - * Spark-datasource tests for the {@code hoodie.meta.fields.mode} property on CoW tables. - * - *

Modes covered: - *

    - *
  • ALL — {@code populate.meta.fields=true} (default).
    • - *
  • NONE — {@code populate.meta.fields=false} and mode empty.
    • - *
  • COMMIT_TIME_ONLY — {@code mode=_hoodie_commit_time}.
    • - *
  • FILE_NAME_ONLY — {@code mode=_hoodie_file_name}.
    • - *
  • COMMIT_TIME_AND_FILE_NAME — both tokens in the mode list.
    • - *
- * - *

Rejection paths: an unknown token or the ambiguous - * {@code populate.meta.fields=true} + non-empty mode combination must fail at writer init. MoR + - * non-empty mode is likewise rejected until log-write support lands (tracked as follow-up). + * Spark-datasource end-to-end tests for the {@code hoodie.meta.fields.mode} property on CoW tables. + * Every {@link MetaFieldsMode} value is exercised via a write / re-read round trip; on-disk column + * population is verified by reading the parquet files back and inspecting the meta-column values. */ class TestMetaFieldsMode extends SparkClientFunctionalTestHarness { - private static final String COMMIT_TIME_TOKEN = HoodieRecord.COMMIT_TIME_METADATA_FIELD; - private static final String FILE_NAME_TOKEN = HoodieRecord.FILENAME_METADATA_FIELD; - private static StructType simpleSchema() { return DataTypes.createStructType(new StructField[]{ DataTypes.createStructField("column1", DataTypes.StringType, true), @@ -95,96 +86,124 @@ private void writeRows(List records, StructType schema, Map .save(path); } - private HoodieTableConfig writeAndReadTableConfig(Map options, String path) { - writeRows(Collections.singletonList(RowFactory.create("k1", "p1", "v1")), + private HoodieTableConfig writeSampleAndGetTableConfig(Map options, String path) { + writeRows(Arrays.asList( + RowFactory.create("k1", "p1", "v1"), + RowFactory.create("k2", "p1", "v2")), simpleSchema(), options, path, SaveMode.Overwrite); HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setBasePath(path).setConf(storageConf()).build(); return metaClient.getTableConfig(); } + /** + * End-to-end assertion of the on-disk meta columns after a write. Reads the parquet files back + * (bypassing Hudi's own read path so we see the raw column values) and asserts which meta + * columns are non-null. + */ + private void assertMetaColumnPopulation(String path, MetaFieldsMode expectedMode) { + Dataset raw = spark().read().parquet(path + "/*/*.parquet"); + Row first = raw.select( + HoodieRecord.COMMIT_TIME_METADATA_FIELD, + HoodieRecord.COMMIT_SEQNO_METADATA_FIELD, + HoodieRecord.RECORD_KEY_METADATA_FIELD, + HoodieRecord.PARTITION_PATH_METADATA_FIELD, + HoodieRecord.FILENAME_METADATA_FIELD).first(); + + if (expectedMode.isCommitTimePopulated()) { + assertNotNull(first.get(0), "expected _hoodie_commit_time to be populated for mode " + expectedMode); + } else { + assertNull(first.get(0), "expected _hoodie_commit_time to be null for mode " + expectedMode); + } + if (expectedMode.isFileNamePopulated()) { + assertNotNull(first.get(4), "expected _hoodie_file_name to be populated for mode " + expectedMode); + } else { + assertNull(first.get(4), "expected _hoodie_file_name to be null for mode " + expectedMode); + } + // Record key, partition path, and commit seq no are ALL-only. + if (expectedMode == MetaFieldsMode.ALL) { + assertNotNull(first.get(2), "record key must be populated in ALL mode"); + assertNotNull(first.get(3), "partition path must be populated in ALL mode"); + assertNotNull(first.get(1), "commit seq no must be populated in ALL mode"); + } else { + assertNull(first.get(2), "record key must be null outside ALL mode, got: " + first.get(2)); + assertNull(first.get(3), "partition path must be null outside ALL mode, got: " + first.get(3)); + assertNull(first.get(1), "commit seq no must be null outside ALL mode, got: " + first.get(1)); + } + } + @Test - void commitTimeOnlyModePersistsPropertyAndReportsMode() { + void allModePersistsAndPopulatesAllColumns() { Map options = baseOptions(); - options.put(HoodieTableConfig.POPULATE_META_FIELDS.key(), "false"); - options.put(HoodieTableConfig.META_FIELDS_MODE.key(), COMMIT_TIME_TOKEN); + // ALL is the default; no need to set the mode explicitly. options.put(DataSourceWriteOptions.OPERATION().key(), DataSourceWriteOptions.BULK_INSERT_OPERATION_OPT_VAL()); - HoodieTableConfig tc = writeAndReadTableConfig(options, basePath()); + HoodieTableConfig tc = writeSampleAndGetTableConfig(options, basePath()); - assertEquals("false", tc.getProps().getProperty(HoodieTableConfig.POPULATE_META_FIELDS.key())); - assertEquals(COMMIT_TIME_TOKEN, tc.getProps().getProperty(HoodieTableConfig.META_FIELDS_MODE.key())); - assertFalse(tc.populateMetaFields()); - assertTrue(tc.isCommitTimePopulated()); - assertFalse(tc.isFileNamePopulated()); - assertFalse(tc.isRecordKeyPopulated()); + assertTrue(tc.populateMetaFields()); + assertEquals(MetaFieldsMode.ALL, tc.getMetaFieldsMode()); + assertMetaColumnPopulation(basePath(), MetaFieldsMode.ALL); } @Test - void fileNameOnlyModePersistsPropertyAndReportsMode() { + void noneModePersistsAndLeavesAllColumnsNull() { Map options = baseOptions(); options.put(HoodieTableConfig.POPULATE_META_FIELDS.key(), "false"); - options.put(HoodieTableConfig.META_FIELDS_MODE.key(), FILE_NAME_TOKEN); options.put(DataSourceWriteOptions.OPERATION().key(), DataSourceWriteOptions.BULK_INSERT_OPERATION_OPT_VAL()); - HoodieTableConfig tc = writeAndReadTableConfig(options, basePath()); + HoodieTableConfig tc = writeSampleAndGetTableConfig(options, basePath()); - assertEquals(FILE_NAME_TOKEN, tc.getProps().getProperty(HoodieTableConfig.META_FIELDS_MODE.key())); assertFalse(tc.populateMetaFields()); - assertFalse(tc.isCommitTimePopulated()); - assertTrue(tc.isFileNamePopulated()); - assertFalse(tc.isRecordKeyPopulated()); + assertEquals(MetaFieldsMode.NONE, tc.getMetaFieldsMode()); + assertMetaColumnPopulation(basePath(), MetaFieldsMode.NONE); } @Test - void commitTimeAndFileNameModePersistsAndReports() { - String combined = COMMIT_TIME_TOKEN + "," + FILE_NAME_TOKEN; + void commitTimeOnlyModePopulatesOnlyCommitTime() { Map options = baseOptions(); options.put(HoodieTableConfig.POPULATE_META_FIELDS.key(), "false"); - options.put(HoodieTableConfig.META_FIELDS_MODE.key(), combined); + options.put(HoodieTableConfig.META_FIELDS_MODE.key(), MetaFieldsMode.COMMIT_TIME_ONLY.name()); options.put(DataSourceWriteOptions.OPERATION().key(), DataSourceWriteOptions.BULK_INSERT_OPERATION_OPT_VAL()); - HoodieTableConfig tc = writeAndReadTableConfig(options, basePath()); + HoodieTableConfig tc = writeSampleAndGetTableConfig(options, basePath()); - assertEquals(combined, tc.getProps().getProperty(HoodieTableConfig.META_FIELDS_MODE.key())); - assertTrue(tc.isCommitTimePopulated()); - assertTrue(tc.isFileNamePopulated()); - assertFalse(tc.isRecordKeyPopulated()); + assertEquals(MetaFieldsMode.COMMIT_TIME_ONLY.name(), + tc.getProps().getProperty(HoodieTableConfig.META_FIELDS_MODE.key())); + assertEquals(MetaFieldsMode.COMMIT_TIME_ONLY, tc.getMetaFieldsMode()); + assertMetaColumnPopulation(basePath(), MetaFieldsMode.COMMIT_TIME_ONLY); } @Test - void noneModePersistsAndReportsCorrectly() { + void fileNameOnlyModePopulatesOnlyFileName() { Map options = baseOptions(); options.put(HoodieTableConfig.POPULATE_META_FIELDS.key(), "false"); + options.put(HoodieTableConfig.META_FIELDS_MODE.key(), MetaFieldsMode.FILE_NAME_ONLY.name()); options.put(DataSourceWriteOptions.OPERATION().key(), DataSourceWriteOptions.BULK_INSERT_OPERATION_OPT_VAL()); - HoodieTableConfig tc = writeAndReadTableConfig(options, basePath()); + HoodieTableConfig tc = writeSampleAndGetTableConfig(options, basePath()); - assertFalse(tc.populateMetaFields()); - assertTrue(tc.getMetaFieldsMode().isEmpty()); - assertFalse(tc.isCommitTimePopulated()); - assertFalse(tc.isFileNamePopulated()); - assertFalse(tc.isRecordKeyPopulated()); + assertEquals(MetaFieldsMode.FILE_NAME_ONLY, tc.getMetaFieldsMode()); + assertMetaColumnPopulation(basePath(), MetaFieldsMode.FILE_NAME_ONLY); } @Test - void defaultAllModeReportsPopulated() { + void commitTimeAndFileNameModePopulatesBoth() { Map options = baseOptions(); + options.put(HoodieTableConfig.POPULATE_META_FIELDS.key(), "false"); + options.put(HoodieTableConfig.META_FIELDS_MODE.key(), MetaFieldsMode.COMMIT_TIME_AND_FILE_NAME.name()); options.put(DataSourceWriteOptions.OPERATION().key(), DataSourceWriteOptions.BULK_INSERT_OPERATION_OPT_VAL()); - HoodieTableConfig tc = writeAndReadTableConfig(options, basePath()); + HoodieTableConfig tc = writeSampleAndGetTableConfig(options, basePath()); - assertTrue(tc.populateMetaFields()); - assertTrue(tc.isCommitTimePopulated()); - assertTrue(tc.isFileNamePopulated()); - assertTrue(tc.isRecordKeyPopulated()); + assertEquals(MetaFieldsMode.COMMIT_TIME_AND_FILE_NAME, tc.getMetaFieldsMode()); + assertMetaColumnPopulation(basePath(), MetaFieldsMode.COMMIT_TIME_AND_FILE_NAME); } @Test - void populateTrueWithNonEmptyModeIsRejected() { + void populateTrueWithSelectiveModeIsRejected() { Map options = baseOptions(); options.put(HoodieTableConfig.POPULATE_META_FIELDS.key(), "true"); - options.put(HoodieTableConfig.META_FIELDS_MODE.key(), COMMIT_TIME_TOKEN); + options.put(HoodieTableConfig.META_FIELDS_MODE.key(), MetaFieldsMode.COMMIT_TIME_ONLY.name()); options.put(DataSourceWriteOptions.OPERATION().key(), DataSourceWriteOptions.BULK_INSERT_OPERATION_OPT_VAL()); Throwable thrown = assertThrows(Throwable.class, () -> @@ -198,10 +217,10 @@ void populateTrueWithNonEmptyModeIsRejected() { } @Test - void unknownTokenInModeIsRejected() { + void unknownModeValueIsRejected() { Map options = baseOptions(); options.put(HoodieTableConfig.POPULATE_META_FIELDS.key(), "false"); - options.put(HoodieTableConfig.META_FIELDS_MODE.key(), HoodieRecord.RECORD_KEY_METADATA_FIELD); + options.put(HoodieTableConfig.META_FIELDS_MODE.key(), "SOMETHING_BOGUS"); options.put(DataSourceWriteOptions.OPERATION().key(), DataSourceWriteOptions.BULK_INSERT_OPERATION_OPT_VAL()); Throwable thrown = assertThrows(Throwable.class, () -> @@ -209,16 +228,193 @@ void unknownTokenInModeIsRejected() { simpleSchema(), options, basePath(), SaveMode.Overwrite)); String rootMessage = rootMessageOf(thrown); - assertTrue(rootMessage.contains(HoodieRecord.RECORD_KEY_METADATA_FIELD), - "Expected error to name the rejected token, got: " + rootMessage); + assertTrue(rootMessage.contains("SOMETHING_BOGUS"), + "Expected error to name the rejected value, got: " + rootMessage); + } + + // ------------------------------------------------------------------------- + // Non-row-writer path coverage. Bulk insert with row.writer.enable=false forces the + // HoodieAvroParquetWriter path (via HoodieCreateHandle) instead of the internal-row writer path. + // Both paths must respect the mode identically. + // ------------------------------------------------------------------------- + + @Test + void nonRowWriterPathAllMode() { + Map options = baseOptions(); + options.put(DataSourceWriteOptions.OPERATION().key(), DataSourceWriteOptions.INSERT_OPERATION_OPT_VAL()); + options.put("hoodie.datasource.write.row.writer.enable", "false"); + + HoodieTableConfig tc = writeSampleAndGetTableConfig(options, basePath()); + assertEquals(MetaFieldsMode.ALL, tc.getMetaFieldsMode()); + assertMetaColumnPopulation(basePath(), MetaFieldsMode.ALL); + } + + @Test + void nonRowWriterPathNoneMode() { + Map options = baseOptions(); + options.put(HoodieTableConfig.POPULATE_META_FIELDS.key(), "false"); + options.put(DataSourceWriteOptions.OPERATION().key(), DataSourceWriteOptions.INSERT_OPERATION_OPT_VAL()); + options.put("hoodie.datasource.write.row.writer.enable", "false"); + + HoodieTableConfig tc = writeSampleAndGetTableConfig(options, basePath()); + assertEquals(MetaFieldsMode.NONE, tc.getMetaFieldsMode()); + assertMetaColumnPopulation(basePath(), MetaFieldsMode.NONE); + } + + @Test + void nonRowWriterPathCommitTimeOnly() { + Map options = baseOptions(); + options.put(HoodieTableConfig.POPULATE_META_FIELDS.key(), "false"); + options.put(HoodieTableConfig.META_FIELDS_MODE.key(), MetaFieldsMode.COMMIT_TIME_ONLY.name()); + options.put(DataSourceWriteOptions.OPERATION().key(), DataSourceWriteOptions.INSERT_OPERATION_OPT_VAL()); + options.put("hoodie.datasource.write.row.writer.enable", "false"); + + HoodieTableConfig tc = writeSampleAndGetTableConfig(options, basePath()); + assertEquals(MetaFieldsMode.COMMIT_TIME_ONLY, tc.getMetaFieldsMode()); + assertMetaColumnPopulation(basePath(), MetaFieldsMode.COMMIT_TIME_ONLY); + } + + @Test + void nonRowWriterPathFileNameOnly() { + Map options = baseOptions(); + options.put(HoodieTableConfig.POPULATE_META_FIELDS.key(), "false"); + options.put(HoodieTableConfig.META_FIELDS_MODE.key(), MetaFieldsMode.FILE_NAME_ONLY.name()); + options.put(DataSourceWriteOptions.OPERATION().key(), DataSourceWriteOptions.INSERT_OPERATION_OPT_VAL()); + options.put("hoodie.datasource.write.row.writer.enable", "false"); + + HoodieTableConfig tc = writeSampleAndGetTableConfig(options, basePath()); + assertEquals(MetaFieldsMode.FILE_NAME_ONLY, tc.getMetaFieldsMode()); + assertMetaColumnPopulation(basePath(), MetaFieldsMode.FILE_NAME_ONLY); + } + + @Test + void nonRowWriterPathCommitTimeAndFileName() { + Map options = baseOptions(); + options.put(HoodieTableConfig.POPULATE_META_FIELDS.key(), "false"); + options.put(HoodieTableConfig.META_FIELDS_MODE.key(), MetaFieldsMode.COMMIT_TIME_AND_FILE_NAME.name()); + options.put(DataSourceWriteOptions.OPERATION().key(), DataSourceWriteOptions.INSERT_OPERATION_OPT_VAL()); + options.put("hoodie.datasource.write.row.writer.enable", "false"); + + HoodieTableConfig tc = writeSampleAndGetTableConfig(options, basePath()); + assertEquals(MetaFieldsMode.COMMIT_TIME_AND_FILE_NAME, tc.getMetaFieldsMode()); + assertMetaColumnPopulation(basePath(), MetaFieldsMode.COMMIT_TIME_AND_FILE_NAME); + } + + // ------------------------------------------------------------------------- + // Clustering coverage. Inline clustering rewrites files through the create/merge handles which + // delegate to the same underlying HoodieAvroParquetWriter / HoodieRowCreateHandle we exercise + // in the write tests. Verifies clustered files preserve the mode's column population semantics. + // ------------------------------------------------------------------------- + + @Test + void clusteringPreservesCommitTimeOnlyMode() { + Map options = baseOptions(); + options.put(HoodieTableConfig.POPULATE_META_FIELDS.key(), "false"); + options.put(HoodieTableConfig.META_FIELDS_MODE.key(), MetaFieldsMode.COMMIT_TIME_ONLY.name()); + options.put(DataSourceWriteOptions.OPERATION().key(), DataSourceWriteOptions.BULK_INSERT_OPERATION_OPT_VAL()); + // Inline clustering after each write. + options.put("hoodie.clustering.inline", "true"); + options.put("hoodie.clustering.inline.max.commits", "1"); + options.put("hoodie.clustering.plan.strategy.target.file.max.bytes", "10485760"); + options.put("hoodie.clustering.plan.strategy.small.file.limit", "10485760"); + + writeRows(Arrays.asList( + RowFactory.create("k1", "p1", "v1"), + RowFactory.create("k2", "p1", "v2"), + RowFactory.create("k3", "p1", "v3")), + simpleSchema(), options, basePath(), SaveMode.Overwrite); + + HoodieTableMetaClient metaClient = + HoodieTableMetaClient.builder().setBasePath(basePath()).setConf(storageConf()).build(); + assertEquals(MetaFieldsMode.COMMIT_TIME_ONLY, metaClient.getTableConfig().getMetaFieldsMode()); + // After clustering, files are rewritten — verify the rewritten files still respect the mode. + assertMetaColumnPopulation(basePath(), MetaFieldsMode.COMMIT_TIME_ONLY); + } + + @Test + void clusteringPreservesFileNameOnlyMode() { + Map options = baseOptions(); + options.put(HoodieTableConfig.POPULATE_META_FIELDS.key(), "false"); + options.put(HoodieTableConfig.META_FIELDS_MODE.key(), MetaFieldsMode.FILE_NAME_ONLY.name()); + options.put(DataSourceWriteOptions.OPERATION().key(), DataSourceWriteOptions.BULK_INSERT_OPERATION_OPT_VAL()); + options.put("hoodie.clustering.inline", "true"); + options.put("hoodie.clustering.inline.max.commits", "1"); + options.put("hoodie.clustering.plan.strategy.target.file.max.bytes", "10485760"); + options.put("hoodie.clustering.plan.strategy.small.file.limit", "10485760"); + + writeRows(Arrays.asList( + RowFactory.create("k1", "p1", "v1"), + RowFactory.create("k2", "p1", "v2"), + RowFactory.create("k3", "p1", "v3")), + simpleSchema(), options, basePath(), SaveMode.Overwrite); + + assertMetaColumnPopulation(basePath(), MetaFieldsMode.FILE_NAME_ONLY); + } + + @Test + void clusteringPreservesCommitTimeAndFileNameMode() { + Map options = baseOptions(); + options.put(HoodieTableConfig.POPULATE_META_FIELDS.key(), "false"); + options.put(HoodieTableConfig.META_FIELDS_MODE.key(), MetaFieldsMode.COMMIT_TIME_AND_FILE_NAME.name()); + options.put(DataSourceWriteOptions.OPERATION().key(), DataSourceWriteOptions.BULK_INSERT_OPERATION_OPT_VAL()); + options.put("hoodie.clustering.inline", "true"); + options.put("hoodie.clustering.inline.max.commits", "1"); + options.put("hoodie.clustering.plan.strategy.target.file.max.bytes", "10485760"); + options.put("hoodie.clustering.plan.strategy.small.file.limit", "10485760"); + + writeRows(Arrays.asList( + RowFactory.create("k1", "p1", "v1"), + RowFactory.create("k2", "p1", "v2"), + RowFactory.create("k3", "p1", "v3")), + simpleSchema(), options, basePath(), SaveMode.Overwrite); + + assertMetaColumnPopulation(basePath(), MetaFieldsMode.COMMIT_TIME_AND_FILE_NAME); + } + + @Test + void clusteringPreservesAllMode() { + Map options = baseOptions(); + // ALL is the default. + options.put(DataSourceWriteOptions.OPERATION().key(), DataSourceWriteOptions.BULK_INSERT_OPERATION_OPT_VAL()); + options.put("hoodie.clustering.inline", "true"); + options.put("hoodie.clustering.inline.max.commits", "1"); + options.put("hoodie.clustering.plan.strategy.target.file.max.bytes", "10485760"); + options.put("hoodie.clustering.plan.strategy.small.file.limit", "10485760"); + + writeRows(Arrays.asList( + RowFactory.create("k1", "p1", "v1"), + RowFactory.create("k2", "p1", "v2"), + RowFactory.create("k3", "p1", "v3")), + simpleSchema(), options, basePath(), SaveMode.Overwrite); + + assertMetaColumnPopulation(basePath(), MetaFieldsMode.ALL); + } + + @Test + void clusteringPreservesNoneMode() { + Map options = baseOptions(); + options.put(HoodieTableConfig.POPULATE_META_FIELDS.key(), "false"); + options.put(DataSourceWriteOptions.OPERATION().key(), DataSourceWriteOptions.BULK_INSERT_OPERATION_OPT_VAL()); + options.put("hoodie.clustering.inline", "true"); + options.put("hoodie.clustering.inline.max.commits", "1"); + options.put("hoodie.clustering.plan.strategy.target.file.max.bytes", "10485760"); + options.put("hoodie.clustering.plan.strategy.small.file.limit", "10485760"); + + writeRows(Arrays.asList( + RowFactory.create("k1", "p1", "v1"), + RowFactory.create("k2", "p1", "v2"), + RowFactory.create("k3", "p1", "v3")), + simpleSchema(), options, basePath(), SaveMode.Overwrite); + + assertMetaColumnPopulation(basePath(), MetaFieldsMode.NONE); } @Test - void morWithNonEmptyModeIsRejected() { + void morWithSelectiveModeIsRejected() { Map options = baseOptions(); options.put(DataSourceWriteOptions.TABLE_TYPE().key(), "MERGE_ON_READ"); options.put(HoodieTableConfig.POPULATE_META_FIELDS.key(), "false"); - options.put(HoodieTableConfig.META_FIELDS_MODE.key(), COMMIT_TIME_TOKEN); + options.put(HoodieTableConfig.META_FIELDS_MODE.key(), MetaFieldsMode.COMMIT_TIME_ONLY.name()); options.put(DataSourceWriteOptions.OPERATION().key(), DataSourceWriteOptions.BULK_INSERT_OPERATION_OPT_VAL()); Throwable thrown = assertThrows(Throwable.class, () -> diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamSync.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamSync.java index 1ffe12f668774..89538e8cfd969 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamSync.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamSync.java @@ -478,6 +478,8 @@ HoodieTableMetaClient initializeEmptyTable(HoodieTableMetaClient.TableBuilder ta .setRecordKeyFields(props.getProperty(DataSourceWriteOptions.RECORDKEY_FIELD().key())) .setPopulateMetaFields(props.getBoolean(HoodieTableConfig.POPULATE_META_FIELDS.key(), HoodieTableConfig.POPULATE_META_FIELDS.defaultValue())) + .setMetaFieldsModeFromString(props.getString(HoodieTableConfig.META_FIELDS_MODE.key(), + HoodieTableConfig.META_FIELDS_MODE.defaultValue())) .setKeyGeneratorClassProp(keyGenClassName) .setPartitionValueExtractorClass(partitionValueExtractorClassName) .setOrderingFields(cfg.sourceOrderingFields) diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieStreamerMetaFieldsMode.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieStreamerMetaFieldsMode.java new file mode 100644 index 0000000000000..ee073d2da0095 --- /dev/null +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieStreamerMetaFieldsMode.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.utilities.deltastreamer; + +import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.MetaFieldsMode; +import org.apache.hudi.common.model.WriteOperationType; +import org.apache.hudi.common.table.HoodieTableConfig; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.testutils.HoodieTestUtils; + +import org.apache.spark.sql.Dataset; +import org.apache.spark.sql.Row; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * End-to-end coverage for {@code hoodie.meta.fields.mode} through the HoodieStreamer entrypoint. + * Each parameterized invocation runs a single ingest cycle in the given {@link MetaFieldsMode} and + * verifies both the persisted table property and the actual on-disk parquet column population. + * + *

Rejection paths (unknown token, populate=true+mode, MoR+mode) are exercised in the datasource + * test {@code TestMetaFieldsMode}; this fixture focuses on the streamer control-flow. + */ +public class TestHoodieStreamerMetaFieldsMode extends HoodieDeltaStreamerTestBase { + + @ParameterizedTest + @EnumSource(value = MetaFieldsMode.class, + names = {"ALL", "NONE", "COMMIT_TIME_ONLY", "FILE_NAME_ONLY", "COMMIT_TIME_AND_FILE_NAME"}) + public void testStreamerRespectsMetaFieldsMode(MetaFieldsMode mode) throws Exception { + String tablePath = basePath + "/streamer_meta_fields_mode_" + mode.name(); + HoodieDeltaStreamer.Config cfg = TestHelpers.makeConfig(tablePath, WriteOperationType.INSERT); + // Force CoW; selective modes are CoW-only until MoR log-write is wired. + cfg.tableType = "COPY_ON_WRITE"; + switch (mode) { + case ALL: + // default; nothing to add + break; + case NONE: + cfg.configs.add(HoodieTableConfig.POPULATE_META_FIELDS.key() + "=false"); + break; + default: + cfg.configs.add(HoodieTableConfig.POPULATE_META_FIELDS.key() + "=false"); + cfg.configs.add(HoodieTableConfig.META_FIELDS_MODE.key() + "=" + mode.name()); + break; + } + HoodieDeltaStreamer streamer = new HoodieDeltaStreamer(cfg, jsc); + streamer.getIngestionService().ingestOnce(); + streamer.shutdownGracefully(); + + HoodieTableMetaClient metaClient = HoodieTestUtils.createMetaClient(context, tablePath); + assertEquals(mode, metaClient.getTableConfig().getMetaFieldsMode(), + "streamer must persist mode=" + mode + " on hoodie.properties"); + assertOnDiskMetaColumns(tablePath, mode); + } + + @Test + public void testStreamerRejectsMorWithSelectiveMode() throws Exception { + String tablePath = basePath + "/streamer_mor_selective_rejected"; + HoodieDeltaStreamer.Config cfg = TestHelpers.makeConfig(tablePath, WriteOperationType.BULK_INSERT); + cfg.tableType = "MERGE_ON_READ"; + cfg.configs.add(HoodieTableConfig.POPULATE_META_FIELDS.key() + "=false"); + cfg.configs.add(HoodieTableConfig.META_FIELDS_MODE.key() + "=" + MetaFieldsMode.COMMIT_TIME_ONLY.name()); + + Throwable thrown = assertThrows(Throwable.class, () -> { + HoodieDeltaStreamer streamer = new HoodieDeltaStreamer(cfg, jsc); + streamer.getIngestionService().ingestOnce(); + streamer.shutdownGracefully(); + }); + + String rootMessage = rootMessageOf(thrown); + assertTrue(rootMessage.contains("COPY_ON_WRITE") || rootMessage.contains("MERGE_ON_READ") + || rootMessage.contains("MoR") || rootMessage.contains(HoodieTableConfig.META_FIELDS_MODE.key()), + "Expected MoR-restriction error, got: " + rootMessage); + } + + private void assertOnDiskMetaColumns(String tablePath, MetaFieldsMode expectedMode) { + // Default HoodieTestDataGenerator partitions are YYYY/MM/DD (three levels). + Dataset raw = sparkSession.read().parquet(tablePath + "/*/*/*/*.parquet"); + Row first = raw.select( + HoodieRecord.COMMIT_TIME_METADATA_FIELD, + HoodieRecord.COMMIT_SEQNO_METADATA_FIELD, + HoodieRecord.RECORD_KEY_METADATA_FIELD, + HoodieRecord.PARTITION_PATH_METADATA_FIELD, + HoodieRecord.FILENAME_METADATA_FIELD).first(); + + if (expectedMode.isCommitTimePopulated()) { + assertNotNull(first.get(0), "commit_time must be populated in mode " + expectedMode); + } else { + assertNull(first.get(0), "commit_time must be null in mode " + expectedMode); + } + if (expectedMode.isFileNamePopulated()) { + assertNotNull(first.get(4), "file_name must be populated in mode " + expectedMode); + } else { + assertNull(first.get(4), "file_name must be null in mode " + expectedMode); + } + if (expectedMode == MetaFieldsMode.ALL) { + assertNotNull(first.get(1), "commit_seq_no must be populated in ALL mode"); + assertNotNull(first.get(2), "record_key must be populated in ALL mode"); + assertNotNull(first.get(3), "partition_path must be populated in ALL mode"); + } else { + assertNull(first.get(1), "commit_seq_no must be null outside ALL mode"); + assertNull(first.get(2), "record_key must be null outside ALL mode"); + assertNull(first.get(3), "partition_path must be null outside ALL mode"); + } + } + + private static String rootMessageOf(Throwable thrown) { + Throwable root = thrown; + while (root.getCause() != null) { + root = root.getCause(); + } + return root.getMessage() == null ? "" : root.getMessage(); + } +} From 77f4e9148c4aa515bc157c7762fa9e06e8db326a Mon Sep 17 00:00:00 2001 From: sivabalan Date: Mon, 13 Jul 2026 12:34:59 -0700 Subject: [PATCH 03/13] fix(writer): make row-writer bulk-insert meta-column stubs nullable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously the populateMetaFields=false branch of HoodieDatasetBulkInsertHelper projected the five Hudi meta columns as Alias(Literal(UTF8String.EMPTY_UTF8, StringType), name). A non-null Literal derives a non-null Alias, so the resulting StructField had nullable=false. Under ALL mode this was harmless because HoodieRowCreateHandle.writeRow unconditionally fills every meta slot with a non-null value. Once selective / NONE modes started leaving the opted-out columns null on the row (per hoodie.meta.fields.mode), the non-nullable StructType led SparkToParquetSchemaConverter to declare the columns as `required binary` on disk, and any subsequent read failed with ParquetDecodingException: could not read bytes at offset 0. Switch the stub to Alias(Literal.create(null, StringType), name) so the projected StructField is nullable and the physical parquet column is written as OPTIONAL. Handle-level population (writeRow / writeRowNoMetaFields / writeRowSelectiveMetaFields) is unchanged — the stub value is never observed on disk; the fix only affects the parquet schema. Fixes all row-writer + clustering selective-mode failures in TestMetaFieldsMode (commitTimeOnly / fileNameOnly / commitTimeAndFileName / none, plus clustering variants). 18/18 tests pass locally. --- .../apache/hudi/HoodieDatasetBulkInsertHelper.scala | 7 ++++++- .../TestHoodieDatasetBulkInsertHelper.java | 12 +++++++----- .../org/apache/hudi/TestHoodieSparkSqlWriter.scala | 2 +- .../TestHoodieSparkSqlWriterWithTestFormat.scala | 2 +- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/HoodieDatasetBulkInsertHelper.scala b/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/HoodieDatasetBulkInsertHelper.scala index 0ee2abd847a51..85c33e3f42741 100644 --- a/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/HoodieDatasetBulkInsertHelper.scala +++ b/hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/HoodieDatasetBulkInsertHelper.scala @@ -137,7 +137,12 @@ object HoodieDatasetBulkInsertHelper // need access to the [[InternalRow]] and therefore can avoid the need // to dereference [[DataFrame]] into [[RDD]] val query = df.queryExecution.logical - val metaFieldsStubs = metaFields.map(f => Alias(Literal(UTF8String.EMPTY_UTF8, dataType = StringType), f.name)()) + // Nullable null stubs — the actual meta-column values are set downstream by + // HoodieRowCreateHandle.write based on hoodie.meta.fields.mode. Using a null literal (rather + // than an empty-string literal) guarantees the resulting StructField's nullable=true so the + // physical Parquet column is written as OPTIONAL and can hold nulls under selective / NONE + // modes. + val metaFieldsStubs = metaFields.map(f => Alias(Literal.create(null, StringType), f.name)()) val prependedQuery = Project(metaFieldsStubs ++ query.output, query) sparkAdapter.getUnsafeUtils.createDataFrameFrom(df.sparkSession, prependedQuery) diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestHoodieDatasetBulkInsertHelper.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestHoodieDatasetBulkInsertHelper.java index 946fb61c3b3a8..41cabe3d886df 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestHoodieDatasetBulkInsertHelper.java +++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestHoodieDatasetBulkInsertHelper.java @@ -188,11 +188,13 @@ public void testBulkInsertHelperNoMetaFields() { } result.toJavaRDD().foreach(entry -> { - assertTrue(entry.get(resultSchema.fieldIndex(HoodieRecord.RECORD_KEY_METADATA_FIELD)).equals("")); - assertTrue(entry.get(resultSchema.fieldIndex(HoodieRecord.PARTITION_PATH_METADATA_FIELD)).equals("")); - assertTrue(entry.get(resultSchema.fieldIndex(HoodieRecord.COMMIT_SEQNO_METADATA_FIELD)).equals("")); - assertTrue(entry.get(resultSchema.fieldIndex(HoodieRecord.COMMIT_TIME_METADATA_FIELD)).equals("")); - assertTrue(entry.get(resultSchema.fieldIndex(HoodieRecord.FILENAME_METADATA_FIELD)).equals("")); + // The stub meta columns are now written as null (nullable=true) rather than empty strings — + // see HoodieDatasetBulkInsertHelper. Handle handle-level population fills them in later. + assertTrue(entry.isNullAt(resultSchema.fieldIndex(HoodieRecord.RECORD_KEY_METADATA_FIELD))); + assertTrue(entry.isNullAt(resultSchema.fieldIndex(HoodieRecord.PARTITION_PATH_METADATA_FIELD))); + assertTrue(entry.isNullAt(resultSchema.fieldIndex(HoodieRecord.COMMIT_SEQNO_METADATA_FIELD))); + assertTrue(entry.isNullAt(resultSchema.fieldIndex(HoodieRecord.COMMIT_TIME_METADATA_FIELD))); + assertTrue(entry.isNullAt(resultSchema.fieldIndex(HoodieRecord.FILENAME_METADATA_FIELD))); }); Dataset trimmedOutput = result.drop(HoodieRecord.PARTITION_PATH_METADATA_FIELD).drop(HoodieRecord.RECORD_KEY_METADATA_FIELD) diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriter.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriter.scala index 8968caaf348f0..cd1bf55f7230d 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriter.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriter.scala @@ -107,7 +107,7 @@ class TestHoodieSparkSqlWriter extends HoodieSparkWriterTestBase { // fetch all records from parquet files generated from write to hudi val actualDf = sqlContext.read.parquet(fullPartitionPaths(0), fullPartitionPaths(1), fullPartitionPaths(2)) if (!populateMetaFields) { - List(0, 1, 2, 3, 4).foreach(i => assertEquals(0, actualDf.select(HoodieRecord.HOODIE_META_COLUMNS.get(i)).filter(entry => !(entry.mkString(",").equals(""))).count())) + List(0, 1, 2, 3, 4).foreach(i => assertEquals(0, actualDf.select(HoodieRecord.HOODIE_META_COLUMNS.get(i)).filter(entry => !entry.isNullAt(0) && entry.getString(0).nonEmpty).count())) } // remove metadata columns so that expected and actual DFs can be compared as is val trimmedDf = dropMetaFields(actualDf) diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriterWithTestFormat.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriterWithTestFormat.scala index 2cf0dc20ad08e..4fa26b3993fbb 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriterWithTestFormat.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriterWithTestFormat.scala @@ -106,7 +106,7 @@ class TestHoodieSparkSqlWriterWithTestFormat extends HoodieSparkWriterTestBase { // fetch all records from parquet files generated from write to hudi val actualDf = sqlContext.read.parquet(fullPartitionPaths(0), fullPartitionPaths(1), fullPartitionPaths(2)) if (!populateMetaFields) { - List(0, 1, 2, 3, 4).foreach(i => assertEquals(0, actualDf.select(HoodieRecord.HOODIE_META_COLUMNS.get(i)).filter(entry => !(entry.mkString(",").equals(""))).count())) + List(0, 1, 2, 3, 4).foreach(i => assertEquals(0, actualDf.select(HoodieRecord.HOODIE_META_COLUMNS.get(i)).filter(entry => !entry.isNullAt(0) && entry.getString(0).nonEmpty).count())) } // remove metadata columns so that expected and actual DFs can be compared as is val trimmedDf = dropMetaFields(actualDf) From 13b39db9fb22b3d4d9906efcce3aea0e1dea6048 Mon Sep 17 00:00:00 2001 From: sivabalan Date: Mon, 13 Jul 2026 15:29:35 -0700 Subject: [PATCH 04/13] fix(reader): include meta fields in read schema for selective meta-fields modes TableSchemaResolver.getTableSchema() and getTableSchema(String timestamp) decided whether to include the five Hudi meta columns in the projected schema based on populateMetaFields(). Under the selective modes introduced in this PR (COMMIT_TIME_ONLY / FILE_NAME_ONLY / COMMIT_TIME_AND_FILE_NAME), populateMetaFields() returns false but the opted-in meta columns are still physically present in every Parquet file. The old logic silently stripped those columns from the read schema, so downstream consumers (notably the incremental relations, whose range filter projects _hoodie_commit_time) saw no meta columns at all and produced zero-row reads. Switch the decision to "include meta fields whenever the table's mode is not NONE." That preserves the previous behaviour for the two originally supported modes (ALL -> true, NONE -> false) and correctly turns the flag on for the three new selective modes. No schema change on disk; this only changes what the read side projects. --- .../apache/hudi/common/table/TableSchemaResolver.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/TableSchemaResolver.java b/hudi-common/src/main/java/org/apache/hudi/common/table/TableSchemaResolver.java index 1e8988a742cbd..08c45e2be4e83 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/TableSchemaResolver.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/TableSchemaResolver.java @@ -23,6 +23,7 @@ import org.apache.hudi.common.model.HoodieCommitMetadata; import org.apache.hudi.common.model.HoodieLogFile; import org.apache.hudi.common.model.HoodieRecord; +import org.apache.hudi.common.model.MetaFieldsMode; import org.apache.hudi.common.model.WriteOperationType; import org.apache.hudi.common.schema.HoodieSchema; import org.apache.hudi.common.schema.HoodieSchemaField; @@ -124,7 +125,11 @@ private Option getTableSchemaFromDataFileInternal() { * @throws Exception */ public HoodieSchema getTableSchema() throws Exception { - return getTableSchema(metaClient.getTableConfig().populateMetaFields()); + // Include meta fields whenever the table's meta-fields mode populates any of them. Under + // selective modes (COMMIT_TIME_ONLY / FILE_NAME_ONLY / COMMIT_TIME_AND_FILE_NAME) the meta + // columns exist as physical nullable Parquet columns even though populateMetaFields() is false, + // and read paths (e.g. incremental relations) must see them in the projected schema. + return getTableSchema(metaClient.getTableConfig().getMetaFieldsMode() != MetaFieldsMode.NONE); } /** @@ -148,7 +153,8 @@ public HoodieSchema getTableSchema(String timestamp) throws Exception { .filterCompletedInstants() .findInstantsBeforeOrEquals(timestamp) .lastInstant(); - return getTableSchemaInternal(metaClient.getTableConfig().populateMetaFields(), instant) + return getTableSchemaInternal( + metaClient.getTableConfig().getMetaFieldsMode() != MetaFieldsMode.NONE, instant) .orElseThrow(schemaNotFoundError()); } From c78dc962d92be8c5088e1972fc42a932a32b19d7 Mon Sep 17 00:00:00 2001 From: sivabalan Date: Sat, 25 Jul 2026 20:02:23 -0700 Subject: [PATCH 05/13] refactor(config): make meta.fields.mode the source of truth, deprecate populate.meta.fields MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses review feedback on #19205: composing hoodie.populate.meta.fields and hoodie.meta.fields.mode to determine meta-column population was confusing, and the two-input resolution silently discarded an explicit mode whenever populate.meta.fields=true. hoodie.meta.fields.mode is now the single control. MetaFieldsMode.fromConfig (boolean, String) is replaced by resolve(String, boolean): an explicit mode is the answer outright, and the deprecated boolean is consulted only when the mode is absent, so tables written before the property existed keep resolving to ALL / NONE exactly as before. - Deprecate POPULATE_META_FIELDS (config property, write-config builder setter, and table builder setter). Full call-site migration is a follow-up; the 55 existing populateMetaFields() callers keep working because the accessor now derives from the mode via toLegacyPopulateMetaFields(). - Drop the populate=true + mode mutual-exclusion check from HoodieWriteConfig.validate() — the combination is no longer ambiguous. - Persist an explicit mode verbatim in hoodie.properties, including ALL / NONE. This also closes a hole where a selective mode supplied without an explicit populateMetaFields boolean skipped both cross-validation guards and was then silently ignored at read time. - setMetaFieldsModeFromString now routes through MetaFieldsMode.parse so bad values report the allowed set instead of a bare "No enum constant". --- .../apache/hudi/config/HoodieWriteConfig.java | 53 +++++++------ .../TestHoodieWriteConfigMetaFieldsMode.java | 74 +++++++++++++++---- .../storage/HoodieSparkFileWriterFactory.java | 7 +- .../hudi/common/model/MetaFieldsMode.java | 58 ++++++++++----- .../hudi/common/table/HoodieTableConfig.java | 39 ++++++++-- .../common/table/HoodieTableMetaClient.java | 36 ++++----- .../hadoop/HoodieAvroFileWriterFactory.java | 7 +- .../table/TestHoodieMetaFieldsMode.java | 17 +++-- 8 files changed, 192 insertions(+), 99 deletions(-) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java index 3365694f56367..1096aa2aa32b7 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java @@ -1773,18 +1773,26 @@ public int getSmallFileGroupCandidatesLimit() { return getInt(MERGE_SMALL_FILE_GROUP_CANDIDATES_LIMIT); } + /** + * @return true when every meta column is populated. + * + *

Derived from {@link #getMetaFieldsMode()} so that call sites still written against the + * deprecated {@code hoodie.populate.meta.fields} boolean observe the same answer as the enum: + * only {@link MetaFieldsMode#ALL} populates every meta column. + */ public boolean populateMetaFields() { - return getBooleanOrDefault(HoodieTableConfig.POPULATE_META_FIELDS); + return getMetaFieldsMode().toLegacyPopulateMetaFields(); } /** - * @return the {@link MetaFieldsMode} resolved from the write config. Older tables without the - * mode property fall back to {@link MetaFieldsMode#ALL} or {@link MetaFieldsMode#NONE} based on - * the legacy {@code hoodie.populate.meta.fields} boolean. + * @return the {@link MetaFieldsMode} resolved from the write config. + * {@code hoodie.meta.fields.mode} is the source of truth; configs written before that property + * existed fall back to {@link MetaFieldsMode#ALL} or {@link MetaFieldsMode#NONE} based on the + * deprecated {@code hoodie.populate.meta.fields} boolean. */ public MetaFieldsMode getMetaFieldsMode() { - return MetaFieldsMode.fromConfig(populateMetaFields(), - getStringOrDefault(HoodieTableConfig.META_FIELDS_MODE)); + return MetaFieldsMode.resolve(getStringOrDefault(HoodieTableConfig.META_FIELDS_MODE), + getBooleanOrDefault(HoodieTableConfig.POPULATE_META_FIELDS)); } /** @@ -3611,15 +3619,20 @@ public Builder withCanIgnorePostCommitFailures(boolean canIgnorePostCommitFailur return this; } + /** + * @deprecated since 1.3.0, use {@link #withMetaFieldsMode(MetaFieldsMode)} instead + * ({@code true} maps to {@link MetaFieldsMode#ALL}, {@code false} to {@link MetaFieldsMode#NONE}). + */ + @Deprecated public Builder withPopulateMetaFields(boolean populateMetaFields) { writeConfig.setValue(HoodieTableConfig.POPULATE_META_FIELDS, Boolean.toString(populateMetaFields)); return this; } public Builder withMetaFieldsMode(MetaFieldsMode metaFieldsMode) { - // ALL is the default (implicit) mode — persist the enum name only for selective modes to - // keep hoodie.properties clean for tables that don't opt in. - if (metaFieldsMode == null || metaFieldsMode == MetaFieldsMode.ALL) { + // Leaving the mode unset defers to the deprecated populate.meta.fields boolean; any explicit + // mode (including ALL / NONE) is persisted so it wins over that fallback. + if (metaFieldsMode == null) { writeConfig.setValue(HoodieTableConfig.META_FIELDS_MODE, ""); } else { writeConfig.setValue(HoodieTableConfig.META_FIELDS_MODE, metaFieldsMode.name()); @@ -3920,32 +3933,26 @@ private void validate() { String.format("%s must be positive, but was %d", HoodieTTLConfig.STATS_MAX_PARALLELISM.key(), ttlStatsMaxParallelism)); - // hoodie.meta.fields.mode is an additive opt-in on top of populate.meta.fields=false. Setting - // populate.meta.fields=true together with a non-ALL mode is ambiguous (the mode has no effect - // when all meta fields are already populated) so reject it explicitly rather than silently - // ignore. MetaFieldsMode.fromConfig also throws on unrecognized on-disk values. + // hoodie.meta.fields.mode is the source of truth for meta-column population; the deprecated + // populate.meta.fields boolean is consulted only when the mode is absent. There is therefore + // no ambiguous combination to reject here — MetaFieldsMode.resolve throws on unrecognized + // values. MetaFieldsMode metaFieldsMode = writeConfig.getMetaFieldsMode(); - boolean populateMetaFields = writeConfig.populateMetaFields(); - String rawMode = writeConfig.getStringOrDefault(HoodieTableConfig.META_FIELDS_MODE); - checkArgument(!(populateMetaFields && rawMode != null && !rawMode.isEmpty()), - String.format("%s must be empty when %s=true. Disable populate.meta.fields or clear the mode.", - HoodieTableConfig.META_FIELDS_MODE.key(), - HoodieTableConfig.POPULATE_META_FIELDS.key())); // Selective meta-field modes are CoW-only in this release. MoR log-write path does not yet // respect the mode, which would silently produce log records with null meta columns. boolean isSelective = metaFieldsMode != MetaFieldsMode.ALL && metaFieldsMode != MetaFieldsMode.NONE; checkArgument(!(writeConfig.getTableType() == HoodieTableType.MERGE_ON_READ && isSelective), String.format("%s=%s is currently supported for COPY_ON_WRITE tables only. MoR support is a follow-up. " - + "For MoR either keep %s=true or use NONE mode.", + + "For MoR use %s=ALL or %s=NONE.", HoodieTableConfig.META_FIELDS_MODE.key(), metaFieldsMode, - HoodieTableConfig.POPULATE_META_FIELDS.key())); + HoodieTableConfig.META_FIELDS_MODE.key(), HoodieTableConfig.META_FIELDS_MODE.key())); // Selective meta-field modes are wired only for the Spark writer path in this release. Flink // RowData / Java-client writers ignore the mode and would silently produce NONE-mode output. checkArgument(!(engineType != EngineType.SPARK && isSelective), String.format("%s=%s is currently supported for the Spark writer only. Support for engine=%s is a follow-up. " - + "Either keep %s=true or use NONE mode.", + + "Use %s=ALL or %s=NONE.", HoodieTableConfig.META_FIELDS_MODE.key(), metaFieldsMode, engineType, - HoodieTableConfig.POPULATE_META_FIELDS.key())); + HoodieTableConfig.META_FIELDS_MODE.key(), HoodieTableConfig.META_FIELDS_MODE.key())); } public HoodieWriteConfig build() { diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/config/TestHoodieWriteConfigMetaFieldsMode.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/config/TestHoodieWriteConfigMetaFieldsMode.java index c578489feb26f..0ec45f1e6a254 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/config/TestHoodieWriteConfigMetaFieldsMode.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/config/TestHoodieWriteConfigMetaFieldsMode.java @@ -18,11 +18,14 @@ package org.apache.hudi.config; +import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.model.MetaFieldsMode; import org.apache.hudi.common.table.HoodieTableConfig; import org.junit.jupiter.api.Test; +import java.util.Properties; + import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -40,6 +43,12 @@ private static HoodieWriteConfig.Builder baseBuilder() { return HoodieWriteConfig.newBuilder().withPath("file:///tmp/test_hudi_meta_fields_mode"); } + private static Properties mergeOnReadProps() { + Properties props = new Properties(); + props.setProperty(HoodieTableConfig.TYPE.key(), HoodieTableType.MERGE_ON_READ.name()); + return props; + } + @Test void defaultsToAllMode() { HoodieWriteConfig cfg = baseBuilder().build(); @@ -92,28 +101,28 @@ void commitTimeAndFileNameCombinationBuilds() { } @Test - void rejectsIncompatibleCombination() { - // populate.meta.fields=true together with a selective mode is ambiguous — reject. - HoodieWriteConfig.Builder builder = baseBuilder() + void modeWinsOverLegacyPopulateMetaFields() { + // The two properties no longer compose: an explicit mode is the answer regardless of the + // deprecated boolean, so this combination is accepted rather than rejected. + HoodieWriteConfig cfg = baseBuilder() .withPopulateMetaFields(true) - .withMetaFieldsMode(MetaFieldsMode.COMMIT_TIME_ONLY); - IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, builder::build); - assertTrue(ex.getMessage().contains("hoodie.meta.fields.mode"), - "exception must name the mode property: " + ex.getMessage()); - assertTrue(ex.getMessage().contains("hoodie.populate.meta.fields"), - "exception must name the legacy property too: " + ex.getMessage()); + .withMetaFieldsMode(MetaFieldsMode.COMMIT_TIME_ONLY) + .build(); + assertEquals(MetaFieldsMode.COMMIT_TIME_ONLY, cfg.getMetaFieldsMode()); + assertTrue(cfg.isCommitTimePopulated()); + assertFalse(cfg.isFileNamePopulated()); + // populateMetaFields() is derived from the mode — only ALL reports true. + assertFalse(cfg.populateMetaFields()); } @Test - void allModeWithPopulateFalseIsAlsoRejectedByBuilder() { - // Explicitly setting ALL is a no-op — the builder normalizes it to empty. Passing ALL directly - // is fine; ensuring populateMetaFields agrees is the caller's responsibility (validate() runs - // the cross-check at build time). + void explicitAllModeOverridesLegacyFalse() { HoodieWriteConfig cfg = baseBuilder() - .withPopulateMetaFields(true) + .withPopulateMetaFields(false) .withMetaFieldsMode(MetaFieldsMode.ALL) .build(); assertEquals(MetaFieldsMode.ALL, cfg.getMetaFieldsMode()); + assertTrue(cfg.populateMetaFields()); } @Test @@ -122,9 +131,44 @@ void noneModeWithExplicitBuildIsStillNone() { .withPopulateMetaFields(false) .withMetaFieldsMode(MetaFieldsMode.NONE) .build(); - // NONE is normalized to empty on-disk (implicit from populate=false). assertEquals(MetaFieldsMode.NONE, cfg.getMetaFieldsMode()); assertFalse(cfg.isCommitTimePopulated()); assertFalse(cfg.isFileNamePopulated()); } + + @Test + void legacyBooleanIsUsedWhenModeIsAbsent() { + // Backward compat: tables written before hoodie.meta.fields.mode existed keep their behavior. + HoodieWriteConfig allCfg = baseBuilder().withPopulateMetaFields(true).build(); + assertEquals(MetaFieldsMode.ALL, allCfg.getMetaFieldsMode()); + + HoodieWriteConfig noneCfg = baseBuilder().withPopulateMetaFields(false).build(); + assertEquals(MetaFieldsMode.NONE, noneCfg.getMetaFieldsMode()); + } + + @Test + void rejectsSelectiveModeOnMergeOnRead() { + // Selective modes are CoW-only until the MoR log-write path honors them. + HoodieWriteConfig.Builder builder = baseBuilder() + .withMetaFieldsMode(MetaFieldsMode.COMMIT_TIME_ONLY) + .withProps(mergeOnReadProps()); + IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, builder::build); + assertTrue(ex.getMessage().contains("hoodie.meta.fields.mode"), + "exception must name the mode property: " + ex.getMessage()); + assertTrue(ex.getMessage().contains("COPY_ON_WRITE"), + "exception must explain the CoW-only restriction: " + ex.getMessage()); + } + + @Test + void allowsAllAndNoneOnMergeOnRead() { + // Only the selective modes are restricted — the two legacy-equivalent modes stay available. + assertEquals(MetaFieldsMode.ALL, baseBuilder() + .withMetaFieldsMode(MetaFieldsMode.ALL) + .withProps(mergeOnReadProps()) + .build().getMetaFieldsMode()); + assertEquals(MetaFieldsMode.NONE, baseBuilder() + .withMetaFieldsMode(MetaFieldsMode.NONE) + .withProps(mergeOnReadProps()) + .build().getMetaFieldsMode()); + } } diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkFileWriterFactory.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkFileWriterFactory.java index 762dc027643da..29e10a1c48f1b 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkFileWriterFactory.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkFileWriterFactory.java @@ -56,10 +56,11 @@ public HoodieSparkFileWriterFactory(HoodieStorage storage) { protected HoodieFileWriter newParquetFileWriter( String instantTime, StoragePath path, HoodieConfig config, HoodieSchema schema, TaskContextSupplier taskContextSupplier) throws IOException { - boolean populateMetaFields = config.getBooleanOrDefault(HoodieTableConfig.POPULATE_META_FIELDS); org.apache.hudi.common.model.MetaFieldsMode metaFieldsMode = - org.apache.hudi.common.model.MetaFieldsMode.fromConfig(populateMetaFields, - config.getStringOrDefault(HoodieTableConfig.META_FIELDS_MODE)); + org.apache.hudi.common.model.MetaFieldsMode.resolve( + config.getStringOrDefault(HoodieTableConfig.META_FIELDS_MODE), + config.getBooleanOrDefault(HoodieTableConfig.POPULATE_META_FIELDS)); + boolean populateMetaFields = metaFieldsMode.toLegacyPopulateMetaFields(); Pair injectedConfigs = HoodieParquetConfigInjector.applyConfigInjector(path, storage.getConf(), config); StorageConfiguration storageConfiguration = injectedConfigs.getLeft(); diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/MetaFieldsMode.java b/hudi-common/src/main/java/org/apache/hudi/common/model/MetaFieldsMode.java index 6766a9edf6836..5f1be7d041c83 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/model/MetaFieldsMode.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/MetaFieldsMode.java @@ -37,16 +37,18 @@ * or none of them beyond the two selectable ones. If you need any of the remaining columns, set * {@code hoodie.populate.meta.fields=true}. * - *

Mapping to the legacy {@code hoodie.populate.meta.fields} boolean: + *

This enum is the single source of truth for meta-column population. The legacy boolean + * {@code hoodie.populate.meta.fields} is deprecated and consulted only when + * {@code hoodie.meta.fields.mode} is absent, so that tables written before the mode property + * existed keep their behavior: * *

    - *
  • {@link #ALL} corresponds to {@code populate.meta.fields=true} — today's default.
    • - *
  • Every other value corresponds to {@code populate.meta.fields=false} plus a selective opt-in.
    • + *
  • {@code populate.meta.fields=true} (or absent) → {@link #ALL} — today's default.
    • + *
  • {@code populate.meta.fields=false} → {@link #NONE}.
    • *
* *

On-disk representation: the enum {@link #name()} is persisted in {@code hoodie.properties} - * under the property {@code hoodie.meta.fields.mode}. For backward compatibility, older tables that - * predate this property fall back to {@link #ALL} or {@link #NONE} based on the legacy boolean. + * under the property {@code hoodie.meta.fields.mode}. */ public enum MetaFieldsMode { /** @@ -103,32 +105,48 @@ public boolean isRecordKeyPopulated() { } /** - * Auto-derive a mode from the legacy {@code hoodie.populate.meta.fields} boolean and the raw - * {@code hoodie.meta.fields.mode} property value. Precedence: + * Resolve the effective mode. {@code hoodie.meta.fields.mode} is the source of truth; the + * deprecated {@code hoodie.populate.meta.fields} boolean is a fallback for tables written before + * the mode property existed. Precedence: * *

    - *
  • {@code populateMetaFields=true} → {@link #ALL} (the mode property is ignored).
    • - *
  • {@code populateMetaFields=false} + null/empty mode → {@link #NONE}.
    • - *
  • {@code populateMetaFields=false} + non-empty mode → the parsed enum value.
    • + *
  • non-empty mode → the parsed enum value (the legacy boolean is not consulted).
    • + *
  • null/empty mode + {@code populateMetaFields=false} → {@link #NONE}.
    • + *
  • null/empty mode + {@code populateMetaFields=true} → {@link #ALL}.
    • *
* - *

Throws {@link IllegalArgumentException} when the raw mode value does not match any enum - * value. This includes the pre-enum comma-separated format — callers that upgrade an old table - * must migrate the value through the hudi-cli. + * @param rawMode raw {@code hoodie.meta.fields.mode} value; may be null or empty. + * @param legacyPopulateMetaFields value of the deprecated {@code hoodie.populate.meta.fields}. + * @throws IllegalArgumentException when the raw mode value does not match any enum value. This + * includes the pre-enum comma-separated format — callers that upgrade an old table must + * migrate the value through the hudi-cli. */ - public static MetaFieldsMode fromConfig(boolean populateMetaFields, String rawMode) { - if (populateMetaFields) { - return ALL; - } + public static MetaFieldsMode resolve(String rawMode, boolean legacyPopulateMetaFields) { if (StringUtils.isNullOrEmpty(rawMode)) { - return NONE; + return legacyPopulateMetaFields ? ALL : NONE; } + return parse(rawMode); + } + + /** + * Parse a raw {@code hoodie.meta.fields.mode} value into an enum constant, with a message that + * lists the allowed values. Prefer this over {@link #valueOf(String)} for user-supplied input. + */ + public static MetaFieldsMode parse(String rawMode) { try { return MetaFieldsMode.valueOf(rawMode.trim()); } catch (IllegalArgumentException e) { throw new IllegalArgumentException(String.format( - "Unsupported value '%s' for hoodie.meta.fields.mode. Allowed values: %s, %s, %s, %s.", - rawMode, COMMIT_TIME_ONLY, FILE_NAME_ONLY, COMMIT_TIME_AND_FILE_NAME, NONE), e); + "Unsupported value '%s' for hoodie.meta.fields.mode. Allowed values: %s, %s, %s, %s, %s.", + rawMode, ALL, NONE, COMMIT_TIME_ONLY, FILE_NAME_ONLY, COMMIT_TIME_AND_FILE_NAME), e); } } + + /** + * @return the equivalent value of the deprecated {@code hoodie.populate.meta.fields} boolean, so + * that call sites not yet migrated to this enum keep observing consistent behavior. + */ + public boolean toLegacyPopulateMetaFields() { + return this == ALL; + } } diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java index cef932e49ba3d..d4ab913cbf95e 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java @@ -328,18 +328,28 @@ public static final String getDefaultPayloadClassName() { .noDefaultValue() .withDocumentation("Base path of the dataset that needs to be bootstrapped as a Hudi table"); + /** + * @deprecated since 1.3.0, use {@link #META_FIELDS_MODE} instead. {@code true} maps to + * {@link MetaFieldsMode#ALL} and {@code false} maps to {@link MetaFieldsMode#NONE}. This property + * is still honored for tables written before {@code hoodie.meta.fields.mode} existed, but it is + * consulted only when the mode property is absent. + */ + @Deprecated public static final ConfigProperty POPULATE_META_FIELDS = ConfigProperty .key("hoodie.populate.meta.fields") .defaultValue(true) - .withDocumentation("When enabled, populates all meta fields. When disabled, no meta fields are populated " + .deprecatedAfter("1.2.0") + .withDocumentation("Deprecated — use hoodie.meta.fields.mode instead (true maps to ALL, false maps to NONE). " + + "When enabled, populates all meta fields. When disabled, no meta fields are populated " + "and incremental queries will not be functional. This is only meant to be used for append only/immutable data for batch processing"); public static final ConfigProperty META_FIELDS_MODE = ConfigProperty .key("hoodie.meta.fields.mode") .defaultValue("") .withDocumentation("Which Hudi meta columns are physically populated on disk. Allowed values are " - + "COMMIT_TIME_ONLY, FILE_NAME_ONLY, COMMIT_TIME_AND_FILE_NAME (or NONE / ALL, though those are " - + "derived automatically from hoodie.populate.meta.fields when unset). Set only at table creation, " + + "ALL, NONE, COMMIT_TIME_ONLY, FILE_NAME_ONLY and COMMIT_TIME_AND_FILE_NAME. This supersedes the " + + "deprecated hoodie.populate.meta.fields boolean, which is consulted only when this property is unset " + + "(true maps to ALL, false maps to NONE). Set only at table creation, " + "via the hudi-cli, or during table upgrade — the property is immutable at runtime because it is a " + "physical-storage decision baked into files at write time."); @@ -1240,18 +1250,33 @@ public String getTimelinePath() { /** * @returns true is meta fields need to be populated. else returns false. + * + *

Derived from {@link #getMetaFieldsMode()} so that call sites still written against the + * deprecated boolean observe the same answer as the enum: only {@link MetaFieldsMode#ALL} + * populates every meta column. Selective modes report {@code false} here, which keeps + * key-dependent machinery (bloom filters, record-level index) correctly disabled. */ public boolean populateMetaFields() { + return getMetaFieldsMode().toLegacyPopulateMetaFields(); + } + + /** + * @return the raw, deprecated {@code hoodie.populate.meta.fields} value, used only as the + * fallback when {@link #META_FIELDS_MODE} is absent. Callers should use + * {@link #getMetaFieldsMode()} instead. + */ + private boolean legacyPopulateMetaFields() { return Boolean.parseBoolean(getStringOrDefault(POPULATE_META_FIELDS)); } /** - * @return the {@link MetaFieldsMode} resolved from the on-disk properties. Older tables without - * the mode property fall back to {@link MetaFieldsMode#ALL} or {@link MetaFieldsMode#NONE} based - * on the legacy {@link #POPULATE_META_FIELDS} boolean. + * @return the {@link MetaFieldsMode} resolved from the on-disk properties. {@link #META_FIELDS_MODE} + * is the source of truth; tables written before that property existed fall back to + * {@link MetaFieldsMode#ALL} or {@link MetaFieldsMode#NONE} based on the deprecated + * {@link #POPULATE_META_FIELDS} boolean. */ public MetaFieldsMode getMetaFieldsMode() { - return MetaFieldsMode.fromConfig(populateMetaFields(), getStringOrDefault(META_FIELDS_MODE)); + return MetaFieldsMode.resolve(getStringOrDefault(META_FIELDS_MODE), legacyPopulateMetaFields()); } /** diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java index 71cd0e6b1faa0..4b0e969a4abd8 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java @@ -1167,6 +1167,11 @@ public TableBuilder setBootstrapIndexEnable(Boolean bootstrapIndexEnable) { return this; } + /** + * @deprecated since 1.3.0, use {@link #setMetaFieldsMode(MetaFieldsMode)} instead + * ({@code true} maps to {@link MetaFieldsMode#ALL}, {@code false} to {@link MetaFieldsMode#NONE}). + */ + @Deprecated public TableBuilder setPopulateMetaFields(boolean populateMetaFields) { this.populateMetaFields = populateMetaFields; return this; @@ -1179,15 +1184,15 @@ public TableBuilder setMetaFieldsMode(MetaFieldsMode metaFieldsMode) { /** * Convenience overload that accepts the raw on-disk string (e.g. from properties files). - * Empty or null values leave the mode unset — the caller-provided populateMetaFields boolean - * determines whether the table is ALL or NONE. + * Empty or null values leave the mode unset — the table then resolves to ALL or NONE from the + * deprecated populate.meta.fields boolean. */ public TableBuilder setMetaFieldsModeFromString(String rawMode) { if (rawMode == null || rawMode.trim().isEmpty()) { this.metaFieldsMode = null; return this; } - this.metaFieldsMode = MetaFieldsMode.valueOf(rawMode.trim()); + this.metaFieldsMode = MetaFieldsMode.parse(rawMode); return this; } @@ -1546,27 +1551,12 @@ public Properties build() { if (null != populateMetaFields) { tableConfig.setValue(HoodieTableConfig.POPULATE_META_FIELDS, Boolean.toString(populateMetaFields)); } - // Persist the mode in one place. Rules: - // - Explicit selective mode wins → validate compatibility with populateMetaFields and write. - // - populateMetaFields=false + no explicit mode → NONE (leave property empty for backward compat). - // - populateMetaFields=true + no explicit mode → ALL (implicit; leave property empty). + // hoodie.meta.fields.mode is the source of truth. Persist it verbatim when supplied — it then + // wins over the deprecated populate.meta.fields boolean at read time, so no cross-validation + // between the two is needed. When no mode is supplied the property stays absent and the table + // resolves to ALL / NONE from the legacy boolean, preserving pre-1.3.0 behavior on disk. if (null != metaFieldsMode) { - if (Boolean.TRUE.equals(populateMetaFields) && metaFieldsMode != MetaFieldsMode.ALL) { - throw new IllegalArgumentException(String.format( - "%s=%s is incompatible with %s=true. Set populate.meta.fields=false or use MetaFieldsMode.ALL.", - HoodieTableConfig.META_FIELDS_MODE.key(), metaFieldsMode, - HoodieTableConfig.POPULATE_META_FIELDS.key())); - } - if (Boolean.FALSE.equals(populateMetaFields) && metaFieldsMode == MetaFieldsMode.ALL) { - throw new IllegalArgumentException(String.format( - "%s=ALL is incompatible with %s=false. Set populate.meta.fields=true or pick a selective mode.", - HoodieTableConfig.META_FIELDS_MODE.key(), - HoodieTableConfig.POPULATE_META_FIELDS.key())); - } - // Persist only the selective modes; ALL/NONE are implicit from populate.meta.fields. - if (metaFieldsMode != MetaFieldsMode.ALL && metaFieldsMode != MetaFieldsMode.NONE) { - tableConfig.setValue(HoodieTableConfig.META_FIELDS_MODE, metaFieldsMode.name()); - } + tableConfig.setValue(HoodieTableConfig.META_FIELDS_MODE, metaFieldsMode.name()); } if (null != keyGeneratorClassProp) { KeyGeneratorType type = KeyGeneratorType.fromClassName(keyGeneratorClassProp); diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroFileWriterFactory.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroFileWriterFactory.java index 7c4b85aa58e87..dc7a91986f27e 100644 --- a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroFileWriterFactory.java +++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroFileWriterFactory.java @@ -69,10 +69,11 @@ public HoodieAvroFileWriterFactory(HoodieStorage storage) { protected HoodieFileWriter newParquetFileWriter( String instantTime, StoragePath path, HoodieConfig config, HoodieSchema schema, TaskContextSupplier taskContextSupplier) throws IOException { - boolean populateMetaFields = config.getBooleanOrDefault(HoodieTableConfig.POPULATE_META_FIELDS); org.apache.hudi.common.model.MetaFieldsMode metaFieldsMode = - org.apache.hudi.common.model.MetaFieldsMode.fromConfig(populateMetaFields, - config.getStringOrDefault(HoodieTableConfig.META_FIELDS_MODE)); + org.apache.hudi.common.model.MetaFieldsMode.resolve( + config.getStringOrDefault(HoodieTableConfig.META_FIELDS_MODE), + config.getBooleanOrDefault(HoodieTableConfig.POPULATE_META_FIELDS)); + boolean populateMetaFields = metaFieldsMode.toLegacyPopulateMetaFields(); Pair injectedConfigs = HoodieParquetConfigInjector.applyConfigInjector(path, storage.getConf(), config); StorageConfiguration storageConfiguration = injectedConfigs.getLeft(); diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestHoodieMetaFieldsMode.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestHoodieMetaFieldsMode.java index b11cedf0a57ed..0723dfbf17d0c 100644 --- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestHoodieMetaFieldsMode.java +++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestHoodieMetaFieldsMode.java @@ -108,12 +108,19 @@ void commitTimeAndFileNameMode() { } @Test - void modeIsIgnoredWhenPopulateMetaFieldsIsTrue() { - // populate.meta.fields=true always resolves to ALL — the raw mode on disk is not consulted. - // Writer-side validate() rejects this combination up-front, but the accessor must still report - // ALL semantics defensively. + void modeWinsOverLegacyPopulateMetaFields() { + // hoodie.meta.fields.mode is the source of truth: when it is set, the deprecated + // populate.meta.fields boolean is not consulted, whichever way it points. HoodieTableConfig cfg = configOf(true, MetaFieldsMode.COMMIT_TIME_ONLY.name()); - assertEquals(MetaFieldsMode.ALL, cfg.getMetaFieldsMode()); + assertEquals(MetaFieldsMode.COMMIT_TIME_ONLY, cfg.getMetaFieldsMode()); + assertTrue(cfg.isCommitTimePopulated()); + assertFalse(cfg.isFileNamePopulated()); + // ...and populateMetaFields() is derived from the mode, so legacy call sites agree. + assertFalse(cfg.populateMetaFields()); + + HoodieTableConfig allWithLegacyFalse = configOf(false, MetaFieldsMode.ALL.name()); + assertEquals(MetaFieldsMode.ALL, allWithLegacyFalse.getMetaFieldsMode()); + assertTrue(allWithLegacyFalse.populateMetaFields()); } @Test From 358fbfdd717a4ac1531084fa255932f55e07f8ce Mon Sep 17 00:00:00 2001 From: sivabalan Date: Sat, 25 Jul 2026 20:02:34 -0700 Subject: [PATCH 06/13] fix(writer): always write the new file name in the selective meta-fields path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit HoodieRowCreateHandle#writeRowSelectiveMetaFields gated _hoodie_file_name on shouldPreserveHoodieMetadata, copying the source row's value during clustering. Because clustering replaces the file it reads from, the rewritten records ended up pointing at a file that no longer serves them. The ALL-mode path (writeRow) has always written the new file name unconditionally, even when shouldPreserveHoodieMetadata is set — only _hoodie_commit_time and _hoodie_commit_seqno are preserved there. Match that behavior. _hoodie_commit_time still honors the preserve flag. Reported in review of #19205. --- .../apache/hudi/io/storage/row/HoodieRowCreateHandle.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowCreateHandle.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowCreateHandle.java index 50e4e9d514539..aa6bdbc7ba3ae 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowCreateHandle.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowCreateHandle.java @@ -187,8 +187,11 @@ private void writeRowSelectiveMetaFields(InternalRow row) { ? row.getUTF8String(HoodieRecord.COMMIT_TIME_METADATA_FIELD_ORD) : commitTime; } if (metaFieldsMode.isFileNamePopulated()) { - metaFields[HoodieRecord.FILENAME_META_FIELD_ORD] = shouldPreserveHoodieMetadata - ? row.getUTF8String(HoodieRecord.FILENAME_META_FIELD_ORD) : fileName; + // Always the file being written, never the source row's value — even when + // shouldPreserveHoodieMetadata is set. Preserving it during clustering would leave + // _hoodie_file_name pointing at a file that clustering just replaced. This matches the + // ALL path in writeRow below. + metaFields[HoodieRecord.FILENAME_META_FIELD_ORD] = fileName; } // The remaining meta columns stay null — Parquet stores nulls as definition-level flags // (zero data bytes). From 2b92165f6c12eb2fe4cbe511f5bc0d10420c8b8e Mon Sep 17 00:00:00 2001 From: sivabalan Date: Sun, 26 Jul 2026 18:52:31 -0700 Subject: [PATCH 07/13] test(config): assert meta.fields.mode wins over the deprecated boolean MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TestMetaFieldsMode.populateTrueWithSelectiveModeIsRejected asserted the mutual-exclusion check that was removed when hoodie.meta.fields.mode became the source of truth: populate.meta.fields=true together with a selective mode used to be rejected as ambiguous, and now the mode simply wins. Rewritten as selectiveModeWinsOverLegacyPopulateTrue, which asserts the new contract end to end — the table resolves to COMMIT_TIME_ONLY and only _hoodie_commit_time is populated on disk. Caught by CI on spark3.4 / spark3.5; the hudi-spark module does not test-compile locally on this branch due to an unrelated pre-existing break in TestMergeIntoHoodieTableCommand. --- .../apache/hudi/functional/TestMetaFieldsMode.java | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestMetaFieldsMode.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestMetaFieldsMode.java index f05d3f3804ab3..ba1f494827f2d 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestMetaFieldsMode.java +++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestMetaFieldsMode.java @@ -200,20 +200,18 @@ void commitTimeAndFileNameModePopulatesBoth() { } @Test - void populateTrueWithSelectiveModeIsRejected() { + void selectiveModeWinsOverLegacyPopulateTrue() { + // hoodie.meta.fields.mode is the source of truth: an explicit mode is honored regardless of + // the deprecated boolean, so this combination is no longer ambiguous and is not rejected. Map options = baseOptions(); options.put(HoodieTableConfig.POPULATE_META_FIELDS.key(), "true"); options.put(HoodieTableConfig.META_FIELDS_MODE.key(), MetaFieldsMode.COMMIT_TIME_ONLY.name()); options.put(DataSourceWriteOptions.OPERATION().key(), DataSourceWriteOptions.BULK_INSERT_OPERATION_OPT_VAL()); - Throwable thrown = assertThrows(Throwable.class, () -> - writeRows(Collections.singletonList(RowFactory.create("k1", "p1", "v1")), - simpleSchema(), options, basePath(), SaveMode.Overwrite)); + HoodieTableConfig tc = writeSampleAndGetTableConfig(options, basePath()); - String rootMessage = rootMessageOf(thrown); - assertTrue(rootMessage.contains(HoodieTableConfig.META_FIELDS_MODE.key()) - || rootMessage.contains(HoodieTableConfig.POPULATE_META_FIELDS.key()), - "Expected validation error to name one of the conflicting properties, got: " + rootMessage); + assertEquals(MetaFieldsMode.COMMIT_TIME_ONLY, tc.getMetaFieldsMode()); + assertMetaColumnPopulation(basePath(), MetaFieldsMode.COMMIT_TIME_ONLY); } @Test From 664ff2ec23ae148b6e4802c6069f47522bf779d7 Mon Sep 17 00:00:00 2001 From: sivabalan Date: Sun, 26 Jul 2026 21:50:25 -0700 Subject: [PATCH 08/13] fix(config): derive the persisted legacy boolean from meta.fields.mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses danny0405's P1 on #19205. hoodie.meta.fields.mode and hoodie.populate.meta.fields were persisted independently, so hoodie.properties could record a boolean that contradicts the mode — e.g. a table written COMMIT_TIME_ONLY while still saying populate.meta.fields=true. Pre-1.3.0 readers ignore the mode property entirely and would treat such a table as ALL. For NONE that is actively unsafe: an older incremental reader is allowed to run against all-null commit times and silently returns no rows rather than failing. Whenever an explicit mode is supplied, the legacy boolean is now written from it (ALL -> true, every other mode -> false) rather than from whatever the caller passed: - HoodieTableMetaClient.TableBuilder — the chokepoint for every table-creation path, including HoodieSparkSqlWriter and StreamSync, which pass both setters. - HoodieWriteConfig.Builder#withMetaFieldsMode — so a write config handed to table creation cannot carry a contradicting boolean either. With no mode supplied, only the legacy boolean is recorded, preserving pre-1.3.0 behavior on disk. Tests: TestHoodieTableMetaClient covers all four builder combinations (selective / NONE / ALL / legacy-only); TestMetaFieldsMode asserts the persisted boolean end to end for selective, NONE and ALL; TestHoodieWriteConfigMetaFieldsMode adds an ordering case so a later withPopulateMetaFields cannot widen a selective config. --- .../apache/hudi/config/HoodieWriteConfig.java | 8 ++- .../TestHoodieWriteConfigMetaFieldsMode.java | 15 ++++++ .../common/table/HoodieTableMetaClient.java | 20 +++++--- .../table/TestHoodieTableMetaClient.java | 49 +++++++++++++++++++ .../hudi/functional/TestMetaFieldsMode.java | 35 +++++++++++++ 5 files changed, 118 insertions(+), 9 deletions(-) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java index 1096aa2aa32b7..f9bd275000f37 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java @@ -3630,12 +3630,16 @@ public Builder withPopulateMetaFields(boolean populateMetaFields) { } public Builder withMetaFieldsMode(MetaFieldsMode metaFieldsMode) { - // Leaving the mode unset defers to the deprecated populate.meta.fields boolean; any explicit - // mode (including ALL / NONE) is persisted so it wins over that fallback. + // Leaving the mode unset defers to the deprecated populate.meta.fields boolean. Setting it + // also rewrites that boolean from the mode, so the two can never disagree — a config carrying + // a selective mode alongside populate.meta.fields=true would otherwise create a table whose + // hoodie.properties misleads pre-1.3.0 readers into treating it as ALL. if (metaFieldsMode == null) { writeConfig.setValue(HoodieTableConfig.META_FIELDS_MODE, ""); } else { writeConfig.setValue(HoodieTableConfig.META_FIELDS_MODE, metaFieldsMode.name()); + writeConfig.setValue(HoodieTableConfig.POPULATE_META_FIELDS, + Boolean.toString(metaFieldsMode.toLegacyPopulateMetaFields())); } return this; } diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/config/TestHoodieWriteConfigMetaFieldsMode.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/config/TestHoodieWriteConfigMetaFieldsMode.java index 0ec45f1e6a254..aff79baa3de3e 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/config/TestHoodieWriteConfigMetaFieldsMode.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/config/TestHoodieWriteConfigMetaFieldsMode.java @@ -113,6 +113,21 @@ void modeWinsOverLegacyPopulateMetaFields() { assertFalse(cfg.isFileNamePopulated()); // populateMetaFields() is derived from the mode — only ALL reports true. assertFalse(cfg.populateMetaFields()); + // The raw legacy property is rewritten too, so a config handed to table creation cannot carry + // a boolean that contradicts the mode. + assertEquals("false", cfg.getStringOrDefault(HoodieTableConfig.POPULATE_META_FIELDS)); + } + + @Test + void legacyBooleanSetAfterModeStillResolvesFromMode() { + // Builder call order must not change the outcome: getMetaFieldsMode() reads the mode property + // first, so a later withPopulateMetaFields(...) cannot silently widen a selective table. + HoodieWriteConfig cfg = baseBuilder() + .withMetaFieldsMode(MetaFieldsMode.COMMIT_TIME_ONLY) + .withPopulateMetaFields(true) + .build(); + assertEquals(MetaFieldsMode.COMMIT_TIME_ONLY, cfg.getMetaFieldsMode()); + assertFalse(cfg.populateMetaFields()); } @Test diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java index 4b0e969a4abd8..d5564ce9b2b68 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java @@ -1548,15 +1548,21 @@ public Properties build() { tableConfig.setValue(HoodieTableConfig.CDC_SUPPLEMENTAL_LOGGING_MODE, cdcSupplementalLoggingMode); } } - if (null != populateMetaFields) { - tableConfig.setValue(HoodieTableConfig.POPULATE_META_FIELDS, Boolean.toString(populateMetaFields)); - } - // hoodie.meta.fields.mode is the source of truth. Persist it verbatim when supplied — it then - // wins over the deprecated populate.meta.fields boolean at read time, so no cross-validation - // between the two is needed. When no mode is supplied the property stays absent and the table - // resolves to ALL / NONE from the legacy boolean, preserving pre-1.3.0 behavior on disk. + // hoodie.meta.fields.mode is the source of truth. When it is supplied, hoodie.properties must + // never contradict it: the legacy boolean is written from the mode (ALL -> true, every other + // mode -> false) rather than from whatever the caller passed. Otherwise a table written + // selectively could still record populate.meta.fields=true, and a pre-1.3.0 reader — which + // ignores the mode property entirely — would treat it as ALL. For NONE that is actively + // unsafe: an older incremental reader would be allowed to run against all-null commit times + // and silently return no rows. if (null != metaFieldsMode) { tableConfig.setValue(HoodieTableConfig.META_FIELDS_MODE, metaFieldsMode.name()); + tableConfig.setValue(HoodieTableConfig.POPULATE_META_FIELDS, + Boolean.toString(metaFieldsMode.toLegacyPopulateMetaFields())); + } else if (null != populateMetaFields) { + // No explicit mode: preserve pre-1.3.0 behavior and record only the legacy boolean, which + // resolves to ALL / NONE on read. + tableConfig.setValue(HoodieTableConfig.POPULATE_META_FIELDS, Boolean.toString(populateMetaFields)); } if (null != keyGeneratorClassProp) { KeyGeneratorType type = KeyGeneratorType.fromClassName(keyGeneratorClassProp); diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableMetaClient.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableMetaClient.java index 7e41c5578cc18..1c0ddbe091c69 100644 --- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableMetaClient.java +++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableMetaClient.java @@ -22,6 +22,7 @@ import org.apache.hudi.common.model.HoodieIndexDefinition; import org.apache.hudi.common.model.HoodieIndexMetadata; import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.common.model.MetaFieldsMode; import org.apache.hudi.common.table.timeline.HoodieActiveTimeline; import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.HoodieTimeline; @@ -145,6 +146,54 @@ void testToString() throws IOException { assertNotEquals(metaClient1.toString(), new Object().toString()); } + @Test + void testMetaFieldsModeRewritesLegacyPopulateMetaFields() throws IOException { + // hoodie.properties must never record a legacy boolean that contradicts the mode: a pre-1.3.0 + // reader ignores hoodie.meta.fields.mode entirely and would otherwise treat a selectively + // written table as ALL. For NONE that is unsafe — an old incremental reader would run against + // all-null commit times and silently return no rows. + final String selectivePath = tempDir.toAbsolutePath() + Path.SEPARATOR + "mfm-selective"; + HoodieTableMetaClient selective = HoodieTableMetaClient.newTableBuilder() + .setTableType(HoodieTableType.COPY_ON_WRITE.name()) + .setTableName("mfm-selective") + .setPopulateMetaFields(true) + .setMetaFieldsMode(MetaFieldsMode.COMMIT_TIME_ONLY) + .initTable(this.metaClient.getStorageConf(), selectivePath); + assertEquals(MetaFieldsMode.COMMIT_TIME_ONLY, selective.getTableConfig().getMetaFieldsMode()); + assertFalse(selective.getTableConfig().populateMetaFields(), + "caller-supplied populate.meta.fields=true must be overridden by the mode"); + + final String nonePath = tempDir.toAbsolutePath() + Path.SEPARATOR + "mfm-none"; + HoodieTableMetaClient none = HoodieTableMetaClient.newTableBuilder() + .setTableType(HoodieTableType.COPY_ON_WRITE.name()) + .setTableName("mfm-none") + .setPopulateMetaFields(true) + .setMetaFieldsMode(MetaFieldsMode.NONE) + .initTable(this.metaClient.getStorageConf(), nonePath); + assertEquals(MetaFieldsMode.NONE, none.getTableConfig().getMetaFieldsMode()); + assertFalse(none.getTableConfig().populateMetaFields()); + + final String allPath = tempDir.toAbsolutePath() + Path.SEPARATOR + "mfm-all"; + HoodieTableMetaClient all = HoodieTableMetaClient.newTableBuilder() + .setTableType(HoodieTableType.COPY_ON_WRITE.name()) + .setTableName("mfm-all") + .setPopulateMetaFields(false) + .setMetaFieldsMode(MetaFieldsMode.ALL) + .initTable(this.metaClient.getStorageConf(), allPath); + assertEquals(MetaFieldsMode.ALL, all.getTableConfig().getMetaFieldsMode()); + assertTrue(all.getTableConfig().populateMetaFields()); + + // No explicit mode: pre-1.3.0 behavior preserved, only the legacy boolean is recorded. + final String legacyPath = tempDir.toAbsolutePath() + Path.SEPARATOR + "mfm-legacy"; + HoodieTableMetaClient legacy = HoodieTableMetaClient.newTableBuilder() + .setTableType(HoodieTableType.COPY_ON_WRITE.name()) + .setTableName("mfm-legacy") + .setPopulateMetaFields(false) + .initTable(this.metaClient.getStorageConf(), legacyPath); + assertEquals(MetaFieldsMode.NONE, legacy.getTableConfig().getMetaFieldsMode()); + assertFalse(legacy.getTableConfig().populateMetaFields()); + } + @Test void testTableVersion() throws IOException { final String basePath = tempDir.toAbsolutePath() + Path.SEPARATOR + "t1"; diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestMetaFieldsMode.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestMetaFieldsMode.java index ba1f494827f2d..f1792f6366ba7 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestMetaFieldsMode.java +++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestMetaFieldsMode.java @@ -212,6 +212,41 @@ void selectiveModeWinsOverLegacyPopulateTrue() { assertEquals(MetaFieldsMode.COMMIT_TIME_ONLY, tc.getMetaFieldsMode()); assertMetaColumnPopulation(basePath(), MetaFieldsMode.COMMIT_TIME_ONLY); + // ...and hoodie.properties must not contradict the mode. A pre-1.3.0 reader ignores the mode + // property entirely, so leaving populate.meta.fields=true here would make it treat a + // selectively-written table as ALL. + assertFalse(tc.populateMetaFields(), + "legacy populate.meta.fields must be derived from the mode, not carried through verbatim"); + } + + @Test + void noneModePersistsLegacyBooleanAsFalse() { + // The unsafe case: an old incremental reader that sees populate.meta.fields=true on a NONE + // table would run against all-null commit times and silently return zero rows. + Map options = baseOptions(); + options.put(HoodieTableConfig.POPULATE_META_FIELDS.key(), "true"); + options.put(HoodieTableConfig.META_FIELDS_MODE.key(), MetaFieldsMode.NONE.name()); + options.put(DataSourceWriteOptions.OPERATION().key(), DataSourceWriteOptions.BULK_INSERT_OPERATION_OPT_VAL()); + + HoodieTableConfig tc = writeSampleAndGetTableConfig(options, basePath()); + + assertEquals(MetaFieldsMode.NONE, tc.getMetaFieldsMode()); + assertFalse(tc.populateMetaFields(), + "NONE must persist populate.meta.fields=false so pre-1.3.0 readers do not treat it as ALL"); + } + + @Test + void allModePersistsLegacyBooleanAsTrue() { + Map options = baseOptions(); + options.put(HoodieTableConfig.POPULATE_META_FIELDS.key(), "false"); + options.put(HoodieTableConfig.META_FIELDS_MODE.key(), MetaFieldsMode.ALL.name()); + options.put(DataSourceWriteOptions.OPERATION().key(), DataSourceWriteOptions.BULK_INSERT_OPERATION_OPT_VAL()); + + HoodieTableConfig tc = writeSampleAndGetTableConfig(options, basePath()); + + assertEquals(MetaFieldsMode.ALL, tc.getMetaFieldsMode()); + assertTrue(tc.populateMetaFields(), + "ALL must persist populate.meta.fields=true for pre-1.3.0 readers"); } @Test From c266b8286b4ebd8360785901a24d7fc964c29f72 Mon Sep 17 00:00:00 2001 From: sivabalan Date: Sun, 26 Jul 2026 21:58:18 -0700 Subject: [PATCH 09/13] fix(client): compare the full meta-fields mode in write-client validation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses danny0405's second P1 on #19205. validateAgainstTableProperties compared only the legacy populateMetaFields() booleans. Those collapse every selective mode to false, so a writer resolving to NONE passed the check against a persisted COMMIT_TIME_ONLY table — both sides read false. The writer then produced files with null _hoodie_commit_time while the table still advertised COMMIT_TIME_ONLY, so incremental queries accepted the table and silently missed those rows. Meta-field population is physical, so the writer must agree with what the table records. The check now compares MetaFieldsMode directly and reports both values. Note this is a mismatch check, not a widening check: NONE against COMMIT_TIME_ONLY is a narrowing that the previous guard, and a widening-only guard, would both let through. Reachable from any direct BaseHoodieWriteClient / SparkRDDWriteClient user; the Spark datasource path was already covered by HoodieWriterUtils.validateTableConfig. Tests: TestBaseHoodieWriteClient gains a case reproducing exactly that scenario (verified to fail against the previous boolean-only guard) plus a matching-mode case covering both the default ALL writer and a selective writer. --- .../hudi/client/BaseHoodieWriteClient.java | 17 +++++- .../client/TestBaseHoodieWriteClient.java | 61 +++++++++++++++++++ 2 files changed, 75 insertions(+), 3 deletions(-) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java index a3229c5bb72f9..c938fc28a53b8 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java @@ -45,6 +45,7 @@ import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieTableType; import org.apache.hudi.common.model.HoodieWriteStat; +import org.apache.hudi.common.model.MetaFieldsMode; import org.apache.hudi.common.model.TableServiceType; import org.apache.hudi.common.model.WriteOperationType; import org.apache.hudi.common.schema.HoodieSchema; @@ -1548,9 +1549,19 @@ public void validateAgainstTableProperties(HoodieTableConfig tableConfig, Hoodie // mismatch of table versions. CommonClientUtils.validateTableVersion(tableConfig, writeConfig); - // Once meta fields are disabled, it cant be re-enabled for a given table. - if (!tableConfig.populateMetaFields() && writeConfig.populateMetaFields()) { - throw new HoodieException(HoodieTableConfig.POPULATE_META_FIELDS.key() + " already disabled for the table. Can't be re-enabled back"); + // Meta-field population is physical: the columns are written into the base files, so a writer + // must agree with what the table already records. Compare the full enum rather than the legacy + // booleans — those collapse every selective mode to false, so a writer resolving to NONE would + // pass this check against a COMMIT_TIME_ONLY table and then write null commit times while the + // table still advertises COMMIT_TIME_ONLY, making incremental queries silently miss those rows. + MetaFieldsMode tableMetaFieldsMode = tableConfig.getMetaFieldsMode(); + MetaFieldsMode writeMetaFieldsMode = writeConfig.getMetaFieldsMode(); + if (tableMetaFieldsMode != writeMetaFieldsMode) { + throw new HoodieException(String.format( + "%s mismatch: table is %s but the writer resolved to %s. Meta columns are physical, so the " + + "writer must match the table. Set %s=%s on the writer, or recreate the table to change it.", + HoodieTableConfig.META_FIELDS_MODE.key(), tableMetaFieldsMode, writeMetaFieldsMode, + HoodieTableConfig.META_FIELDS_MODE.key(), tableMetaFieldsMode)); } // Meta fields can be disabled only when either {@code SimpleKeyGenerator}, {@code ComplexKeyGenerator}, diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/TestBaseHoodieWriteClient.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/TestBaseHoodieWriteClient.java index 4396e061a49ed..1880ca150bdbe 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/TestBaseHoodieWriteClient.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/TestBaseHoodieWriteClient.java @@ -24,6 +24,7 @@ import org.apache.hudi.common.engine.HoodieLocalEngineContext; import org.apache.hudi.common.model.HoodieCommitMetadata; import org.apache.hudi.common.model.HoodieTimelineTimeZone; +import org.apache.hudi.common.model.MetaFieldsMode; import org.apache.hudi.common.model.WriteConcurrencyMode; import org.apache.hudi.common.model.WriteOperationType; import org.apache.hudi.common.table.HoodieTableConfig; @@ -43,6 +44,7 @@ import org.apache.hudi.config.HoodieLockConfig; import org.apache.hudi.config.HoodieWriteConfig; import org.apache.hudi.core.transaction.lock.InProcessLockProvider; +import org.apache.hudi.exception.HoodieException; import org.apache.hudi.index.HoodieIndex; import org.apache.hudi.index.HoodieSimpleIndex; import org.apache.hudi.keygen.ComplexAvroKeyGenerator; @@ -74,6 +76,8 @@ import static org.apache.hudi.common.testutils.HoodieTestUtils.getDefaultStorageConf; import static org.apache.hudi.testutils.Assertions.assertComplexKeyGeneratorValidationThrows; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.RETURNS_DEEP_STUBS; import static org.mockito.Mockito.mock; @@ -82,6 +86,63 @@ class TestBaseHoodieWriteClient extends HoodieCommonTestHarness { + private static HoodieTableConfig tableConfigWithMode(MetaFieldsMode mode) { + HoodieTableConfig tableConfig = new HoodieTableConfig(); + tableConfig.setValue(HoodieTableConfig.META_FIELDS_MODE, mode.name()); + tableConfig.setValue(HoodieTableConfig.POPULATE_META_FIELDS, + Boolean.toString(mode.toLegacyPopulateMetaFields())); + tableConfig.setValue(HoodieTableConfig.VERSION, + String.valueOf(HoodieTableVersion.current().versionCode())); + return tableConfig; + } + + private static BaseHoodieWriteClient validatorClient(HoodieWriteConfig writeConfig) { + return new TestWriteClient(writeConfig, mock(HoodieTable.class), Option.empty(), + mock(BaseHoodieTableServiceClient.class)); + } + + @Test + void validateAgainstTablePropertiesRejectsMetaFieldsModeMismatch() throws IOException { + initMetaClient(); + // A writer that resolves to NONE against a COMMIT_TIME_ONLY table: both legacy booleans are + // false, so a boolean-only check passes and the writer goes on to produce null commit times + // while the table still advertises COMMIT_TIME_ONLY. + HoodieWriteConfig noneWriteConfig = HoodieWriteConfig.newBuilder() + .withPath(basePath) + .withPopulateMetaFields(false) + .build(); + assertEquals(MetaFieldsMode.NONE, noneWriteConfig.getMetaFieldsMode()); + + HoodieTableConfig commitTimeOnlyTable = tableConfigWithMode(MetaFieldsMode.COMMIT_TIME_ONLY); + assertFalse(commitTimeOnlyTable.populateMetaFields(), + "precondition: both legacy booleans are false, so only the enum comparison can catch this"); + + HoodieException ex = assertThrows(HoodieException.class, () -> + validatorClient(noneWriteConfig).validateAgainstTableProperties(commitTimeOnlyTable, noneWriteConfig)); + assertTrue(ex.getMessage().contains(HoodieTableConfig.META_FIELDS_MODE.key()), + "error must name the mode property: " + ex.getMessage()); + assertTrue(ex.getMessage().contains("COMMIT_TIME_ONLY") && ex.getMessage().contains("NONE"), + "error must name both modes: " + ex.getMessage()); + } + + @Test + void validateAgainstTablePropertiesAcceptsMatchingMetaFieldsMode() throws IOException { + initMetaClient(); + // Default writer and default table both resolve to ALL — the overwhelmingly common case. + HoodieWriteConfig defaultWriteConfig = HoodieWriteConfig.newBuilder().withPath(basePath).build(); + assertEquals(MetaFieldsMode.ALL, defaultWriteConfig.getMetaFieldsMode()); + validatorClient(defaultWriteConfig) + .validateAgainstTableProperties(tableConfigWithMode(MetaFieldsMode.ALL), defaultWriteConfig); + + // And a selective writer against a table recorded with the same mode. + HoodieWriteConfig selectiveWriteConfig = HoodieWriteConfig.newBuilder() + .withPath(basePath) + .withMetaFieldsMode(MetaFieldsMode.COMMIT_TIME_ONLY) + .build(); + validatorClient(selectiveWriteConfig).validateAgainstTableProperties( + tableConfigWithMode(MetaFieldsMode.COMMIT_TIME_ONLY), selectiveWriteConfig); + } + @Test void startCommitWillRollbackFailedWritesInEagerMode() throws IOException { initMetaClient(); From 7dcd9d1b77f5f2034dae3eb94a096e05fbc444b7 Mon Sep 17 00:00:00 2001 From: sivabalan Date: Sun, 26 Jul 2026 22:12:58 -0700 Subject: [PATCH 10/13] refactor(writer): drop the unused boolean ctor and make mode parsing case-insensitive MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three review comments from danny0405 on #19205: - HoodieSparkParquetWriter's boolean constructor had no callers at all — the only construction site (HoodieSparkFileWriterFactory) already passes a MetaFieldsMode. Removed. - HoodieAvroParquetWriter's boolean constructor is still used by SparkHelpers.scala plus four tests, so it stays, but is now marked @Deprecated pointing at the MetaFieldsMode overload since it cannot express the selective modes. - MetaFieldsMode.parse now upper-cases before valueOf, so a hand-edited hoodie.properties or a write option does not have to match the enum's casing. TestHoodieMetaFieldsMode covers lower, mixed and padded forms. --- .../apache/hudi/io/storage/HoodieSparkParquetWriter.java | 9 --------- .../org/apache/hudi/common/model/MetaFieldsMode.java | 6 +++++- .../hudi/io/storage/hadoop/HoodieAvroParquetWriter.java | 6 ++++++ .../hudi/common/table/TestHoodieMetaFieldsMode.java | 9 ++++++--- 4 files changed, 17 insertions(+), 13 deletions(-) diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkParquetWriter.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkParquetWriter.java index 02e637a52fee9..9480d85f9ea34 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkParquetWriter.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkParquetWriter.java @@ -51,15 +51,6 @@ public class HoodieSparkParquetWriter extends HoodieBaseParquetWriter seqIdGenerator; - public HoodieSparkParquetWriter(StoragePath file, - HoodieRowParquetConfig parquetConfig, - String instantTime, - TaskContextSupplier taskContextSupplier, - boolean populateMetaFields) throws IOException { - this(file, parquetConfig, instantTime, taskContextSupplier, - populateMetaFields ? MetaFieldsMode.ALL : MetaFieldsMode.NONE); - } - public HoodieSparkParquetWriter(StoragePath file, HoodieRowParquetConfig parquetConfig, String instantTime, diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/MetaFieldsMode.java b/hudi-common/src/main/java/org/apache/hudi/common/model/MetaFieldsMode.java index 5f1be7d041c83..c0c2b3c43bac8 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/model/MetaFieldsMode.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/MetaFieldsMode.java @@ -20,6 +20,8 @@ import org.apache.hudi.common.util.StringUtils; +import java.util.Locale; + /** * Which of Hudi's meta columns are physically populated on disk. * @@ -134,7 +136,9 @@ public static MetaFieldsMode resolve(String rawMode, boolean legacyPopulateMetaF */ public static MetaFieldsMode parse(String rawMode) { try { - return MetaFieldsMode.valueOf(rawMode.trim()); + // Case-insensitive: users hand-editing hoodie.properties or passing write options should not + // have to match the enum's casing exactly. + return MetaFieldsMode.valueOf(rawMode.trim().toUpperCase(Locale.ROOT)); } catch (IllegalArgumentException e) { throw new IllegalArgumentException(String.format( "Unsupported value '%s' for hoodie.meta.fields.mode. Allowed values: %s, %s, %s, %s, %s.", diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroParquetWriter.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroParquetWriter.java index c2357a7165374..30a4e579deb75 100644 --- a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroParquetWriter.java +++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroParquetWriter.java @@ -54,6 +54,12 @@ public class HoodieAvroParquetWriter private final MetaFieldsMode metaFieldsMode; private final HoodieAvroWriteSupport writeSupport; + /** + * @deprecated since 1.3.0, use the {@link MetaFieldsMode} overload. Retained for existing callers + * that only distinguish all-or-nothing meta fields ({@code true} maps to {@link MetaFieldsMode#ALL}, + * {@code false} to {@link MetaFieldsMode#NONE}); it cannot express the selective modes. + */ + @Deprecated @SuppressWarnings({"unchecked", "rawtypes"}) public HoodieAvroParquetWriter(StoragePath file, HoodieParquetConfig parquetConfig, diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestHoodieMetaFieldsMode.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestHoodieMetaFieldsMode.java index 0723dfbf17d0c..3242f6ba1eba6 100644 --- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestHoodieMetaFieldsMode.java +++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestHoodieMetaFieldsMode.java @@ -134,9 +134,12 @@ void unknownTokenIsRejected() { } @Test - void modeStringIsCaseSensitiveAndTrimmed() { - // Enum-name form is uppercase-only; whitespace around the value is tolerated. - HoodieTableConfig cfg = configOf(false, " COMMIT_TIME_ONLY "); + void modeStringIsCaseInsensitiveAndTrimmed() { + // Whitespace around the value is tolerated, and casing does not have to match the enum. + assertEquals(MetaFieldsMode.COMMIT_TIME_ONLY, configOf(false, " COMMIT_TIME_ONLY ").getMetaFieldsMode()); + assertEquals(MetaFieldsMode.COMMIT_TIME_ONLY, configOf(false, "commit_time_only").getMetaFieldsMode()); + assertEquals(MetaFieldsMode.COMMIT_TIME_AND_FILE_NAME, configOf(false, " Commit_Time_And_File_Name ").getMetaFieldsMode()); + HoodieTableConfig cfg = configOf(false, "CoMmIt_TiMe_OnLy"); assertEquals(MetaFieldsMode.COMMIT_TIME_ONLY, cfg.getMetaFieldsMode()); } } From 9495687a7813be6470f1609cf05c0f0beaf4a8c6 Mon Sep 17 00:00:00 2001 From: sivabalan Date: Sun, 26 Jul 2026 22:19:04 -0700 Subject: [PATCH 11/13] refactor(config): add a HoodieConfig overload for MetaFieldsMode.resolve MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses danny0405's review comment on #19205: every caller was repeating the same two-line dance — read hoodie.meta.fields.mode, read the deprecated boolean, pass both — which duplicated the property keys and the precedence rule at each site. MetaFieldsMode.resolve(HoodieConfig) now encapsulates that. Both HoodieTableConfig and HoodieWriteConfig extend HoodieConfig, so one overload serves the write config and both writer factories. HoodieTableConfig.getMetaFieldsMode deliberately keeps the two-argument form: its fallback must read the raw property, because populateMetaFields() on that class is itself derived from the mode. Commented in place so the asymmetry is not mistaken for an oversight. --- .../org/apache/hudi/config/HoodieWriteConfig.java | 3 +-- .../io/storage/HoodieSparkFileWriterFactory.java | 4 +--- .../apache/hudi/common/model/MetaFieldsMode.java | 13 +++++++++++++ .../apache/hudi/common/table/HoodieTableConfig.java | 2 ++ .../storage/hadoop/HoodieAvroFileWriterFactory.java | 4 +--- 5 files changed, 18 insertions(+), 8 deletions(-) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java index f9bd275000f37..db7901ee67c6d 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java @@ -1791,8 +1791,7 @@ public boolean populateMetaFields() { * deprecated {@code hoodie.populate.meta.fields} boolean. */ public MetaFieldsMode getMetaFieldsMode() { - return MetaFieldsMode.resolve(getStringOrDefault(HoodieTableConfig.META_FIELDS_MODE), - getBooleanOrDefault(HoodieTableConfig.POPULATE_META_FIELDS)); + return MetaFieldsMode.resolve(this); } /** diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkFileWriterFactory.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkFileWriterFactory.java index 29e10a1c48f1b..94c827dd9362c 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkFileWriterFactory.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkFileWriterFactory.java @@ -57,9 +57,7 @@ protected HoodieFileWriter newParquetFileWriter( String instantTime, StoragePath path, HoodieConfig config, HoodieSchema schema, TaskContextSupplier taskContextSupplier) throws IOException { org.apache.hudi.common.model.MetaFieldsMode metaFieldsMode = - org.apache.hudi.common.model.MetaFieldsMode.resolve( - config.getStringOrDefault(HoodieTableConfig.META_FIELDS_MODE), - config.getBooleanOrDefault(HoodieTableConfig.POPULATE_META_FIELDS)); + org.apache.hudi.common.model.MetaFieldsMode.resolve(config); boolean populateMetaFields = metaFieldsMode.toLegacyPopulateMetaFields(); Pair injectedConfigs = HoodieParquetConfigInjector.applyConfigInjector(path, storage.getConf(), config); diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/MetaFieldsMode.java b/hudi-common/src/main/java/org/apache/hudi/common/model/MetaFieldsMode.java index c0c2b3c43bac8..799bb88a61d56 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/model/MetaFieldsMode.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/MetaFieldsMode.java @@ -18,6 +18,8 @@ package org.apache.hudi.common.model; +import org.apache.hudi.common.config.HoodieConfig; +import org.apache.hudi.common.table.HoodieTableConfig; import org.apache.hudi.common.util.StringUtils; import java.util.Locale; @@ -123,6 +125,17 @@ public boolean isRecordKeyPopulated() { * includes the pre-enum comma-separated format — callers that upgrade an old table must * migrate the value through the hudi-cli. */ + /** + * Resolve the effective mode from any {@link HoodieConfig} that may carry the two properties — + * a table config, a write config, or a bare config built from write options. Preferred over the + * two-argument overload: it keeps the property keys and the precedence rule in one place instead + * of repeating them at every call site. + */ + public static MetaFieldsMode resolve(HoodieConfig config) { + return resolve(config.getStringOrDefault(HoodieTableConfig.META_FIELDS_MODE), + config.getBooleanOrDefault(HoodieTableConfig.POPULATE_META_FIELDS)); + } + public static MetaFieldsMode resolve(String rawMode, boolean legacyPopulateMetaFields) { if (StringUtils.isNullOrEmpty(rawMode)) { return legacyPopulateMetaFields ? ALL : NONE; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java index d4ab913cbf95e..db7ed17ad06d2 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java @@ -1276,6 +1276,8 @@ private boolean legacyPopulateMetaFields() { * {@link #POPULATE_META_FIELDS} boolean. */ public MetaFieldsMode getMetaFieldsMode() { + // Deliberately the two-argument form rather than resolve(this): the fallback must be the *raw* + // property, and populateMetaFields() on this class is itself derived from the mode. return MetaFieldsMode.resolve(getStringOrDefault(META_FIELDS_MODE), legacyPopulateMetaFields()); } diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroFileWriterFactory.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroFileWriterFactory.java index dc7a91986f27e..8fa235f3eaead 100644 --- a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroFileWriterFactory.java +++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroFileWriterFactory.java @@ -70,9 +70,7 @@ protected HoodieFileWriter newParquetFileWriter( String instantTime, StoragePath path, HoodieConfig config, HoodieSchema schema, TaskContextSupplier taskContextSupplier) throws IOException { org.apache.hudi.common.model.MetaFieldsMode metaFieldsMode = - org.apache.hudi.common.model.MetaFieldsMode.resolve( - config.getStringOrDefault(HoodieTableConfig.META_FIELDS_MODE), - config.getBooleanOrDefault(HoodieTableConfig.POPULATE_META_FIELDS)); + org.apache.hudi.common.model.MetaFieldsMode.resolve(config); boolean populateMetaFields = metaFieldsMode.toLegacyPopulateMetaFields(); Pair injectedConfigs = HoodieParquetConfigInjector.applyConfigInjector(path, storage.getConf(), config); From 4134dac622573188ffbcf1b2c24d80fbb2e6c634 Mon Sep 17 00:00:00 2001 From: sivabalan Date: Sun, 26 Jul 2026 22:29:00 -0700 Subject: [PATCH 12/13] feat(upgrade): handle hoodie.meta.fields.mode across the v9/v10 boundary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses danny0405's review comment on #19205: the new table property had no upgrade or downgrade handling. Upgrade (9 -> 10): version 9 tables predate the property and resolve to ALL / NONE from the deprecated populate.meta.fields boolean. The upgrade now records that derived value explicitly, so an upgraded table describes its meta-field layout the same way a freshly created version 10 table does rather than relying on the legacy fallback at every read. Behavior is unchanged either way. Downgrade (10 -> 9): version 9 does not understand the property, so it is dropped — but populate.meta.fields is deliberately left exactly as it stands. ALL and NONE tables therefore round-trip unchanged, since those are precisely the two states the legacy boolean can express. Selective modes cannot be expressed in version 9; the downgrade is still allowed, and logs a warning that the table will behave as NONE to version 9 readers and that incremental queries relying on the selective mode will stop returning rows. Already-written files keep their populated meta columns either way. The mode lookup in the downgrade handler is best-effort: dropping the property is what matters, and some callers drive the change set without a helper. Tests: new TestNineToTenUpgradeHandler covers both derived values and asserts the upgrade never rewrites the legacy boolean; TestTenToNineDowngradeHandler now asserts the mode is deleted while populate.meta.fields is left untouched. --- .../upgrade/NineToTenUpgradeHandler.java | 20 ++++- .../upgrade/TenToNineDowngradeHandler.java | 38 ++++++++- .../upgrade/TestNineToTenUpgradeHandler.java | 85 +++++++++++++++++++ .../TestTenToNineDowngradeHandler.java | 14 ++- 4 files changed, 152 insertions(+), 5 deletions(-) create mode 100644 hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/upgrade/TestNineToTenUpgradeHandler.java diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/NineToTenUpgradeHandler.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/NineToTenUpgradeHandler.java index 97fdea90036fa..ea7224991f512 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/NineToTenUpgradeHandler.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/NineToTenUpgradeHandler.java @@ -18,13 +18,25 @@ package org.apache.hudi.table.upgrade; +import org.apache.hudi.common.config.ConfigProperty; import org.apache.hudi.common.engine.HoodieEngineContext; +import org.apache.hudi.common.model.MetaFieldsMode; +import org.apache.hudi.common.table.HoodieTableConfig; import org.apache.hudi.config.HoodieWriteConfig; +import java.util.Collections; +import java.util.Map; + /** * Version 10 enables native log format by default for new writes. Existing version 9 * inline log files remain readable by version 10 readers, so there is no table metadata * rewrite required for the upgrade. + * + *

Version 10 also introduced {@code hoodie.meta.fields.mode}. Version 9 tables predate it and + * resolve to {@code ALL} / {@code NONE} from the deprecated {@code hoodie.populate.meta.fields} + * boolean. The upgrade records that derived value explicitly so upgraded tables describe their + * meta-field layout the same way newly created version 10 tables do, rather than depending on the + * legacy fallback. Behavior is unchanged either way — this only makes the on-disk state explicit. */ public class NineToTenUpgradeHandler implements UpgradeHandler { @@ -34,6 +46,12 @@ public UpgradeDowngrade.TableConfigChangeSet upgrade( HoodieEngineContext context, String instantTime, SupportsUpgradeDowngrade upgradeDowngradeHelper) { - return new UpgradeDowngrade.TableConfigChangeSet(); + HoodieTableConfig tableConfig = + upgradeDowngradeHelper.getTable(config, context).getMetaClient().getTableConfig(); + // Resolves from the legacy boolean for a version 9 table, since the mode property is absent. + MetaFieldsMode metaFieldsMode = tableConfig.getMetaFieldsMode(); + Map propertiesToUpdate = Collections.singletonMap( + HoodieTableConfig.META_FIELDS_MODE, metaFieldsMode.name()); + return new UpgradeDowngrade.TableConfigChangeSet(propertiesToUpdate, Collections.emptySet()); } } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/TenToNineDowngradeHandler.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/TenToNineDowngradeHandler.java index 393dd6986d822..dbeb53d9b07a9 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/TenToNineDowngradeHandler.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/TenToNineDowngradeHandler.java @@ -18,25 +18,61 @@ package org.apache.hudi.table.upgrade; +import org.apache.hudi.common.config.ConfigProperty; import org.apache.hudi.common.engine.HoodieEngineContext; +import org.apache.hudi.common.model.MetaFieldsMode; import org.apache.hudi.common.table.HoodieTableConfig; import org.apache.hudi.config.HoodieWriteConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.util.Collections; +import java.util.HashSet; +import java.util.Set; /** * Version 10 writes native log files by default. Downgrading to version 9 requires * full compaction of native data/delete logs before the downgrade completes. + * + *

Version 10 also introduced {@code hoodie.meta.fields.mode}. Version 9 does not understand it, + * so the property is dropped here while {@code hoodie.populate.meta.fields} is left exactly as it + * stands — {@code ALL} and {@code NONE} tables round-trip unchanged because those are precisely the + * two states the legacy boolean can express. Selective modes cannot be expressed in version 9, so + * the table degrades to what its legacy boolean says (which is {@code false}, i.e. NONE) and we warn. */ public class TenToNineDowngradeHandler implements DowngradeHandler { + + private static final Logger LOG = LoggerFactory.getLogger(TenToNineDowngradeHandler.class); + @Override public UpgradeDowngrade.TableConfigChangeSet downgrade( HoodieWriteConfig config, HoodieEngineContext context, String instantTime, SupportsUpgradeDowngrade upgradeDowngradeHelper) { + Set propertiesToDelete = new HashSet<>(); + propertiesToDelete.add(HoodieTableConfig.TABLE_STORAGE_LAYOUT); + + // The warning is best-effort: dropping the property is what matters, and the helper is not + // always available (some callers drive the change set directly). + MetaFieldsMode metaFieldsMode = upgradeDowngradeHelper == null + ? MetaFieldsMode.ALL + : upgradeDowngradeHelper.getTable(config, context).getMetaClient().getTableConfig().getMetaFieldsMode(); + if (metaFieldsMode != MetaFieldsMode.ALL && metaFieldsMode != MetaFieldsMode.NONE) { + LOG.warn("Table is using {}={}, which table version 9 cannot express. The property is being " + + "removed and the table will behave as {}=false (no meta columns) to version 9 readers. " + + "Already-written files keep their populated meta columns, but incremental queries that " + + "relied on {} will stop returning rows. Recreate the table if you need that behavior back.", + HoodieTableConfig.META_FIELDS_MODE.key(), metaFieldsMode, + HoodieTableConfig.POPULATE_META_FIELDS.key(), metaFieldsMode); + } + // hoodie.populate.meta.fields is deliberately left untouched: whatever the table recorded before + // the downgrade stays, so ALL and NONE tables are bit-identical afterwards. + propertiesToDelete.add(HoodieTableConfig.META_FIELDS_MODE); + return new UpgradeDowngrade.TableConfigChangeSet( Collections.emptyMap(), - Collections.singleton(HoodieTableConfig.TABLE_STORAGE_LAYOUT)); + propertiesToDelete); } } diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/upgrade/TestNineToTenUpgradeHandler.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/upgrade/TestNineToTenUpgradeHandler.java new file mode 100644 index 0000000000000..b4774cad863fc --- /dev/null +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/upgrade/TestNineToTenUpgradeHandler.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.table.upgrade; + +import org.apache.hudi.common.engine.HoodieEngineContext; +import org.apache.hudi.common.model.MetaFieldsMode; +import org.apache.hudi.common.table.HoodieTableConfig; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.table.HoodieTable; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.RETURNS_DEEP_STUBS; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Version 9 tables predate {@code hoodie.meta.fields.mode}, so the upgrade records the value + * derived from the deprecated {@code hoodie.populate.meta.fields} boolean. This makes an upgraded + * table describe its meta-field layout the same way a freshly created version 10 table does, + * instead of relying on the legacy fallback at every read. + */ +class TestNineToTenUpgradeHandler { + + private static SupportsUpgradeDowngrade helperFor(MetaFieldsMode resolvedMode) { + HoodieTable table = mock(HoodieTable.class, RETURNS_DEEP_STUBS); + HoodieTableMetaClient metaClient = mock(HoodieTableMetaClient.class, RETURNS_DEEP_STUBS); + HoodieTableConfig tableConfig = mock(HoodieTableConfig.class); + when(tableConfig.getMetaFieldsMode()).thenReturn(resolvedMode); + when(metaClient.getTableConfig()).thenReturn(tableConfig); + when(table.getMetaClient()).thenReturn(metaClient); + + SupportsUpgradeDowngrade helper = mock(SupportsUpgradeDowngrade.class); + when(helper.getTable(org.mockito.ArgumentMatchers.any(HoodieWriteConfig.class), + org.mockito.ArgumentMatchers.any(HoodieEngineContext.class))).thenReturn(table); + return helper; + } + + @ParameterizedTest + @CsvSource({"ALL", "NONE"}) + void upgradeRecordsTheModeDerivedFromTheLegacyBoolean(String modeName) { + MetaFieldsMode expected = MetaFieldsMode.valueOf(modeName); + UpgradeDowngrade.TableConfigChangeSet changeSet = new NineToTenUpgradeHandler().upgrade( + mock(HoodieWriteConfig.class), mock(HoodieEngineContext.class), "001", helperFor(expected)); + + assertTrue(changeSet.propertiesToDelete().isEmpty()); + assertEquals(1, changeSet.propertiesToUpdate().size()); + assertEquals(expected.name(), + changeSet.propertiesToUpdate().get(HoodieTableConfig.META_FIELDS_MODE)); + } + + @Test + void upgradeLeavesTheLegacyBooleanAlone() { + // The boolean stays authoritative for any reader that has not learned about the mode yet, and + // the two must agree — so the upgrade only adds the mode, never rewrites populate.meta.fields. + UpgradeDowngrade.TableConfigChangeSet changeSet = new NineToTenUpgradeHandler().upgrade( + mock(HoodieWriteConfig.class), mock(HoodieEngineContext.class), "001", + helperFor(MetaFieldsMode.NONE)); + + assertTrue(changeSet.propertiesToDelete().isEmpty()); + assertEquals(1, changeSet.propertiesToUpdate().size()); + assertTrue(changeSet.propertiesToUpdate().containsKey(HoodieTableConfig.META_FIELDS_MODE)); + } +} diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/upgrade/TestTenToNineDowngradeHandler.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/upgrade/TestTenToNineDowngradeHandler.java index 12faa97369303..b9383f6d2be86 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/upgrade/TestTenToNineDowngradeHandler.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/upgrade/TestTenToNineDowngradeHandler.java @@ -24,18 +24,25 @@ import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; class TestTenToNineDowngradeHandler { @Test - void testDowngradeRemovesStorageLayoutOnly() { + void testDowngradeRemovesStorageLayoutAndMetaFieldsMode() { UpgradeDowngrade.TableConfigChangeSet changeSet = new TenToNineDowngradeHandler().downgrade(null, null, null, null); assertTrue(changeSet.propertiesToUpdate().isEmpty()); - assertEquals(1, changeSet.propertiesToDelete().size()); + assertEquals(2, changeSet.propertiesToDelete().size()); assertTrue(changeSet.propertiesToDelete().contains(HoodieTableConfig.TABLE_STORAGE_LAYOUT)); + // Version 9 does not understand hoodie.meta.fields.mode, so it is dropped... + assertTrue(changeSet.propertiesToDelete().contains(HoodieTableConfig.META_FIELDS_MODE)); + // ...while hoodie.populate.meta.fields is deliberately left in place, so ALL and NONE tables + // round-trip unchanged — those are exactly the two states the legacy boolean can express. + assertFalse(changeSet.propertiesToDelete().contains(HoodieTableConfig.POPULATE_META_FIELDS)); + assertFalse(changeSet.propertiesToUpdate().containsKey(HoodieTableConfig.POPULATE_META_FIELDS)); } @Test @@ -44,7 +51,8 @@ void testTenToNineDowngradeRouteIsSupported() { new UpgradeDowngrade(null, null, null, null) .downgrade(HoodieTableVersion.TEN, HoodieTableVersion.NINE, "001"); - assertEquals(1, changeSet.propertiesToDelete().size()); + assertEquals(2, changeSet.propertiesToDelete().size()); assertTrue(changeSet.propertiesToDelete().contains(HoodieTableConfig.TABLE_STORAGE_LAYOUT)); + assertTrue(changeSet.propertiesToDelete().contains(HoodieTableConfig.META_FIELDS_MODE)); } } From a308a37f515301db26c69ba9a1da4dcd60d28618 Mon Sep 17 00:00:00 2001 From: sivabalan Date: Mon, 27 Jul 2026 01:02:51 -0700 Subject: [PATCH 13/13] fix(client): reject only widening meta-fields modes, not narrowing Comparing the resolved MetaFieldsMode with strict equality rejected a combination that has always been legal: a writer that resolves to NONE against an ALL table. Many callers build a HoodieWriteConfig without restating the table's meta-field settings, so this broke writes across the Spark, Java and spark-client engines. Split the guard in two: - Widening is always rejected. Enabling a meta column now would leave earlier commits without it, and readers cannot tell the two apart. - Any disagreement is rejected when the writer *explicitly* sets hoodie.meta.fields.mode. That still catches an explicit NONE against a COMMIT_TIME_ONLY table, which would write null commit times while the table advertises COMMIT_TIME_ONLY and make incremental queries silently miss those rows. A writer that never mentions the mode is left alone: writing fewer meta columns than the table advertises cannot make a reader believe in data that is absent. Adds MetaFieldsMode.isWiderThan for the asymmetric comparison, and tests for both new branches. Co-Authored-By: Claude Opus 5 (1M context) --- .../hudi/client/BaseHoodieWriteClient.java | 37 +++++++++++++++---- .../client/TestBaseHoodieWriteClient.java | 35 ++++++++++++++++-- .../hudi/common/model/MetaFieldsMode.java | 22 +++++++++++ 3 files changed, 83 insertions(+), 11 deletions(-) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java index c938fc28a53b8..78919746219c1 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java @@ -1549,17 +1549,38 @@ public void validateAgainstTableProperties(HoodieTableConfig tableConfig, Hoodie // mismatch of table versions. CommonClientUtils.validateTableVersion(tableConfig, writeConfig); - // Meta-field population is physical: the columns are written into the base files, so a writer - // must agree with what the table already records. Compare the full enum rather than the legacy - // booleans — those collapse every selective mode to false, so a writer resolving to NONE would - // pass this check against a COMMIT_TIME_ONLY table and then write null commit times while the - // table still advertises COMMIT_TIME_ONLY, making incremental queries silently miss those rows. + // Meta-field population is physical, so a writer must not claim columns the table does not + // have. Compare the full enum rather than the legacy booleans: those collapse every selective + // mode to false, so a writer claiming COMMIT_TIME_ONLY against a NONE table would slip through + // and advertise commit times that were never written. + // + // Two distinct cases, because writers routinely omit meta-field settings entirely: + // + // - Widening is always rejected. Enabling a column now would leave earlier commits without it, + // and readers cannot tell the two apart. + // - Any disagreement is rejected when the writer *explicitly* sets hoodie.meta.fields.mode. + // That covers narrowing too, e.g. an explicit NONE against a COMMIT_TIME_ONLY table, which + // would write null commit times while the table still advertises COMMIT_TIME_ONLY and make + // incremental queries silently miss those rows. + // + // A writer that never mentions the mode is left alone: resolving to NONE against an ALL table + // is long-standing behavior for callers that build a write config without restating the table's + // settings, and writing fewer meta columns cannot make a reader believe in absent data. MetaFieldsMode tableMetaFieldsMode = tableConfig.getMetaFieldsMode(); MetaFieldsMode writeMetaFieldsMode = writeConfig.getMetaFieldsMode(); - if (tableMetaFieldsMode != writeMetaFieldsMode) { + boolean writerStatedMode = writeConfig.contains(HoodieTableConfig.META_FIELDS_MODE) + && !StringUtils.isNullOrEmpty(writeConfig.getString(HoodieTableConfig.META_FIELDS_MODE)); + if (writeMetaFieldsMode.isWiderThan(tableMetaFieldsMode)) { throw new HoodieException(String.format( - "%s mismatch: table is %s but the writer resolved to %s. Meta columns are physical, so the " - + "writer must match the table. Set %s=%s on the writer, or recreate the table to change it.", + "%s cannot be widened for an existing table: table is %s but the writer requests %s. Meta " + + "columns are physical, so enabling one now would leave earlier commits without it. " + + "Set %s=%s on the writer, or recreate the table to change it.", + HoodieTableConfig.META_FIELDS_MODE.key(), tableMetaFieldsMode, writeMetaFieldsMode, + HoodieTableConfig.META_FIELDS_MODE.key(), tableMetaFieldsMode)); + } else if (writerStatedMode && writeMetaFieldsMode != tableMetaFieldsMode) { + throw new HoodieException(String.format( + "%s mismatch: table is %s but the writer explicitly requests %s. Meta columns are physical, " + + "so the writer must match the table. Set %s=%s on the writer, or recreate the table.", HoodieTableConfig.META_FIELDS_MODE.key(), tableMetaFieldsMode, writeMetaFieldsMode, HoodieTableConfig.META_FIELDS_MODE.key(), tableMetaFieldsMode)); } diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/TestBaseHoodieWriteClient.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/TestBaseHoodieWriteClient.java index 1880ca150bdbe..c318f5286a5e4 100644 --- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/TestBaseHoodieWriteClient.java +++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/TestBaseHoodieWriteClient.java @@ -104,12 +104,12 @@ private static HoodieTableConfig tableConfigWithMode(MetaFieldsMode mode) { @Test void validateAgainstTablePropertiesRejectsMetaFieldsModeMismatch() throws IOException { initMetaClient(); - // A writer that resolves to NONE against a COMMIT_TIME_ONLY table: both legacy booleans are - // false, so a boolean-only check passes and the writer goes on to produce null commit times + // A writer that explicitly asks for NONE against a COMMIT_TIME_ONLY table: both legacy booleans + // are false, so a boolean-only check passes and the writer goes on to produce null commit times // while the table still advertises COMMIT_TIME_ONLY. HoodieWriteConfig noneWriteConfig = HoodieWriteConfig.newBuilder() .withPath(basePath) - .withPopulateMetaFields(false) + .withMetaFieldsMode(MetaFieldsMode.NONE) .build(); assertEquals(MetaFieldsMode.NONE, noneWriteConfig.getMetaFieldsMode()); @@ -125,6 +125,35 @@ void validateAgainstTablePropertiesRejectsMetaFieldsModeMismatch() throws IOExce "error must name both modes: " + ex.getMessage()); } + @Test + void validateAgainstTablePropertiesAllowsUnstatedWriterToNarrow() throws IOException { + initMetaClient(); + // Long-standing behavior: a writer that sets only populate.meta.fields=false (never naming a + // mode) resolves to NONE, and must still be able to write to an ALL table. Many callers build + // a write config without restating the table's meta-field settings. + HoodieWriteConfig unstated = HoodieWriteConfig.newBuilder() + .withPath(basePath) + .withPopulateMetaFields(false) + .build(); + assertEquals(MetaFieldsMode.NONE, unstated.getMetaFieldsMode()); + validatorClient(unstated) + .validateAgainstTableProperties(tableConfigWithMode(MetaFieldsMode.ALL), unstated); + } + + @Test + void validateAgainstTablePropertiesRejectsWideningEvenWhenUnstated() throws IOException { + initMetaClient(); + // Widening is rejected regardless of whether the writer named a mode: a default writer resolves + // to ALL, which would claim meta columns a NONE table never wrote. + HoodieWriteConfig defaultWriteConfig = HoodieWriteConfig.newBuilder().withPath(basePath).build(); + assertEquals(MetaFieldsMode.ALL, defaultWriteConfig.getMetaFieldsMode()); + + HoodieException ex = assertThrows(HoodieException.class, () -> + validatorClient(defaultWriteConfig) + .validateAgainstTableProperties(tableConfigWithMode(MetaFieldsMode.NONE), defaultWriteConfig)); + assertTrue(ex.getMessage().contains("cannot be widened"), ex.getMessage()); + } + @Test void validateAgainstTablePropertiesAcceptsMatchingMetaFieldsMode() throws IOException { initMetaClient(); diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/MetaFieldsMode.java b/hudi-common/src/main/java/org/apache/hudi/common/model/MetaFieldsMode.java index 799bb88a61d56..6467337f571e6 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/model/MetaFieldsMode.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/model/MetaFieldsMode.java @@ -166,4 +166,26 @@ public static MetaFieldsMode parse(String rawMode) { public boolean toLegacyPopulateMetaFields() { return this == ALL; } + + /** + * @return true when this mode populates at least one meta column that {@code other} does not. + * + *

Meta-field population is a physical-storage decision baked into files at write time, so it + * can never be widened for an existing table: earlier commits would be missing columns that later + * commits have, and readers cannot tell the two apart. Every transition that adds a column is + * therefore rejected — {@code NONE -> COMMIT_TIME_ONLY} and + * {@code FILE_NAME_ONLY -> COMMIT_TIME_AND_FILE_NAME} just as much as {@code NONE -> ALL}. + * + *

Narrowing is not flagged here: writing fewer meta columns than the table advertises cannot + * make a reader believe in data that is absent, and it is long-standing behavior for a writer to + * resolve to {@link #NONE} against an {@link #ALL} table without restating its settings. + */ + public boolean isWiderThan(MetaFieldsMode other) { + if (other == null) { + return false; + } + return (commitTimePopulated && !other.commitTimePopulated) + || (fileNamePopulated && !other.fileNamePopulated) + || (isRecordKeyPopulated() && !other.isRecordKeyPopulated()); + } }