fix(config): resolve meta-fields mode at sites that still read the deprecated boolean

hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java

@@ -45,6 +45,7 @@
 import org.apache.hudi.common.model.HoodieRecord;
 import org.apache.hudi.common.model.HoodieTableType;
 import org.apache.hudi.common.model.HoodieWriteStat;
+import org.apache.hudi.common.model.MetaFieldsMode;
 import org.apache.hudi.common.model.TableServiceType;
 import org.apache.hudi.common.model.WriteOperationType;
 import org.apache.hudi.common.schema.HoodieSchema;
@@ -1548,9 +1549,40 @@ public void validateAgainstTableProperties(HoodieTableConfig tableConfig, Hoodie
     // mismatch of table versions.
     CommonClientUtils.validateTableVersion(tableConfig, writeConfig);
 
-    // Once meta fields are disabled, it cant be re-enabled for a given table.
-    if (!tableConfig.populateMetaFields() && writeConfig.populateMetaFields()) {
-      throw new HoodieException(HoodieTableConfig.POPULATE_META_FIELDS.key() + " already disabled for the table. Can't be re-enabled back");
+    // Meta-field population is physical, so a writer must not claim columns the table does not
+    // have. Compare the full enum rather than the legacy booleans: those collapse every selective
+    // mode to false, so a writer claiming COMMIT_TIME_ONLY against a NONE table would slip through
+    // and advertise commit times that were never written.
+    //
+    // Two distinct cases, because writers routinely omit meta-field settings entirely:
+    //
+    //  - Widening is always rejected. Enabling a column now would leave earlier commits without it,
+    //    and readers cannot tell the two apart.
+    //  - Any disagreement is rejected when the writer *explicitly* sets hoodie.meta.fields.mode.
+    //    That covers narrowing too, e.g. an explicit NONE against a COMMIT_TIME_ONLY table, which
+    //    would write null commit times while the table still advertises COMMIT_TIME_ONLY and make
+    //    incremental queries silently miss those rows.
+    //
+    // A writer that never mentions the mode is left alone: resolving to NONE against an ALL table
+    // is long-standing behavior for callers that build a write config without restating the table's
+    // settings, and writing fewer meta columns cannot make a reader believe in absent data.
+    MetaFieldsMode tableMetaFieldsMode = tableConfig.getMetaFieldsMode();
+    MetaFieldsMode writeMetaFieldsMode = writeConfig.getMetaFieldsMode();
+    boolean writerStatedMode = writeConfig.contains(HoodieTableConfig.META_FIELDS_MODE)
+        && !StringUtils.isNullOrEmpty(writeConfig.getString(HoodieTableConfig.META_FIELDS_MODE));
+    if (writeMetaFieldsMode.isWiderThan(tableMetaFieldsMode)) {
+      throw new HoodieException(String.format(
+          "%s cannot be widened for an existing table: table is %s but the writer requests %s. Meta "
+              + "columns are physical, so enabling one now would leave earlier commits without it. "
+              + "Set %s=%s on the writer, or recreate the table to change it.",
+          HoodieTableConfig.META_FIELDS_MODE.key(), tableMetaFieldsMode, writeMetaFieldsMode,
+          HoodieTableConfig.META_FIELDS_MODE.key(), tableMetaFieldsMode));
+    } else if (writerStatedMode && writeMetaFieldsMode != tableMetaFieldsMode) {
+      throw new HoodieException(String.format(
+          "%s mismatch: table is %s but the writer explicitly requests %s. Meta columns are physical, "
+              + "so the writer must match the table. Set %s=%s on the writer, or recreate the table.",
+          HoodieTableConfig.META_FIELDS_MODE.key(), tableMetaFieldsMode, writeMetaFieldsMode,
+          HoodieTableConfig.META_FIELDS_MODE.key(), tableMetaFieldsMode));
     }
 
     // Meta fields can be disabled only when either {@code SimpleKeyGenerator}, {@code ComplexKeyGenerator},

hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java

@@ -50,6 +50,7 @@
 import org.apache.hudi.common.model.HoodieRecordMerger;
 import org.apache.hudi.common.model.HoodieRecordPayload;
 import org.apache.hudi.common.model.HoodieTableType;
+import org.apache.hudi.common.model.MetaFieldsMode;
 import org.apache.hudi.common.model.WriteConcurrencyMode;
 import org.apache.hudi.common.model.WriteOperationType;
 import org.apache.hudi.common.table.HoodieTableConfig;
@@ -1772,8 +1773,39 @@ public int getSmallFileGroupCandidatesLimit() {
     return getInt(MERGE_SMALL_FILE_GROUP_CANDIDATES_LIMIT);
   }
 
+  /**
+   * @return true when every meta column is populated.
+   *
+   * <p>Derived from {@link #getMetaFieldsMode()} so that call sites still written against the
+   * deprecated {@code hoodie.populate.meta.fields} boolean observe the same answer as the enum:
+   * only {@link MetaFieldsMode#ALL} populates every meta column.
+   */
   public boolean populateMetaFields() {
-    return getBooleanOrDefault(HoodieTableConfig.POPULATE_META_FIELDS);
+    return getMetaFieldsMode().toLegacyPopulateMetaFields();
+  }
+
+  /**
+   * @return the {@link MetaFieldsMode} resolved from the write config.
+   * {@code hoodie.meta.fields.mode} is the source of truth; configs written before that property
+   * existed fall back to {@link MetaFieldsMode#ALL} or {@link MetaFieldsMode#NONE} based on the
+   * deprecated {@code hoodie.populate.meta.fields} boolean.
+   */
+  public MetaFieldsMode getMetaFieldsMode() {
+    return MetaFieldsMode.resolve(this);
+  }
+
+  /**
+   * @return true when {@code _hoodie_commit_time} is physically populated on every row.
+   */
+  public boolean isCommitTimePopulated() {
+    return getMetaFieldsMode().isCommitTimePopulated();
+  }
+
+  /**
+   * @return true when {@code _hoodie_file_name} is physically populated on every row.
+   */
+  public boolean isFileNamePopulated() {
+    return getMetaFieldsMode().isFileNamePopulated();
   }
 
   /**
@@ -3586,11 +3618,31 @@ public Builder withCanIgnorePostCommitFailures(boolean canIgnorePostCommitFailur
       return this;
     }
 
+    /**
+     * @deprecated since 1.3.0, use {@link #withMetaFieldsMode(MetaFieldsMode)} instead
+     * ({@code true} maps to {@link MetaFieldsMode#ALL}, {@code false} to {@link MetaFieldsMode#NONE}).
+     */
+    @Deprecated
     public Builder withPopulateMetaFields(boolean populateMetaFields) {
       writeConfig.setValue(HoodieTableConfig.POPULATE_META_FIELDS, Boolean.toString(populateMetaFields));
       return this;
     }
 
+    public Builder withMetaFieldsMode(MetaFieldsMode metaFieldsMode) {
+      // Leaving the mode unset defers to the deprecated populate.meta.fields boolean. Setting it
+      // also rewrites that boolean from the mode, so the two can never disagree — a config carrying
+      // a selective mode alongside populate.meta.fields=true would otherwise create a table whose
+      // hoodie.properties misleads pre-1.3.0 readers into treating it as ALL.
+      if (metaFieldsMode == null) {
+        writeConfig.setValue(HoodieTableConfig.META_FIELDS_MODE, "");
+      } else {
+        writeConfig.setValue(HoodieTableConfig.META_FIELDS_MODE, metaFieldsMode.name());
+        writeConfig.setValue(HoodieTableConfig.POPULATE_META_FIELDS,
+            Boolean.toString(metaFieldsMode.toLegacyPopulateMetaFields()));
+      }
+      return this;
+    }
+
     public Builder withAllowOperationMetadataField(boolean allowOperationMetadataField) {
       writeConfig.setValue(ALLOW_OPERATION_METADATA_FIELD, Boolean.toString(allowOperationMetadataField));
       return this;
@@ -3883,6 +3935,27 @@ private void validate() {
       checkArgument(ttlStatsMaxParallelism > 0,
           String.format("%s must be positive, but was %d",
               HoodieTTLConfig.STATS_MAX_PARALLELISM.key(), ttlStatsMaxParallelism));
+
+      // hoodie.meta.fields.mode is the source of truth for meta-column population; the deprecated
+      // populate.meta.fields boolean is consulted only when the mode is absent. There is therefore
+      // no ambiguous combination to reject here — MetaFieldsMode.resolve throws on unrecognized
+      // values.
+      MetaFieldsMode metaFieldsMode = writeConfig.getMetaFieldsMode();
+      // Selective meta-field modes are CoW-only in this release. MoR log-write path does not yet
+      // respect the mode, which would silently produce log records with null meta columns.
+      boolean isSelective = metaFieldsMode != MetaFieldsMode.ALL && metaFieldsMode != MetaFieldsMode.NONE;
+      checkArgument(!(writeConfig.getTableType() == HoodieTableType.MERGE_ON_READ && isSelective),
+          String.format("%s=%s is currently supported for COPY_ON_WRITE tables only. MoR support is a follow-up. "
+                  + "For MoR use %s=ALL or %s=NONE.",
+              HoodieTableConfig.META_FIELDS_MODE.key(), metaFieldsMode,
+              HoodieTableConfig.META_FIELDS_MODE.key(), HoodieTableConfig.META_FIELDS_MODE.key()));
+      // Selective meta-field modes are wired only for the Spark writer path in this release. Flink
+      // RowData / Java-client writers ignore the mode and would silently produce NONE-mode output.
+      checkArgument(!(engineType != EngineType.SPARK && isSelective),
+          String.format("%s=%s is currently supported for the Spark writer only. Support for engine=%s is a follow-up. "
+                  + "Use %s=ALL or %s=NONE.",
+              HoodieTableConfig.META_FIELDS_MODE.key(), metaFieldsMode, engineType,
+              HoodieTableConfig.META_FIELDS_MODE.key(), HoodieTableConfig.META_FIELDS_MODE.key()));
     }
 
     public HoodieWriteConfig build() {

hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/index/partitionstats/PartitionStatsIndexer.java

@@ -28,6 +28,7 @@
 import org.apache.hudi.common.model.HoodieRecord;
 import org.apache.hudi.common.model.HoodieReplaceCommitMetadata;
 import org.apache.hudi.common.model.HoodieWriteStat;
+import org.apache.hudi.common.model.MetaFieldsMode;
 import org.apache.hudi.common.model.WriteOperationType;
 import org.apache.hudi.common.schema.HoodieSchema;
 import org.apache.hudi.common.schema.HoodieSchemaUtils;
@@ -141,7 +142,10 @@ public static HoodieData<HoodieRecord> convertMetadataToPartitionStatsRecords(Ho
                       ? Option.empty()
                       : Option.of(HoodieSchema.parse(writerSchemaStr)));
       HoodieTableConfig tableConfig = dataMetaClient.getTableConfig();
-      Option<HoodieSchema> tableSchema = writerSchema.map(schema -> tableConfig.populateMetaFields() ? HoodieSchemaUtils.addMetadataFields(schema) : schema);
+      // Selective meta-fields modes write the meta columns as physical nullable columns, so they
+      // belong in the table schema whenever the mode populates any of them.
+      Option<HoodieSchema> tableSchema = writerSchema.map(schema ->
+          tableConfig.getMetaFieldsMode() != MetaFieldsMode.NONE ? HoodieSchemaUtils.addMetadataFields(schema) : schema);
 
       if (tableSchema.isEmpty()) {
         return engineContext.emptyHoodieData();

hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/NineToTenUpgradeHandler.java

@@ -18,13 +18,25 @@
 
 package org.apache.hudi.table.upgrade;
 
+import org.apache.hudi.common.config.ConfigProperty;
 import org.apache.hudi.common.engine.HoodieEngineContext;
+import org.apache.hudi.common.model.MetaFieldsMode;
+import org.apache.hudi.common.table.HoodieTableConfig;
 import org.apache.hudi.config.HoodieWriteConfig;
 
+import java.util.Collections;
+import java.util.Map;
+
 /**
  * Version 10 enables native log format by default for new writes. Existing version 9
  * inline log files remain readable by version 10 readers, so there is no table metadata
  * rewrite required for the upgrade.
+ *
+ * <p>Version 10 also introduced {@code hoodie.meta.fields.mode}. Version 9 tables predate it and
+ * resolve to {@code ALL} / {@code NONE} from the deprecated {@code hoodie.populate.meta.fields}
+ * boolean. The upgrade records that derived value explicitly so upgraded tables describe their
+ * meta-field layout the same way newly created version 10 tables do, rather than depending on the
+ * legacy fallback. Behavior is unchanged either way — this only makes the on-disk state explicit.
  */
 public class NineToTenUpgradeHandler implements UpgradeHandler {
 
@@ -34,6 +46,12 @@ public UpgradeDowngrade.TableConfigChangeSet upgrade(
       HoodieEngineContext context,
       String instantTime,
       SupportsUpgradeDowngrade upgradeDowngradeHelper) {
-    return new UpgradeDowngrade.TableConfigChangeSet();
+    HoodieTableConfig tableConfig =
+        upgradeDowngradeHelper.getTable(config, context).getMetaClient().getTableConfig();
+    // Resolves from the legacy boolean for a version 9 table, since the mode property is absent.
+    MetaFieldsMode metaFieldsMode = tableConfig.getMetaFieldsMode();
+    Map<ConfigProperty, String> propertiesToUpdate = Collections.singletonMap(
+        HoodieTableConfig.META_FIELDS_MODE, metaFieldsMode.name());
+    return new UpgradeDowngrade.TableConfigChangeSet(propertiesToUpdate, Collections.emptySet());
   }
 }

hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/TenToNineDowngradeHandler.java

@@ -18,25 +18,61 @@
 
 package org.apache.hudi.table.upgrade;
 
+import org.apache.hudi.common.config.ConfigProperty;
 import org.apache.hudi.common.engine.HoodieEngineContext;
+import org.apache.hudi.common.model.MetaFieldsMode;
 import org.apache.hudi.common.table.HoodieTableConfig;
 import org.apache.hudi.config.HoodieWriteConfig;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
 
 /**
  * Version 10 writes native log files by default. Downgrading to version 9 requires
  * full compaction of native data/delete logs before the downgrade completes.
+ *
+ * <p>Version 10 also introduced {@code hoodie.meta.fields.mode}. Version 9 does not understand it,
+ * so the property is dropped here while {@code hoodie.populate.meta.fields} is left exactly as it
+ * stands — {@code ALL} and {@code NONE} tables round-trip unchanged because those are precisely the
+ * two states the legacy boolean can express. Selective modes cannot be expressed in version 9, so
+ * the table degrades to what its legacy boolean says (which is {@code false}, i.e. NONE) and we warn.
  */
 public class TenToNineDowngradeHandler implements DowngradeHandler {
+
+  private static final Logger LOG = LoggerFactory.getLogger(TenToNineDowngradeHandler.class);
+
   @Override
   public UpgradeDowngrade.TableConfigChangeSet downgrade(
       HoodieWriteConfig config,
       HoodieEngineContext context,
       String instantTime,
       SupportsUpgradeDowngrade upgradeDowngradeHelper) {
+    Set<ConfigProperty> propertiesToDelete = new HashSet<>();
+    propertiesToDelete.add(HoodieTableConfig.TABLE_STORAGE_LAYOUT);
+
+    // The warning is best-effort: dropping the property is what matters, and the helper is not
+    // always available (some callers drive the change set directly).
+    MetaFieldsMode metaFieldsMode = upgradeDowngradeHelper == null
+        ? MetaFieldsMode.ALL
+        : upgradeDowngradeHelper.getTable(config, context).getMetaClient().getTableConfig().getMetaFieldsMode();
+    if (metaFieldsMode != MetaFieldsMode.ALL && metaFieldsMode != MetaFieldsMode.NONE) {
+      LOG.warn("Table is using {}={}, which table version 9 cannot express. The property is being "
+              + "removed and the table will behave as {}=false (no meta columns) to version 9 readers. "
+              + "Already-written files keep their populated meta columns, but incremental queries that "
+              + "relied on {} will stop returning rows. Recreate the table if you need that behavior back.",
+          HoodieTableConfig.META_FIELDS_MODE.key(), metaFieldsMode,
+          HoodieTableConfig.POPULATE_META_FIELDS.key(), metaFieldsMode);
+    }
+    // hoodie.populate.meta.fields is deliberately left untouched: whatever the table recorded before
+    // the downgrade stays, so ALL and NONE tables are bit-identical afterwards.
+    propertiesToDelete.add(HoodieTableConfig.META_FIELDS_MODE);
+
     return new UpgradeDowngrade.TableConfigChangeSet(
         Collections.emptyMap(),
-        Collections.singleton(HoodieTableConfig.TABLE_STORAGE_LAYOUT));
+        propertiesToDelete);
   }
 }