Skip to content

API, Core: Implement v4 file-level tight_stats #18598

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
anoopj wants to merge 7 commits into
base: main
Choose a base branch
from
+180 −171
Open

API, Core: Implement v4 file-level tight_stats #18598

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
69aacf9
Core: Implement v4 file-level tight_stats
anoopj Oct 21, 2026
888d1d1
Adjust offsets
anoopj Oct 21, 2026
038ad2d
Add to contentfile
anoopj Oct 22, 2026
1cd9239
updte
anoopj Oct 22, 2026
0ccc4fb
Tweak again
anoopj Oct 22, 2026
8bb88c2
Apply spotless, fix version
anoopj Oct 22, 2026
b83e1df
Rebase and fix
anoopj Oct 22, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions api/src/main/java/org/apache/iceberg/ContentFile.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,11 @@ default String location() {
/** Returns if collected, map from column ID to value upper bounds, null otherwise. */
Map<Integer, ByteBuffer> upperBounds();

/** Returns whether the file's stats are tight, or null if unknown. */
default Boolean tightStats() {

@nastra nastra Oct 22, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

what's the reason for adding v4-specific things to v1-v3 APIs?

@anoopj anoopj Oct 22, 2026

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ContentFile is still the engine-facing API in v4 as well, since TrackedFile family is internal. So don't we need a way to expose this during reads and writes?

return null;
}

/**
* Returns map from column ID to the total uncompressed size in memory in bytes of non-null values
* if collected, null otherwise.
Expand Down
5 changes: 5 additions & 0 deletions core/src/main/java/org/apache/iceberg/Delegates.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -247,6 +247,11 @@ public Map<Integer, ByteBuffer> upperBounds() {
return wrapped.upperBounds();
}

@Override
public Boolean tightStats() {
return wrapped.tightStats();
}

@Override
public ByteBuffer keyMetadata() {
return wrapped.keyMetadata();
Expand Down
11 changes: 8 additions & 3 deletions core/src/main/java/org/apache/iceberg/FieldStats.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,10 +38,15 @@ public interface FieldStats<T> {
T upperBound();

/**
* Whether {@link #lowerBound()} and {@link #upperBound()} are equal to the min and max values for
* the column.
* Returns whether the bounds are known to be exact.
*
* @deprecated since 1.13.0, will be removed in 1.14.0; use {@link ContentFile#tightStats()}
* instead.
*/
boolean tightBounds();
@Deprecated
default boolean tightBounds() {
return false;
}

/** Whether a value count is tracked for this field. */
boolean hasValueCount();
Expand Down
12 changes: 0 additions & 12 deletions core/src/main/java/org/apache/iceberg/FieldStatsStruct.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,6 @@ class FieldStatsStruct<T> implements FieldStats<T>, StructLike, Serializable {

private Object lowerBound = null;
private Object upperBound = null;
private boolean tightBounds = false;
private Long valueCount = null;
private Long nullValueCount = null;
private Long nanValueCount = null;
Expand All @@ -53,15 +52,13 @@ class FieldStatsStruct<T> implements FieldStats<T>, StructLike, Serializable {
Types.StructType struct,
T lowerBound,
T upperBound,
boolean tightBounds,
long valueCount,
long nullValueCount,
long nanValueCount,
long totalBytes) {
this(struct);
setLowerBound(lowerBound);
setUpperBound(upperBound);
this.tightBounds = tightBounds;
this.valueCount = valueCount;
this.nullValueCount = nullValueCount;
this.nanValueCount = nanValueCount;
Expand All @@ -72,7 +69,6 @@ private FieldStatsStruct(FieldStatsStruct<T> toCopy) {
this(toCopy.struct);
this.lowerBound = copyBound(toCopy.lowerBound);
this.upperBound = copyBound(toCopy.upperBound);
this.tightBounds = toCopy.tightBounds;
this.valueCount = toCopy.valueCount;
this.nullValueCount = toCopy.nullValueCount;
this.nanValueCount = toCopy.nanValueCount;
Expand All @@ -88,7 +84,6 @@ void fromFieldMetrics(FieldMetrics<T> fieldMetrics) {

setLowerBound(fieldMetrics.lowerBound());
setUpperBound(fieldMetrics.upperBound());
this.tightBounds = false;
this.valueCount = fieldMetrics.valueCount();
this.nullValueCount = fieldMetrics.nullValueCount() < 0 ? null : fieldMetrics.nullValueCount();
this.nanValueCount = fieldMetrics.nanValueCount() < 0 ? null : fieldMetrics.nanValueCount();
Expand Down Expand Up @@ -132,11 +127,6 @@ public T upperBound() {
(upperBound != null && isBinary() ? ByteBuffer.wrap((byte[]) upperBound) : upperBound);
}

@Override
public boolean tightBounds() {
return tightBounds;
}

@Override
public boolean hasValueCount() {
return valueCount != null;
Expand Down Expand Up @@ -186,7 +176,6 @@ private Object getOffset(int offset) {
return switch (offset) {
case StatsUtil.LOWER_BOUND_OFFSET -> lowerBound();
case StatsUtil.UPPER_BOUND_OFFSET -> upperBound();
case StatsUtil.TIGHT_BOUNDS_OFFSET -> tightBounds;
case StatsUtil.VALUE_COUNT_OFFSET -> valueCount;
case StatsUtil.NULL_VALUE_COUNT_OFFSET -> nullValueCount;
case StatsUtil.NAN_VALUE_COUNT_OFFSET -> nanValueCount;
Expand All @@ -204,7 +193,6 @@ private void setOffset(int offset, Object value) {
switch (offset) {
case StatsUtil.LOWER_BOUND_OFFSET -> setLowerBound(value);
case StatsUtil.UPPER_BOUND_OFFSET -> setUpperBound(value);
case StatsUtil.TIGHT_BOUNDS_OFFSET -> this.tightBounds = (Boolean) value;
case StatsUtil.VALUE_COUNT_OFFSET -> this.valueCount = (Long) value;
case StatsUtil.NULL_VALUE_COUNT_OFFSET -> this.nullValueCount = (Long) value;
case StatsUtil.NAN_VALUE_COUNT_OFFSET -> this.nanValueCount = (Long) value;
Expand Down
5 changes: 0 additions & 5 deletions core/src/main/java/org/apache/iceberg/MapBackedContentStats.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -192,11 +192,6 @@ private T decodeBound(Map<Integer, ByteBuffer> bounds) {
return (T) Conversions.fromByteBuffer(fieldType, bounds.get(fieldId));
}

@Override
public boolean tightBounds() {
return false;
}

@Override
public boolean hasValueCount() {
return count(valueCounts) != null;
Expand Down
22 changes: 4 additions & 18 deletions core/src/main/java/org/apache/iceberg/StatsUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,11 +49,10 @@ private StatsUtil() {}
// Offsets used for individual stats columns
static final int LOWER_BOUND_OFFSET = 1;
static final int UPPER_BOUND_OFFSET = 2;
static final int TIGHT_BOUNDS_OFFSET = 3;
static final int VALUE_COUNT_OFFSET = 4;
static final int NULL_VALUE_COUNT_OFFSET = 5;
static final int NAN_VALUE_COUNT_OFFSET = 6;
static final int TOTAL_BYTES_OFFSET = 7;
static final int VALUE_COUNT_OFFSET = 3;
static final int NULL_VALUE_COUNT_OFFSET = 4;
static final int NAN_VALUE_COUNT_OFFSET = 5;
static final int TOTAL_BYTES_OFFSET = 6;

static final String LOWER_BOUND_NAME = "lower_bound";
static final String UPPER_BOUND_NAME = "upper_bound";
Expand Down Expand Up @@ -198,10 +197,6 @@ private static boolean inBaseIdRange(int baseId, int statId) {
return statId >= baseId && statId < (baseId + NUM_RESERVED_FIELD_STATS_IDS);
}

private static boolean tracksTightBounds(Type type) {
return !type.isVariantType() && !isGeoType(type);
}

private static boolean isFloatingPoint(Type type) {
return type.typeId() == Type.TypeID.FLOAT || type.typeId() == Type.TypeID.DOUBLE;
}
Expand Down Expand Up @@ -270,15 +265,6 @@ static Types.StructType fieldStatsStruct(Type type, int baseId, MetricsModes.Met
if (mode.hasBounds()) {
fields.add(lowerBoundField(type, baseId));
fields.add(upperBoundField(type, baseId));

if (tracksTightBounds(type)) {
fields.add(
optional(
baseId + TIGHT_BOUNDS_OFFSET,
"tight_bounds",
Types.BooleanType.get(),
"True if lower=min and upper=max; false otherwise"));
}
}

fields.add(
Expand Down
7 changes: 7 additions & 0 deletions core/src/main/java/org/apache/iceberg/TrackedFile.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,9 @@ interface TrackedFile {
String CONTENT_STATS_NAME = "content_stats";
String CONTENT_STATS_DOC = "Content statistics for this entry";

Types.NestedField TIGHT_STATS =
Types.NestedField.optional(
151, "tight_stats", Types.BooleanType.get(), "Whether the file's statistics are tight");
Types.NestedField SORT_ORDER_ID =
Types.NestedField.optional(
140, "sort_order_id", Types.IntegerType.get(), "ID of the sort order for this file");
Expand Down Expand Up @@ -105,6 +108,7 @@ PARTITION_ID, PARTITION_NAME, typeOrUnknown(partitionType), PARTITION_DOC),
CONTENT_STATS_NAME,
typeOrUnknown(contentStatsType),
CONTENT_STATS_DOC),
TIGHT_STATS,
SORT_ORDER_ID,
DELETION_VECTOR,
MANIFEST_INFO,
Expand Down Expand Up @@ -163,6 +167,9 @@ static Schema readSchema(Types.StructType partitionType, Types.StructType conten
/** Returns the content stats for this entry. */
ContentStats contentStats();

/** Returns whether the file's statistics are tight, or null if not set. */
Boolean tightStats();

/** Returns the ID of the sort order for this file, or null. */
Integer sortOrderId();

Expand Down
15 changes: 15 additions & 0 deletions core/src/main/java/org/apache/iceberg/TrackedFileAdapters.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -212,6 +212,11 @@ public Map<Integer, ByteBuffer> lowerBounds() {
public Map<Integer, ByteBuffer> upperBounds() {
return ContentStatsBackedMap.upperBounds(file().contentStats());
}

@Override
public Boolean tightStats() {
return file().tightStats();
}
}

/** Adapts a TrackedFile DATA entry to the {@link DataFile} interface. */
Expand Down Expand Up @@ -627,6 +632,11 @@ public ContentStats contentStats() {
return stats;
}

@Override
public Boolean tightStats() {
return file.tightStats();
}

@Override
public Integer sortOrderId() {
return file.sortOrderId();
Expand Down Expand Up @@ -748,6 +758,11 @@ public ContentStats contentStats() {
return null;
}

@Override
public Boolean tightStats() {
return null;
}

@Override
public Integer sortOrderId() {
// Manifests have no table sort order.
Expand Down
33 changes: 23 additions & 10 deletions core/src/main/java/org/apache/iceberg/TrackedFileStruct.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ class TrackedFileStruct extends SupportsIndexProjection implements TrackedFile,
TrackedFile.CONTENT_STATS_NAME,
EMPTY_STRUCT_TYPE,
TrackedFile.CONTENT_STATS_DOC),
TrackedFile.TIGHT_STATS,
TrackedFile.SORT_ORDER_ID,
TrackedFile.DELETION_VECTOR,
TrackedFile.MANIFEST_INFO,
Expand All @@ -72,6 +73,7 @@ class TrackedFileStruct extends SupportsIndexProjection implements TrackedFile,
// optional fields
private Integer specId = null;
private ContentStats contentStats = null;
private Boolean tightStats = null;
private Integer sortOrderId = null;
private DeletionVector deletionVector = null;
private ManifestInfo manifestInfo = null;
Expand Down Expand Up @@ -100,6 +102,7 @@ class TrackedFileStruct extends SupportsIndexProjection implements TrackedFile,
Integer specId,
PartitionData partition,
ContentStats contentStats,
Boolean tightStats,
Integer sortOrderId,
DeletionVector deletionVector,
ManifestInfo manifestInfo,
Expand All @@ -115,6 +118,7 @@ class TrackedFileStruct extends SupportsIndexProjection implements TrackedFile,
this.specId = specId;
this.partition = partition;
this.contentStats = contentStats;
this.tightStats = tightStats;
this.sortOrderId = sortOrderId;
this.deletionVector = deletionVector;
this.manifestInfo = manifestInfo;
Expand Down Expand Up @@ -143,6 +147,7 @@ private TrackedFileStruct(TrackedFileStruct toCopy, Set<Integer> statsIds) {
this.contentStats = null;
}

this.tightStats = toCopy.tightStats;
this.manifestInfo = toCopy.manifestInfo != null ? toCopy.manifestInfo.copy() : null;
this.keyMetadata =
toCopy.keyMetadata != null
Expand Down Expand Up @@ -219,6 +224,11 @@ public ContentStats contentStats() {
return contentStats;
}

@Override
public Boolean tightStats() {
return tightStats;
}

@Override
public Integer sortOrderId() {
return sortOrderId;
Expand Down Expand Up @@ -270,11 +280,12 @@ private Object getByPos(int pos) {
case 6 -> specId;
case 7 -> partition;
case 8 -> contentStats;
case 9 -> sortOrderId;
case 10 -> deletionVector;
case 11 -> manifestInfo;
case 12 -> keyMetadata();
case 13 -> splitOffsets();
case 9 -> tightStats;
case 10 -> sortOrderId;
case 11 -> deletionVector;
case 12 -> manifestInfo;
case 13 -> keyMetadata();
case 14 -> splitOffsets();
default -> throw new UnsupportedOperationException("Unknown field ordinal: " + pos);
};
}
Expand All @@ -293,11 +304,12 @@ protected <T> void internalSet(int pos, T value) {
case 6 -> this.specId = (Integer) value;
case 7 -> this.partition = (StructLike) value;
case 8 -> this.contentStats = (ContentStats) value;
case 9 -> this.sortOrderId = (Integer) value;
case 10 -> this.deletionVector = (DeletionVector) value;
case 11 -> this.manifestInfo = (ManifestInfo) value;
case 12 -> this.keyMetadata = ByteBuffers.toByteArray((ByteBuffer) value);
case 13 -> this.splitOffsets = ArrayUtil.toLongArray((List<Long>) value);
case 9 -> this.tightStats = (Boolean) value;
case 10 -> this.sortOrderId = (Integer) value;
case 11 -> this.deletionVector = (DeletionVector) value;
case 12 -> this.manifestInfo = (ManifestInfo) value;
case 13 -> this.keyMetadata = ByteBuffers.toByteArray((ByteBuffer) value);
case 14 -> this.splitOffsets = ArrayUtil.toLongArray((List<Long>) value);
default -> {
// ignore the object, it must be from a newer version of the format
}
Expand All @@ -316,6 +328,7 @@ public String toString() {
.add("partition", partition)
.add("tracking", tracking)
.add("content_stats", contentStats)
.add("tight_stats", tightStats)
.add("sort_order_id", sortOrderId)
.add("deletion_vector", deletionVector)
.add("manifest_info", manifestInfo)
Expand Down
6 changes: 1 addition & 5 deletions core/src/test/java/org/apache/iceberg/StatsTestUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ static TrackedFile trackedFile(String location, long recordCount, ContentStats s
null,
null,
null,
null,
null);
}

Expand Down Expand Up @@ -128,11 +129,6 @@ public Object upperBound() {
return upperBound;
}

@Override
public boolean tightBounds() {
return false;
}

@Override
public boolean hasValueCount() {
return valueCount != null;
Expand Down
13 changes: 3 additions & 10 deletions core/src/test/java/org/apache/iceberg/TestContentStatsStruct.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,17 +55,10 @@ public class TestContentStatsStruct {

private static final FieldStats<Long> ID_STATS =
new FieldStatsStruct<>(
CONTENT_STATS_STRUCT.field("id").type().asStructType(), 0L, 25L, true, 26L, 0L, 0L, 0L);
CONTENT_STATS_STRUCT.field("id").type().asStructType(), 0L, 25L, 26L, 0L, 0L, 0L);
private static final FieldStats<String> DATA_STATS =
new FieldStatsStruct<>(
CONTENT_STATS_STRUCT.field("data").type().asStructType(),
"a",
"z",
true,
26L,
0L,
0L,
2L);
CONTENT_STATS_STRUCT.field("data").type().asStructType(), "a", "z", 26L, 0L, 0L, 2L);

@Test
public void emptyContentStats() {
Expand Down Expand Up @@ -103,7 +96,7 @@ public void setStatsUnknownField() {
ContentStatsStruct stats = new ContentStatsStruct(CONTENT_STATS_STRUCT);

FieldStats<Integer> fieldStats =
new FieldStatsStruct<>(UNKNOWN_FIELD_STATS_STRUCT, 0, 10, false, 8, 3, 0, 23L);
new FieldStatsStruct<>(UNKNOWN_FIELD_STATS_STRUCT, 0, 10, 8, 3, 0, 23L);

assertThatThrownBy(() -> stats.setStats(4, fieldStats))
.hasMessage("Cannot set stats for unknown field ID: 4")
Expand Down
Loading
Loading