diff --git a/core/src/main/java/org/apache/iceberg/BaseContentStats.java b/core/src/main/java/org/apache/iceberg/BaseContentStats.java deleted file mode 100644 index 0a83f7eaa099..000000000000 --- a/core/src/main/java/org/apache/iceberg/BaseContentStats.java +++ /dev/null @@ -1,271 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iceberg; - -import java.io.Serializable; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.stream.Collectors; -import org.apache.iceberg.data.GenericRecord; -import org.apache.iceberg.relocated.com.google.common.base.MoreObjects; -import org.apache.iceberg.relocated.com.google.common.base.Preconditions; -import org.apache.iceberg.relocated.com.google.common.collect.Lists; -import org.apache.iceberg.relocated.com.google.common.collect.Maps; -import org.apache.iceberg.types.Type; -import org.apache.iceberg.types.Types; - -class BaseContentStats implements ContentStats, Serializable { - - private final List> fieldStats; - private final Map> fieldStatsById; - private final Types.StructType statsStruct; - - /** Used by Avro reflection to instantiate this class when reading manifest files. */ - BaseContentStats(Types.StructType projection) { - this.statsStruct = projection; - this.fieldStats = Lists.newArrayListWithCapacity(projection.fields().size()); - this.fieldStatsById = Maps.newLinkedHashMapWithExpectedSize(projection.fields().size()); - for (int i = 0; i < projection.fields().size(); i++) { - Types.NestedField field = projection.fields().get(i); - Preconditions.checkArgument( - field.type().isStructType(), "Field stats must be a struct type: %s", field.type()); - Types.StructType structType = field.type().asStructType(); - Type type = null; - if (null != structType.field("lower_bound")) { - type = structType.field("lower_bound").type(); - } else if (null != structType.field("upper_bound")) { - type = structType.field("upper_bound").type(); - } - - fieldStats.add( - BaseFieldStats.builder() - .fieldId(StatsUtil.fieldIdForStatsField(field.fieldId())) - .statsStruct(structType) - .type(type) - .build()); - } - } - - private BaseContentStats(Types.StructType struct, List> fieldStats) { - this.statsStruct = struct; - this.fieldStats = Lists.newArrayList(fieldStats); - this.fieldStatsById = Maps.newLinkedHashMapWithExpectedSize(fieldStats.size()); - } - - @Override - public List> fieldStats() { - return fieldStats; - } - - @Override - public Types.StructType statsStruct() { - return statsStruct; - } - - @SuppressWarnings("unchecked") - @Override - public FieldStats statsFor(int fieldId) { - if (fieldStatsById.isEmpty() && !fieldStats.isEmpty()) { - fieldStats.stream() - .filter(Objects::nonNull) - .forEach(stat -> fieldStatsById.put(stat.fieldId(), stat)); - } - - return (FieldStats) fieldStatsById.get(fieldId); - } - - @Override - public int size() { - return fieldStats.size(); - } - - @Override - public T get(int pos, Class javaClass) { - if (pos > statsStruct.fields().size() - 1) { - // return null in case there are more stats schemas than actual stats available as Avro calls - // get() for all available stats schemas of a given table - return null; - } - - int statsFieldId = statsStruct.fields().get(pos).fieldId(); - FieldStats value = statsFor(StatsUtil.fieldIdForStatsField(statsFieldId)); - if (value == null || javaClass.isInstance(value)) { - return javaClass.cast(value); - } - - throw new IllegalArgumentException( - String.format( - "Wrong class, expected %s but was %s for object: %s", - javaClass.getName(), value.getClass().getName(), value)); - } - - @SuppressWarnings({"unchecked", "rawtypes", "CyclomaticComplexity"}) - @Override - public void set(int pos, T value) { - if (value instanceof GenericRecord record) { - FieldStats stat = fieldStats.get(pos); - BaseFieldStats.Builder builder = BaseFieldStats.buildFrom(stat); - Type type = stat.type(); - - Object lowerBound = record.getField(FieldStatistic.LOWER_BOUND.fieldName()); - if (null != type && null != lowerBound) { - Preconditions.checkArgument( - type.typeId().javaClass().isInstance(lowerBound), - "Invalid lower bound type, expected a subtype of %s: %s", - type.typeId().javaClass(), - lowerBound.getClass().getName()); - builder.lowerBound(type.typeId().javaClass().cast(lowerBound)); - } - - Object upperBound = record.getField(FieldStatistic.UPPER_BOUND.fieldName()); - if (null != type && null != upperBound) { - Preconditions.checkArgument( - type.typeId().javaClass().isInstance(upperBound), - "Invalid upper bound type, expected a subtype of %s: %s", - type.typeId().javaClass(), - upperBound.getClass().getName()); - builder.upperBound(type.typeId().javaClass().cast(upperBound)); - } - - if (null != record.getField(FieldStatistic.TIGHT_BOUNDS.fieldName())) { - Boolean tightBounds = (Boolean) record.getField(FieldStatistic.TIGHT_BOUNDS.fieldName()); - builder.tightBounds(null != tightBounds && tightBounds); - } - - if (null != record.getField(FieldStatistic.VALUE_COUNT.fieldName())) { - builder.valueCount((Long) record.getField(FieldStatistic.VALUE_COUNT.fieldName())); - } - - if (null != record.getField(FieldStatistic.NULL_VALUE_COUNT.fieldName())) { - builder.nullValueCount((Long) record.getField(FieldStatistic.NULL_VALUE_COUNT.fieldName())); - } - - if (null != record.getField(FieldStatistic.NAN_VALUE_COUNT.fieldName())) { - builder.nanValueCount((Long) record.getField(FieldStatistic.NAN_VALUE_COUNT.fieldName())); - } - - if (null != record.getField(FieldStatistic.AVG_VALUE_SIZE_IN_BYTES.fieldName())) { - builder.avgValueSizeInBytes( - (Integer) record.getField(FieldStatistic.AVG_VALUE_SIZE_IN_BYTES.fieldName())); - } - - BaseFieldStats newStat = builder.build(); - fieldStats.set(pos, newStat); - } - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(this).add("fieldStats", fieldStats).toString(); - } - - @Override - public boolean equals(Object o) { - if (!(o instanceof BaseContentStats)) { - return false; - } - - BaseContentStats that = (BaseContentStats) o; - return Objects.equals(fieldStats, that.fieldStats) - && Objects.equals(statsStruct, that.statsStruct); - } - - @Override - public int hashCode() { - return Objects.hash(fieldStats, statsStruct); - } - - public static Builder builder() { - return new Builder(); - } - - public static Builder buildFrom(ContentStats stats) { - return builder().withStatsStruct(stats.statsStruct()).withFieldStats(stats.fieldStats()); - } - - public static Builder buildFrom(ContentStats stats, Set requestedColumnIds) { - if (null == requestedColumnIds) { - return buildFrom(stats); - } - - return builder() - .withStatsStruct(stats.statsStruct()) - .withFieldStats( - stats.fieldStats().stream() - .filter(stat -> requestedColumnIds.contains(stat.fieldId())) - .collect(Collectors.toList())); - } - - public static class Builder { - private final List> stats = Lists.newArrayList(); - private Types.StructType statsStruct; - private Schema schema; - - private Builder() {} - - public Builder withStatsStruct(Types.StructType struct) { - this.statsStruct = struct; - return this; - } - - public Builder withTableSchema(Schema tableSchema) { - this.schema = tableSchema; - return this; - } - - public Builder withFieldStats(FieldStats fieldStats) { - stats.add(fieldStats); - return this; - } - - public Builder withFieldStats(List> fieldStats) { - stats.addAll(fieldStats); - return this; - } - - @SuppressWarnings("rawtypes") - public BaseContentStats build() { - Preconditions.checkArgument( - null != statsStruct || null != schema, "Either stats struct or table schema must be set"); - Preconditions.checkArgument( - null == statsStruct || null == schema, "Cannot set stats struct and table schema"); - if (null != schema) { - this.statsStruct = StatsUtil.contentStatsFor(schema).type().asStructType(); - } - - List> resolvedStats = Lists.newArrayListWithCapacity(stats.size()); - for (FieldStats stat : stats) { - int statsFieldId = StatsUtil.statsFieldIdForField(stat.fieldId()); - Types.NestedField statsField = statsStruct.field(statsFieldId); - if (null != statsField && statsField.type().isStructType()) { - resolvedStats.add( - ((BaseFieldStats.Builder) BaseFieldStats.buildFrom(stat)) - .statsStruct(statsField.type().asStructType()) - .build()); - } else { - resolvedStats.add(stat); - } - } - - return new BaseContentStats(statsStruct, resolvedStats); - } - } -} diff --git a/core/src/main/java/org/apache/iceberg/BaseFieldStats.java b/core/src/main/java/org/apache/iceberg/BaseFieldStats.java deleted file mode 100644 index b2945f8c30ef..000000000000 --- a/core/src/main/java/org/apache/iceberg/BaseFieldStats.java +++ /dev/null @@ -1,363 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iceberg; - -import java.nio.ByteBuffer; -import java.nio.CharBuffer; -import java.util.Objects; -import org.apache.iceberg.avro.SupportsIndexProjection; -import org.apache.iceberg.relocated.com.google.common.base.MoreObjects; -import org.apache.iceberg.relocated.com.google.common.base.Preconditions; -import org.apache.iceberg.types.Type; -import org.apache.iceberg.types.Types; -import org.apache.iceberg.util.ByteBuffers; - -class BaseFieldStats extends SupportsIndexProjection implements FieldStats { - private static final int[] IDENTITY_MAPPING = identityMapping(); - private final int fieldId; - private final Type type; - private final T lowerBound; - private final T upperBound; - private final boolean tightBounds; - private final Long valueCount; - private final Long nullValueCount; - private final Long nanValueCount; - private final Integer avgValueSizeInBytes; - - private BaseFieldStats( - int fieldId, - int[] fromProjectionPos, - Type type, - T lowerBound, - T upperBound, - boolean tightBounds, - Long valueCount, - Long nullValueCount, - Long nanValueCount, - Integer avgValueSizeInBytes) { - super(fromProjectionPos != null ? fromProjectionPos : IDENTITY_MAPPING); - this.fieldId = fieldId; - this.type = type; - this.lowerBound = lowerBound; - this.upperBound = upperBound; - this.tightBounds = tightBounds; - this.valueCount = valueCount; - this.nullValueCount = nullValueCount; - this.nanValueCount = nanValueCount; - this.avgValueSizeInBytes = avgValueSizeInBytes; - } - - private static int[] identityMapping() { - int numStats = FieldStatistic.values().length; - int[] mapping = new int[numStats]; - for (int i = 0; i < numStats; i++) { - mapping[i] = i; - } - - return mapping; - } - - /** - * Computes a position mapping from the column-specific stats struct to the full stats struct. - * Each entry maps a projected position to its base position (0-based) using the field ID offsets - * from the column's base stats field ID. - */ - private static int[] projectionMapping(Types.StructType statsStruct, int dataFieldId) { - if (statsStruct == null) { - return null; - } - - int baseStatsFieldId = StatsUtil.statsFieldIdForField(dataFieldId); - int[] mapping = new int[statsStruct.fields().size()]; - for (int i = 0; i < mapping.length; i++) { - // offset is 1-based (matching FieldStatistic.offset()), position is 0-based - mapping[i] = statsStruct.fields().get(i).fieldId() - baseStatsFieldId - 1; - } - - return mapping; - } - - @Override - public int fieldId() { - return fieldId; - } - - @Override - public Type type() { - return type; - } - - @Override - public Long valueCount() { - return valueCount; - } - - @Override - public Long nullValueCount() { - return nullValueCount; - } - - @Override - public Long nanValueCount() { - return nanValueCount; - } - - @Override - public Integer avgValueSizeInBytes() { - return avgValueSizeInBytes; - } - - @SuppressWarnings("unchecked") - private static T serializableBound(T bound) { - if (bound instanceof CharBuffer) { - // CharBuffer is not serializable, use String instead - return (T) bound.toString(); - } else if (bound instanceof ByteBuffer) { - // ByteBuffer is not serializable, use byte[] instead - return (T) ByteBuffers.toByteArray((ByteBuffer) bound); - } - - return bound; - } - - @SuppressWarnings("unchecked") - @Override - public T lowerBound() { - if (null != type - && type.typeId().javaClass().equals(ByteBuffer.class) - && lowerBound instanceof byte[]) { - // for serializability we store binary types as byte[] and must convert back to - // ByteBuffer - return (T) ByteBuffer.wrap((byte[]) lowerBound); - } else { - return lowerBound; - } - } - - @SuppressWarnings("unchecked") - @Override - public T upperBound() { - if (null != type - && type.typeId().javaClass().equals(ByteBuffer.class) - && upperBound instanceof byte[]) { - // for serializability we store binary types as byte[] and must convert back to - // ByteBuffer - return (T) ByteBuffer.wrap((byte[]) upperBound); - } else { - return upperBound; - } - } - - @Override - public boolean tightBounds() { - return tightBounds; - } - - @Override - protected X internalGet(int pos, Class javaClass) { - return switch (FieldStatistic.fromPosition(pos)) { - case LOWER_BOUND -> javaClass.cast(lowerBound()); - case UPPER_BOUND -> javaClass.cast(upperBound()); - case TIGHT_BOUNDS -> javaClass.cast(tightBounds); - case VALUE_COUNT -> javaClass.cast(valueCount); - case NULL_VALUE_COUNT -> javaClass.cast(nullValueCount); - case NAN_VALUE_COUNT -> javaClass.cast(nanValueCount); - case AVG_VALUE_SIZE_IN_BYTES -> javaClass.cast(avgValueSizeInBytes); - default -> throw new UnsupportedOperationException("Unknown field ordinal: " + pos); - }; - } - - @Override - protected void internalSet(int pos, X value) { - throw new UnsupportedOperationException("set() not supported"); - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(this) - .add("fieldId", fieldId) - .add("type", type) - .add("lowerBound", lowerBound) - .add("upperBound", upperBound) - .add("tightBounds", tightBounds) - .add("valueCount", valueCount) - .add("nullValueCount", nullValueCount) - .add("nanValueCount", nanValueCount) - .add("avgValueSizeInBytes", avgValueSizeInBytes) - .toString(); - } - - @Override - public boolean equals(Object o) { - if (!(o instanceof BaseFieldStats)) { - return false; - } - - BaseFieldStats that = (BaseFieldStats) o; - return fieldId == that.fieldId - && tightBounds == that.tightBounds - && Objects.equals(type, that.type) - && Objects.deepEquals(lowerBound, that.lowerBound) - && Objects.deepEquals(upperBound, that.upperBound) - && Objects.equals(valueCount, that.valueCount) - && Objects.equals(nullValueCount, that.nullValueCount) - && Objects.equals(nanValueCount, that.nanValueCount) - && Objects.equals(avgValueSizeInBytes, that.avgValueSizeInBytes); - } - - @Override - public int hashCode() { - return Objects.hash( - fieldId, - type, - lowerBound, - upperBound, - tightBounds, - valueCount, - nullValueCount, - nanValueCount, - avgValueSizeInBytes); - } - - public static Builder builder() { - return new Builder<>(); - } - - public static Builder buildFrom(FieldStats value) { - Preconditions.checkArgument(null != value, "Invalid column stats: null"); - return BaseFieldStats.builder() - .type(value.type()) - .fieldId(value.fieldId()) - .lowerBound(value.lowerBound()) - .upperBound(value.upperBound()) - .tightBounds(value.tightBounds()) - .valueCount(value.valueCount()) - .nullValueCount(value.nullValueCount()) - .nanValueCount(value.nanValueCount()) - .avgValueSizeInBytes(value.avgValueSizeInBytes()); - } - - public static class Builder { - private int fieldId; - private int[] fromProjectionPos; - private Type type; - private T lowerBound; - private T upperBound; - private boolean tightBounds; - private Long valueCount; - private Long nullValueCount; - private Long nanValueCount; - private Integer avgValueSizeInBytes; - - private Builder() {} - - public Builder statsStruct(Types.StructType statsStruct) { - this.fromProjectionPos = projectionMapping(statsStruct, fieldId); - return this; - } - - public Builder type(Type newType) { - this.type = newType; - return this; - } - - public Builder valueCount(Long newValueCount) { - this.valueCount = newValueCount; - return this; - } - - public Builder nullValueCount(Long newNullValueCount) { - this.nullValueCount = newNullValueCount; - return this; - } - - public Builder nanValueCount(Long newNanValueCount) { - this.nanValueCount = newNanValueCount; - return this; - } - - public Builder avgValueSizeInBytes(Integer newAvgValueSizeInBytes) { - this.avgValueSizeInBytes = newAvgValueSizeInBytes; - return this; - } - - public Builder lowerBound(T newLowerBound) { - this.lowerBound = serializableBound(newLowerBound); - return this; - } - - public Builder upperBound(T newUpperBound) { - this.upperBound = serializableBound(newUpperBound); - return this; - } - - public Builder fieldId(int newFieldId) { - this.fieldId = newFieldId; - return this; - } - - public Builder tightBounds(boolean newTightBounds) { - this.tightBounds = newTightBounds; - return this; - } - - public Builder tightBounds() { - this.tightBounds = true; - return this; - } - - public BaseFieldStats build() { - if (null != lowerBound) { - Preconditions.checkArgument( - null != type, "Invalid type (required when lower bound is set): null"); - Preconditions.checkArgument( - type.typeId().javaClass().isInstance(lowerBound) - || (type.typeId().javaClass().equals(ByteBuffer.class) - && lowerBound instanceof byte[]), - "Invalid lower bound type, expected a subtype of %s: %s", - type.typeId().javaClass().getName(), - lowerBound.getClass().getName()); - } - - if (null != upperBound) { - Preconditions.checkArgument( - null != type, "Invalid type (required when lower bound is set): null"); - Preconditions.checkArgument( - type.typeId().javaClass().isInstance(upperBound) - || (type.typeId().javaClass().equals(ByteBuffer.class) - && upperBound instanceof byte[]), - "Invalid upper bound type, expected a subtype of %s: %s", - type.typeId().javaClass().getName(), - upperBound.getClass().getName()); - } - - return new BaseFieldStats<>( - fieldId, - fromProjectionPos, - type, - lowerBound, - upperBound, - tightBounds, - valueCount, - nullValueCount, - nanValueCount, - avgValueSizeInBytes); - } - } -} diff --git a/core/src/main/java/org/apache/iceberg/ContentStats.java b/core/src/main/java/org/apache/iceberg/ContentStats.java index 623a8eb39baf..77a515af6cd8 100644 --- a/core/src/main/java/org/apache/iceberg/ContentStats.java +++ b/core/src/main/java/org/apache/iceberg/ContentStats.java @@ -18,13 +18,13 @@ */ package org.apache.iceberg; -import java.util.List; +import java.util.Set; import org.apache.iceberg.types.Types; -interface ContentStats extends StructLike { +interface ContentStats { /** A list of all the {@link FieldStats} */ - List> fieldStats(); + Iterable> fieldStats(); /** * Returns a {@link FieldStats} instance holding field stats for the given field ID. @@ -35,6 +35,12 @@ interface ContentStats extends StructLike { */ FieldStats statsFor(int fieldId); - /** The stats struct holding nested structs with their respective field stats */ - Types.StructType statsStruct(); + /** Container struct type containing tracked field-level stats structs. */ + Types.StructType type(); + + /** Returns a copy, deep-copying all field stats. */ + ContentStats copy(); + + /** Returns a copy, of only the selected field stats by ID. */ + ContentStats copy(Set fieldIds); } diff --git a/core/src/main/java/org/apache/iceberg/ContentStatsStruct.java b/core/src/main/java/org/apache/iceberg/ContentStatsStruct.java new file mode 100644 index 000000000000..b3abdf0648d1 --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/ContentStatsStruct.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg; + +import java.io.Serializable; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.types.TypeUtil; +import org.apache.iceberg.types.Types; + +class ContentStatsStruct implements ContentStats, StructLike, Serializable { + private final Map> idToFieldStats = Maps.newHashMap(); + private final Types.StructType struct; + private final int[] posToId; + + ContentStatsStruct(Types.StructType struct) { + this.struct = struct; + this.posToId = posToId(struct); + } + + private ContentStatsStruct(ContentStatsStruct toCopy, Set fieldIds) { + this(fieldIds != null ? TypeUtil.select(toCopy.struct, fieldIds) : toCopy.struct); + if (fieldIds != null) { + for (int fieldId : fieldIds) { + idToFieldStats.put(fieldId, toCopy.idToFieldStats.get(fieldId).copy()); + } + } else { + for (Map.Entry> entry : toCopy.idToFieldStats.entrySet()) { + idToFieldStats.put(entry.getKey(), entry.getValue().copy()); + } + } + } + + @Override + public Iterable> fieldStats() { + return idToFieldStats.values(); + } + + @Override + @SuppressWarnings("unchecked") + public FieldStats statsFor(int id) { + return (FieldStats) idToFieldStats.get(id); + } + + public void setStats(int id, FieldStats fieldStats) { + Preconditions.checkArgument( + struct.field(StatsUtil.toBaseId(id)) != null, + "Cannot set stats for unknown field ID: %s", + id); + Preconditions.checkArgument( + id == fieldStats.fieldId(), + "Mismatched field stats for ID %s: actual ID %s", + id, + fieldStats.fieldId()); + + idToFieldStats.put(id, fieldStats); + } + + @Override + public Types.StructType type() { + return struct; + } + + @Override + public int size() { + return struct.fields().size(); + } + + @Override + public T get(int pos, Class javaClass) { + return javaClass.cast(idToFieldStats.get(posToId[pos])); + } + + @Override + public void set(int pos, T value) { + idToFieldStats.put(posToId[pos], (FieldStats) value); + } + + @Override + public ContentStatsStruct copy() { + return new ContentStatsStruct(this, null); + } + + @Override + public ContentStatsStruct copy(Set fieldIds) { + Preconditions.checkArgument( + fieldIds != null && !fieldIds.isEmpty(), "Invalid ID set to copy: %s", fieldIds); + + return new ContentStatsStruct(this, fieldIds); + } + + private static int[] posToId(Types.StructType struct) { + List fields = struct.fields(); + int[] posToId = new int[fields.size()]; + for (int i = 0; i < posToId.length; i += 1) { + int statsFieldId = fields.get(i).fieldId(); + posToId[i] = StatsUtil.toFieldId(statsFieldId); + } + + return posToId; + } +} diff --git a/core/src/main/java/org/apache/iceberg/FieldStatistic.java b/core/src/main/java/org/apache/iceberg/FieldStatistic.java deleted file mode 100644 index 7061517b0559..000000000000 --- a/core/src/main/java/org/apache/iceberg/FieldStatistic.java +++ /dev/null @@ -1,218 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iceberg; - -import static org.apache.iceberg.types.Types.NestedField.optional; -import static org.apache.iceberg.types.Types.NestedField.required; - -import java.util.List; -import org.apache.iceberg.relocated.com.google.common.collect.Lists; -import org.apache.iceberg.types.Type; -import org.apache.iceberg.types.Types; - -enum FieldStatistic { - LOWER_BOUND(1, "lower_bound"), - UPPER_BOUND(2, "upper_bound"), - TIGHT_BOUNDS(3, "tight_bounds"), - VALUE_COUNT(4, "value_count"), - NULL_VALUE_COUNT(5, "null_value_count"), - NAN_VALUE_COUNT(6, "nan_value_count"), - AVG_VALUE_SIZE_IN_BYTES(7, "avg_value_size_in_bytes"); - - // Offsets used within geo_lower struct (relative to the parent stats struct base ID). - private static final int GEO_LOWER_X_OFFSET = 10; - private static final int GEO_LOWER_Y_OFFSET = 11; - private static final int GEO_LOWER_Z_OFFSET = 12; - private static final int GEO_LOWER_M_OFFSET = 13; - // Offsets used within geo_upper struct (relative to the parent stats struct base ID). - private static final int GEO_UPPER_X_OFFSET = 14; - private static final int GEO_UPPER_Y_OFFSET = 15; - private static final int GEO_UPPER_Z_OFFSET = 16; - private static final int GEO_UPPER_M_OFFSET = 17; - - private final int offset; - private final String fieldName; - - FieldStatistic(int offset, String fieldName) { - this.offset = offset; - this.fieldName = fieldName; - } - - /** - * The offset from the field ID of the base stats structure - * - * @return The offset from the field ID of the base stats structure - */ - public int offset() { - return offset; - } - - /** - * The ordinal position (0-based) within the stats structure - * - * @return The ordinal position (0-based) within the stats structure - */ - public int position() { - return offset - 1; - } - - /** - * The field name - * - * @return The field name - */ - public String fieldName() { - return fieldName; - } - - /** - * Returns the {@link FieldStatistic} from its ordinal position (0-based) in the stats structure - * - * @param position The ordinal position (0-based) in the stats structure - * @return The {@link FieldStatistic} from its ordinal position (0-based) in the stats structure - */ - public static FieldStatistic fromPosition(int position) { - return switch (position) { - case 0 -> LOWER_BOUND; - case 1 -> UPPER_BOUND; - case 2 -> TIGHT_BOUNDS; - case 3 -> VALUE_COUNT; - case 4 -> NULL_VALUE_COUNT; - case 5 -> NAN_VALUE_COUNT; - case 6 -> AVG_VALUE_SIZE_IN_BYTES; - default -> throw new IllegalArgumentException("Invalid statistic position: " + position); - }; - } - - @SuppressWarnings("checkstyle:CyclomaticComplexity") - public static Types.StructType fieldStatsFor(Types.NestedField field, int baseFieldId) { - List fields = Lists.newArrayListWithCapacity(7); - Type type = field.type(); - boolean isGeo = type.typeId() == Type.TypeID.GEOMETRY || type.typeId() == Type.TypeID.GEOGRAPHY; - boolean isVariant = type.isVariantType(); - - // For geo types, lower/upper bounds are XYZM points stored in geo_lower / geo_upper structs. - // For variant types, bounds are unshredded variant values (same type as the field). - // For all other primitive types, bounds use the field's type. - Type lowerBoundType = isGeo ? geoLowerBoundStruct(baseFieldId) : type; - Type upperBoundType = isGeo ? geoUpperBoundStruct(baseFieldId) : type; - String lowerBoundDoc = - isGeo - ? "Lower bound XYZM point of the bounding box for the geo column" - : "Lower bound stored as the field's type"; - String upperBoundDoc = - isGeo - ? "Upper bound XYZM point of the bounding box for the geo column" - : "Upper bound stored as the field's type"; - - fields.add( - optional( - baseFieldId + LOWER_BOUND.offset(), - LOWER_BOUND.fieldName(), - lowerBoundType, - lowerBoundDoc)); - fields.add( - optional( - baseFieldId + UPPER_BOUND.offset(), - UPPER_BOUND.fieldName(), - upperBoundType, - upperBoundDoc)); - - if (!isGeo && !type.isVariantType()) { - fields.add( - optional( - baseFieldId + TIGHT_BOUNDS.offset(), - TIGHT_BOUNDS.fieldName(), - Types.BooleanType.get(), - "When true, lower_bound and upper_bound must be equal to the min and max values")); - } - - fields.add( - optional( - baseFieldId + VALUE_COUNT.offset(), - VALUE_COUNT.fieldName(), - Types.LongType.get(), - "Number of values in the column (including null and NaN values)")); - - if (field.isOptional()) { - fields.add( - optional( - baseFieldId + NULL_VALUE_COUNT.offset(), - NULL_VALUE_COUNT.fieldName(), - Types.LongType.get(), - "Number of null values in the column")); - } - - if (type.typeId() == Type.TypeID.FLOAT || type.typeId() == Type.TypeID.DOUBLE) { - fields.add( - optional( - baseFieldId + NAN_VALUE_COUNT.offset(), - NAN_VALUE_COUNT.fieldName(), - Types.LongType.get(), - "Number of NaN values in the column")); - } - - if (type.typeId() == Type.TypeID.STRING || type.typeId() == Type.TypeID.BINARY || isVariant) { - fields.add( - optional( - baseFieldId + AVG_VALUE_SIZE_IN_BYTES.offset(), - AVG_VALUE_SIZE_IN_BYTES.fieldName(), - Types.IntegerType.get(), - "Avg value size in memory (uncompressed) in bytes to estimate memory consumption")); - } - - return Types.StructType.of(fields); - } - - private static Types.StructType geoLowerBoundStruct(int baseFieldId) { - return Types.StructType.of( - required( - baseFieldId + GEO_LOWER_X_OFFSET, - "x", - Types.DoubleType.get(), - "Bounding box westernmost/xmin; [-180..180]"), - required( - baseFieldId + GEO_LOWER_Y_OFFSET, - "y", - Types.DoubleType.get(), - "Bounding box southernmost/ymin; [-90..90]"), - optional( - baseFieldId + GEO_LOWER_Z_OFFSET, "z", Types.DoubleType.get(), "Bounding box zmin"), - optional( - baseFieldId + GEO_LOWER_M_OFFSET, "m", Types.DoubleType.get(), "Bounding box mmin")); - } - - private static Types.StructType geoUpperBoundStruct(int baseFieldId) { - return Types.StructType.of( - required( - baseFieldId + GEO_UPPER_X_OFFSET, - "x", - Types.DoubleType.get(), - "Bounding box easternmost/xmax; [-180..180]"), - required( - baseFieldId + GEO_UPPER_Y_OFFSET, - "y", - Types.DoubleType.get(), - "Bounding box northernmost/ymax; [-90..90]"), - optional( - baseFieldId + GEO_UPPER_Z_OFFSET, "z", Types.DoubleType.get(), "Bounding box zmax"), - optional( - baseFieldId + GEO_UPPER_M_OFFSET, "m", Types.DoubleType.get(), "Bounding box mmax")); - } -} diff --git a/core/src/main/java/org/apache/iceberg/FieldStats.java b/core/src/main/java/org/apache/iceberg/FieldStats.java index ce74dd95b9a2..dc46d11b817c 100644 --- a/core/src/main/java/org/apache/iceberg/FieldStats.java +++ b/core/src/main/java/org/apache/iceberg/FieldStats.java @@ -18,20 +18,18 @@ */ package org.apache.iceberg; -import org.apache.iceberg.types.Type; +import org.apache.iceberg.types.Types; -interface FieldStats extends StructLike { +interface FieldStats { /** The field ID of the statistic */ int fieldId(); /** - * The field type of the statistic. + * Struct type describing the stats tracked for the field identified by {@link #fieldId()}. * - *

For geo types (geometry/geography), this returns the bounding box struct type (geo_lower / - * geo_upper) rather than the column's geometry or geography type, because the type is inferred - * from the lower/upper bound schema fields. + *

Note: This type may be a projection of the stats stored in manifest files. */ - Type type(); + Types.StructType type(); /** The lower bound */ T lowerBound(); @@ -46,17 +44,20 @@ interface FieldStats extends StructLike { boolean tightBounds(); /** The total value count, including null and NaN */ - Long valueCount(); + long valueCount(); /** The total null value count */ - Long nullValueCount(); + long nullValueCount(); /** The total NaN value count */ - Long nanValueCount(); + long nanValueCount(); /** * The avg value size in memory (uncompressed) in bytes for variable-length types (string, binary, * variant) */ Integer avgValueSizeInBytes(); + + /** Returns a copy of this {@link FieldStats}. */ + FieldStats copy(); } diff --git a/core/src/main/java/org/apache/iceberg/FieldStatsStruct.java b/core/src/main/java/org/apache/iceberg/FieldStatsStruct.java new file mode 100644 index 000000000000..442873f72059 --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/FieldStatsStruct.java @@ -0,0 +1,220 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg; + +import java.io.Serializable; +import java.nio.ByteBuffer; +import java.util.List; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.types.Type; +import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.ByteBuffers; + +class FieldStatsStruct implements FieldStats, StructLike, Serializable { + private final Types.StructType struct; + private final Type boundType; + private final int[] posToOffset; + private final int fieldId; + + private Object lowerBound = null; + private Object upperBound = null; + private boolean tightBounds = false; + private Long valueCount = null; + private Long nullValueCount = null; + private Long nanValueCount = null; + private Integer avgValueSize = null; + + FieldStatsStruct(Types.StructType struct) { + this.struct = struct; + this.posToOffset = posToOffset(struct); + this.fieldId = StatsUtil.toFieldId(struct.fields().get(0).fieldId()); + this.boundType = struct.fieldType("lower_bound"); + } + + FieldStatsStruct( + Types.StructType struct, + T lowerBound, + T upperBound, + boolean tightBounds, + long valueCount, + long nullValueCount, + long nanValueCount, + Integer avgValueSize) { + this(struct); + setLowerBound(lowerBound); + setUpperBound(upperBound); + this.tightBounds = tightBounds; + this.valueCount = valueCount; + this.nullValueCount = nullValueCount; + this.nanValueCount = nanValueCount; + this.avgValueSize = avgValueSize; + } + + private FieldStatsStruct(FieldStatsStruct toCopy) { + this(toCopy.struct); + // bounds are stored using the internal representation, which is a byte array for binary types + this.lowerBound = + toCopy.lowerBound instanceof byte[] + ? ((byte[]) toCopy.lowerBound).clone() + : toCopy.lowerBound; + this.upperBound = + toCopy.upperBound instanceof byte[] + ? ((byte[]) toCopy.upperBound).clone() + : toCopy.upperBound; + this.tightBounds = toCopy.tightBounds; + this.valueCount = toCopy.valueCount; + this.nullValueCount = toCopy.nullValueCount; + this.nanValueCount = toCopy.nanValueCount; + this.avgValueSize = toCopy.avgValueSize; + } + + void fromFieldMetrics(FieldMetrics fieldMetrics) { + Preconditions.checkArgument( + fieldMetrics.id() == fieldId, + "Cannot store stats for field ID: %s (expected %s)", + fieldMetrics.id(), + fieldId); + + setLowerBound(fieldMetrics.lowerBound()); + setUpperBound(fieldMetrics.upperBound()); + this.tightBounds = false; + this.valueCount = fieldMetrics.valueCount(); + this.nullValueCount = fieldMetrics.nullValueCount() < 0 ? null : fieldMetrics.nullValueCount(); + this.nanValueCount = fieldMetrics.nanValueCount() < 0 ? null : fieldMetrics.nanValueCount(); + this.avgValueSize = null; + } + + private boolean isBinary() { + return boundType != null + && (boundType.typeId() == Type.TypeID.FIXED || boundType.typeId() == Type.TypeID.BINARY); + } + + private void setLowerBound(Object lowerBound) { + this.lowerBound = isBinary() ? ByteBuffers.toByteArray((ByteBuffer) lowerBound) : lowerBound; + } + + private void setUpperBound(Object upperBound) { + this.upperBound = isBinary() ? ByteBuffers.toByteArray((ByteBuffer) upperBound) : upperBound; + } + + @Override + public int fieldId() { + return fieldId; + } + + @Override + public Types.StructType type() { + return struct; + } + + @Override + @SuppressWarnings("unchecked") + public T lowerBound() { + return (T) + (lowerBound != null && isBinary() ? ByteBuffer.wrap((byte[]) lowerBound) : lowerBound); + } + + @Override + @SuppressWarnings("unchecked") + public T upperBound() { + return (T) + (upperBound != null && isBinary() ? ByteBuffer.wrap((byte[]) upperBound) : upperBound); + } + + @Override + public boolean tightBounds() { + return tightBounds; + } + + @Override + public long valueCount() { + return valueCount; + } + + @Override + public long nullValueCount() { + return nullValueCount; + } + + @Override + public long nanValueCount() { + return nanValueCount; + } + + @Override + public Integer avgValueSizeInBytes() { + return avgValueSize; + } + + @Override + public int size() { + return struct.fields().size(); + } + + private Object getOffset(int offset) { + return switch (offset) { + case StatsUtil.LOWER_BOUND_OFFSET -> lowerBound(); + case StatsUtil.UPPER_BOUND_OFFSET -> upperBound(); + case StatsUtil.TIGHT_BOUNDS_OFFSET -> tightBounds; + case StatsUtil.VALUE_COUNT_OFFSET -> valueCount; + case StatsUtil.NULL_VALUE_COUNT_OFFSET -> nullValueCount; + case StatsUtil.NAN_VALUE_COUNT_OFFSET -> nanValueCount; + case StatsUtil.AVG_VALUE_SIZE_OFFSET -> avgValueSize; + default -> throw new UnsupportedOperationException("Unsupported stats offset: " + offset); + }; + } + + @Override + public C get(int pos, Class javaClass) { + return javaClass.cast(getOffset(posToOffset[pos])); + } + + private void setOffset(int offset, Object value) { + switch (offset) { + case StatsUtil.LOWER_BOUND_OFFSET -> setLowerBound(value); + case StatsUtil.UPPER_BOUND_OFFSET -> setUpperBound(value); + case StatsUtil.TIGHT_BOUNDS_OFFSET -> this.tightBounds = (Boolean) value; + case StatsUtil.VALUE_COUNT_OFFSET -> this.valueCount = (Long) value; + case StatsUtil.NULL_VALUE_COUNT_OFFSET -> this.nullValueCount = (Long) value; + case StatsUtil.NAN_VALUE_COUNT_OFFSET -> this.nanValueCount = (Long) value; + case StatsUtil.AVG_VALUE_SIZE_OFFSET -> this.avgValueSize = (Integer) value; + default -> throw new UnsupportedOperationException("Unsupported stats offset: " + offset); + } + } + + @Override + public void set(int pos, C value) { + setOffset(posToOffset[pos], value); + } + + @Override + public FieldStatsStruct copy() { + return new FieldStatsStruct<>(this); + } + + private static int[] posToOffset(Types.StructType struct) { + List fields = struct.fields(); + int[] posToOffset = new int[fields.size()]; + for (int i = 0; i < posToOffset.length; i += 1) { + posToOffset[i] = StatsUtil.statOffset(fields.get(i).fieldId()); + } + + return posToOffset; + } +} diff --git a/core/src/main/java/org/apache/iceberg/MetricsUtil.java b/core/src/main/java/org/apache/iceberg/MetricsUtil.java index e6767db341b7..07dd1065db7a 100644 --- a/core/src/main/java/org/apache/iceberg/MetricsUtil.java +++ b/core/src/main/java/org/apache/iceberg/MetricsUtil.java @@ -486,7 +486,7 @@ static Map valueCounts(ContentStats stats) { Map result = Maps.newHashMap(); for (FieldStats fs : stats.fieldStats()) { - if (fs != null && fs.valueCount() != null) { + if (fs != null) { result.put(fs.fieldId(), fs.valueCount()); } } @@ -501,7 +501,7 @@ static Map nullValueCounts(ContentStats stats) { Map result = Maps.newHashMap(); for (FieldStats fs : stats.fieldStats()) { - if (fs != null && fs.nullValueCount() != null) { + if (fs != null) { result.put(fs.fieldId(), fs.nullValueCount()); } } @@ -516,8 +516,11 @@ static Map nanValueCounts(ContentStats stats) { Map result = Maps.newHashMap(); for (FieldStats fs : stats.fieldStats()) { - if (fs != null && fs.nanValueCount() != null) { - result.put(fs.fieldId(), fs.nanValueCount()); + if (fs != null) { + Type boundType = fs.type().fieldType("lower_bound"); + if (boundType.typeId() == Type.TypeID.FLOAT || boundType.typeId() == Type.TypeID.DOUBLE) { + result.put(fs.fieldId(), fs.nanValueCount()); + } } } @@ -531,8 +534,11 @@ static Map lowerBounds(ContentStats stats) { Map result = Maps.newHashMap(); for (FieldStats fs : stats.fieldStats()) { - if (fs != null && fs.lowerBound() != null && fs.type() != null) { - result.put(fs.fieldId(), Conversions.toByteBuffer(fs.type(), fs.lowerBound())); + if (fs != null) { + Type boundType = fs.type().fieldType("lower_bound"); + if (fs.lowerBound() != null && boundType != null) { + result.put(fs.fieldId(), Conversions.toByteBuffer(boundType, fs.lowerBound())); + } } } @@ -546,8 +552,11 @@ static Map upperBounds(ContentStats stats) { Map result = Maps.newHashMap(); for (FieldStats fs : stats.fieldStats()) { - if (fs != null && fs.upperBound() != null && fs.type() != null) { - result.put(fs.fieldId(), Conversions.toByteBuffer(fs.type(), fs.upperBound())); + if (fs != null) { + Type boundType = fs.type().fieldType("upper_bound"); + if (fs.upperBound() != null && boundType != null) { + result.put(fs.fieldId(), Conversions.toByteBuffer(boundType, fs.upperBound())); + } } } diff --git a/core/src/main/java/org/apache/iceberg/StatsUtil.java b/core/src/main/java/org/apache/iceberg/StatsUtil.java index 5be86b3eb147..b655ccfb0bca 100644 --- a/core/src/main/java/org/apache/iceberg/StatsUtil.java +++ b/core/src/main/java/org/apache/iceberg/StatsUtil.java @@ -19,171 +19,291 @@ package org.apache.iceberg; import static org.apache.iceberg.types.Types.NestedField.optional; +import static org.apache.iceberg.types.Types.NestedField.required; -import java.util.Collections; -import java.util.Comparator; import java.util.List; -import java.util.Objects; -import java.util.Set; -import java.util.stream.Collectors; -import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; +import java.util.Map; +import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.collect.Lists; -import org.apache.iceberg.relocated.com.google.common.collect.Sets; +import org.apache.iceberg.types.Type; import org.apache.iceberg.types.TypeUtil; import org.apache.iceberg.types.Types; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; class StatsUtil { - private static final Logger LOG = LoggerFactory.getLogger(StatsUtil.class); - static final Set SUPPORTED_METADATA_FIELD_IDS = - ImmutableSet.of( - MetadataColumns.LAST_UPDATED_SEQUENCE_NUMBER.fieldId(), MetadataColumns.ROW_ID.fieldId()); - private static final int FIRST_SUPPORTED_METADATA_FIELD_ID = - Collections.min(SUPPORTED_METADATA_FIELD_IDS); - static final int NUM_SUPPORTED_STATS_PER_COLUMN = 200; - static final int STATS_SPACE_FIELD_ID_START_FOR_METADATA_FIELDS = 9_000; - static final int STATS_SPACE_FIELD_ID_START_FOR_DATA_FIELDS = 10_000; - // exclusive upper bound of the stats field ID range reserved for content_stats - static final int STATS_SPACE_FIELD_ID_END = 200_000_000; - static final int MAX_DATA_STATS_FIELD_ID = - STATS_SPACE_FIELD_ID_END - NUM_SUPPORTED_STATS_PER_COLUMN; - // the max data field ID whose stats struct fits within the reserved range - static final int MAX_DATA_FIELD_ID = - (MAX_DATA_STATS_FIELD_ID - STATS_SPACE_FIELD_ID_START_FOR_DATA_FIELDS) - / NUM_SUPPORTED_STATS_PER_COLUMN; - private StatsUtil() {} - public static int statsFieldIdForField(int fieldId) { - return SUPPORTED_METADATA_FIELD_IDS.contains(fieldId) - ? statsFieldIdForReservedField(fieldId) - : statsFieldIdForDataField(fieldId); - } + private static final int NUM_RESERVED_FIELD_STATS_IDS = 200; + private static final int METADATA_STATS_RANGE_START = 9_000; + private static final int CONTENT_STATS_RANGE_START = 10_000; + private static final int CONTENT_STATS_RANGE_END = 200_000_000; // exclusive + + private static final int DATA_FIELD_ID_START = 0; + private static final int DATA_FIELD_ID_END = 999_950; // exclusive + + private static final int LAST_UPDATED_SEQ_NUM_BASE_ID = 9_000; + private static final int ROW_ID_BASE_ID = 9_200; + + // Offsets used for individual stats columns + static final int LOWER_BOUND_OFFSET = 1; + static final int UPPER_BOUND_OFFSET = 2; + static final int TIGHT_BOUNDS_OFFSET = 3; + static final int VALUE_COUNT_OFFSET = 4; + static final int NULL_VALUE_COUNT_OFFSET = 5; + static final int NAN_VALUE_COUNT_OFFSET = 6; + static final int AVG_VALUE_SIZE_OFFSET = 7; + + // Offsets used within geo_lower struct + private static final int GEO_LOWER_X_OFFSET = 10; + private static final int GEO_LOWER_Y_OFFSET = 11; + private static final int GEO_LOWER_Z_OFFSET = 12; + private static final int GEO_LOWER_M_OFFSET = 13; + + // Offsets used within geo_upper struct + private static final int GEO_UPPER_X_OFFSET = 14; + private static final int GEO_UPPER_Y_OFFSET = 15; + private static final int GEO_UPPER_Z_OFFSET = 16; + private static final int GEO_UPPER_M_OFFSET = 17; - private static int statsFieldIdForDataField(int fieldId) { - if (fieldId < 0 || fieldId > MAX_DATA_FIELD_ID) { - return -1; + private static final Map METADATA_ID_TO_BASE_ID = + ImmutableMap.of( + MetadataColumns.LAST_UPDATED_SEQUENCE_NUMBER.fieldId(), LAST_UPDATED_SEQ_NUM_BASE_ID, + MetadataColumns.ROW_ID.fieldId(), ROW_ID_BASE_ID); + + /** + * Return the base ID of the stats struct for the given field ID, or -1 if the ID is out of range. + * + * @param fieldId a table field ID + * @return the base ID for a field stats struct, or -1 if stats cannot be stored + */ + @VisibleForTesting + static int toBaseId(int fieldId) { + if (fieldId >= DATA_FIELD_ID_START && fieldId < DATA_FIELD_ID_END) { + return (fieldId * NUM_RESERVED_FIELD_STATS_IDS) + CONTENT_STATS_RANGE_START; } - return STATS_SPACE_FIELD_ID_START_FOR_DATA_FIELDS + (NUM_SUPPORTED_STATS_PER_COLUMN * fieldId); + return METADATA_ID_TO_BASE_ID.getOrDefault(fieldId, -1); } - private static int statsFieldIdForReservedField(int fieldId) { - return STATS_SPACE_FIELD_ID_START_FOR_METADATA_FIELDS - + (NUM_SUPPORTED_STATS_PER_COLUMN * (fieldId - FIRST_SUPPORTED_METADATA_FIELD_ID)); - } + /** + * Return the field ID corresponding to the stats field ID. + * + * @param statId the field ID of a field stats struct or field within a stats struct + * @return ID of the corresponding table field + * @throws IllegalArgumentException if the stats ID is not valid + * @throws UnsupportedOperationException if the stats ID is for an unsupported metadata field + */ + static int toFieldId(int statId) { + Preconditions.checkArgument(isValidStatId(statId), "Invalid stats field ID: %s", statId); - public static int fieldIdForStatsField(int statsFieldId) { - if (statsFieldId < STATS_SPACE_FIELD_ID_START_FOR_METADATA_FIELDS - || statsFieldId >= STATS_SPACE_FIELD_ID_END - || statsFieldId % NUM_SUPPORTED_STATS_PER_COLUMN != 0) { - return -1; + if (statId < CONTENT_STATS_RANGE_START) { + if (inBaseIdRange(LAST_UPDATED_SEQ_NUM_BASE_ID, statId)) { + return MetadataColumns.LAST_UPDATED_SEQUENCE_NUMBER.fieldId(); + } else if (inBaseIdRange(ROW_ID_BASE_ID, statId)) { + return MetadataColumns.ROW_ID.fieldId(); + } else { + throw new UnsupportedOperationException("Unsupported metadata stats field ID: " + statId); + } } - return statsFieldId < STATS_SPACE_FIELD_ID_START_FOR_DATA_FIELDS - ? fieldIdForStatsFieldFromReservedField(statsFieldId) - : fieldIdForStatsFieldFromDataField(statsFieldId); + return (statId - CONTENT_STATS_RANGE_START) / NUM_RESERVED_FIELD_STATS_IDS; } - private static int fieldIdForStatsFieldFromDataField(int statsFieldId) { - return (statsFieldId - STATS_SPACE_FIELD_ID_START_FOR_DATA_FIELDS) - / NUM_SUPPORTED_STATS_PER_COLUMN; + /** + * Return the stats offset of a stats field ID. + * + * @param statId the field ID of a field stats struct or field within a stats struct + * @return offset that identifies the stored metric, or 0 for a stats struct's ID + * @throws IllegalArgumentException if the stats ID is not valid + * @throws UnsupportedOperationException if the stats ID is for an unsupported metadata field + */ + static int statOffset(int statId) { + Preconditions.checkArgument(isValidStatId(statId), "Invalid stats field ID: %s", statId); + + return statId % 200; } - private static int fieldIdForStatsFieldFromReservedField(int statsFieldId) { - int fieldId = - (statsFieldId - STATS_SPACE_FIELD_ID_START_FOR_METADATA_FIELDS) - / NUM_SUPPORTED_STATS_PER_COLUMN - + FIRST_SUPPORTED_METADATA_FIELD_ID; - return SUPPORTED_METADATA_FIELD_IDS.contains(fieldId) ? fieldId : -1; + public static Types.NestedField contentStatsField(Types.StructType contentStats) { + return optional(146, "content_stats", contentStats); } - public static Types.NestedField contentStatsFor(Schema schema) { - ContentStatsSchemaVisitor visitor = new ContentStatsSchemaVisitor(schema); - Types.NestedField result = TypeUtil.visit(schema, visitor); - if (!visitor.skippedFieldIds.isEmpty()) { - LOG.warn("Could not create stats schema for field ids: {}", visitor.skippedFieldIds); + public static Types.StructType statsWriteSchema(Schema tableSchema, MetricsConfig metricsConfig) { + Map idToStatsName = TypeUtil.indexStatsNames(tableSchema.asStruct()); + List fieldStructs = Lists.newArrayList(); + + for (int id : metricsConfig.metricsFieldIds()) { + String fieldName = idToStatsName.get(id); + Types.NestedField field = tableSchema.findField(id); + + int baseId = toBaseId(id); + Types.StructType fieldStruct = + fieldStatsStruct(field.isOptional(), field.type(), baseId, metricsConfig.columnMode(id)); + + if (fieldStruct != null) { + fieldStructs.add(optional(baseId, fieldName, fieldStruct)); + } } - return result; + return Types.StructType.of(fieldStructs); } - private static class ContentStatsSchemaVisitor extends TypeUtil.SchemaVisitor { - private final Schema tableSchema; - private final List statsFields = Lists.newArrayList(); - private final Set skippedFieldIds = Sets.newLinkedHashSet(); + /** + * Produce a schema to read content stats for the given table field IDs. + * + * @param tableSchema a schema + * @param fieldIds an iterable of field IDs to project stats for + * @return a content stats struct for a read + */ + public static Types.StructType statsReadSchema(Schema tableSchema, Iterable fieldIds) { + Map idToStatsName = TypeUtil.indexStatsNames(tableSchema.asStruct()); + List fieldStructs = Lists.newArrayList(); - ContentStatsSchemaVisitor(Schema tableSchema) { - this.tableSchema = tableSchema; - } + for (int id : fieldIds) { + String fieldName = idToStatsName.get(id); + Types.NestedField field = tableSchema.findField(id); - @Override - public Types.NestedField schema(Schema schema, Types.NestedField structResult) { - return optional( - 146, - "content_stats", - Types.StructType.of( - statsFields.stream() - .filter(Objects::nonNull) - .sorted(Comparator.comparing(Types.NestedField::fieldId)) - .collect(Collectors.toList()))); - } + int baseId = toBaseId(id); + Types.StructType fieldStruct = + fieldStatsStruct(field.isOptional(), field.type(), baseId, MetricsModes.Full.get()); - @Override - public Types.NestedField list(Types.ListType list, Types.NestedField elementResult) { - list.fields() - .forEach( - field -> { - Types.NestedField result = field(field, null); - if (null != result) { - statsFields.add(result); - } - }); - return null; + if (fieldStruct != null) { + fieldStructs.add(optional(baseId, fieldName, fieldStruct)); + } } - @Override - public Types.NestedField map( - Types.MapType map, Types.NestedField keyResult, Types.NestedField valueResult) { - map.fields() - .forEach( - field -> { - Types.NestedField result = field(field, null); - if (null != result) { - statsFields.add(result); - } - }); - return null; - } + return Types.StructType.of(fieldStructs); + } + + /** Return whether the stat ID is valid for either a data or metadata column. */ + private static boolean isValidStatId(int statId) { + return statId >= METADATA_STATS_RANGE_START && statId < CONTENT_STATS_RANGE_END; + } + + /** Return whether the stat ID belongs to the base ID. */ + private static boolean inBaseIdRange(int baseId, int statId) { + return statId >= baseId && statId < (baseId + NUM_RESERVED_FIELD_STATS_IDS); + } - @Override - public Types.NestedField struct(Types.StructType struct, List fields) { - statsFields.addAll(fields); + private static boolean tracksTightBounds(Type type) { + return !type.isVariantType() && !isGeoType(type); + } + + private static boolean isFloatingPoint(Type type) { + return type.typeId() == Type.TypeID.FLOAT || type.typeId() == Type.TypeID.DOUBLE; + } + + private static boolean isVariableLength(Type type) { + return type.isVariantType() + || type.typeId() == Type.TypeID.STRING + || type.typeId() == Type.TypeID.BINARY + || isGeoType(type); + } + + private static boolean isGeoType(Type type) { + return type.typeId() == Type.TypeID.GEOMETRY || type.typeId() == Type.TypeID.GEOGRAPHY; + } + + private static Types.StructType geoLowerBound(int baseId) { + return Types.StructType.of( + required( + baseId + GEO_LOWER_X_OFFSET, + "x", + Types.DoubleType.get(), + "Bounding box westernmost/xmin; [-180..180]"), + required( + baseId + GEO_LOWER_Y_OFFSET, + "y", + Types.DoubleType.get(), + "Bounding box southernmost/ymin; [-90..90]"), + optional(baseId + GEO_LOWER_Z_OFFSET, "z", Types.DoubleType.get(), "Bounding box zmin"), + optional(baseId + GEO_LOWER_M_OFFSET, "m", Types.DoubleType.get(), "Bounding box mmin")); + } + + private static Types.StructType geoUpperBound(int baseId) { + return Types.StructType.of( + required( + baseId + GEO_UPPER_X_OFFSET, + "x", + Types.DoubleType.get(), + "Bounding box easternmost/xmax; [-180..180]"), + required( + baseId + GEO_UPPER_Y_OFFSET, + "y", + Types.DoubleType.get(), + "Bounding box northernmost/ymax; [-90..90]"), + optional(baseId + GEO_UPPER_Z_OFFSET, "z", Types.DoubleType.get(), "Bounding box zmax"), + optional(baseId + GEO_UPPER_M_OFFSET, "m", Types.DoubleType.get(), "Bounding box mmax")); + } + + private static Types.NestedField lowerBoundField(Type type, int baseId) { + Type boundType = isGeoType(type) ? geoLowerBound(baseId) : type; + return optional(baseId + LOWER_BOUND_OFFSET, "lower_bound", boundType); + } + + private static Types.NestedField upperBoundField(Type type, int baseId) { + Type boundType = isGeoType(type) ? geoUpperBound(baseId) : type; + return optional(baseId + UPPER_BOUND_OFFSET, "upper_bound", boundType); + } + + @VisibleForTesting + static Types.StructType fieldStatsStruct( + boolean isOptional, Type type, int baseId, MetricsModes.MetricsMode mode) { + if (null == mode + || mode == MetricsModes.None.get() + || type.isListType() + || type.isMapType() + || baseId < 0) { return null; } - @Override - public Types.NestedField field(Types.NestedField field, Types.NestedField fieldResult) { - if (field.type().isNestedType()) { - return null; - } + List fields = Lists.newArrayList(); - int fieldId = StatsUtil.statsFieldIdForField(field.fieldId()); - if (fieldId >= 0) { - Types.StructType structType = FieldStatistic.fieldStatsFor(field, fieldId); - String fullName = tableSchema.findColumnName(field.fieldId()); - return optional(fieldId, fullName, structType); - } else { - skippedFieldIds.add(field.fieldId()); + if (mode.hasBounds()) { + fields.add(lowerBoundField(type, baseId)); + fields.add(upperBoundField(type, baseId)); + + if (tracksTightBounds(type)) { + fields.add( + optional( + baseId + TIGHT_BOUNDS_OFFSET, + "tight_bounds", + Types.BooleanType.get(), + "True if lower=min and upper=max; false otherwise")); } + } - return null; + fields.add( + optional( + baseId + VALUE_COUNT_OFFSET, + "value_count", + Types.LongType.get(), + "Number of values (including null and NaN)")); + + if (isOptional) { + fields.add( + optional( + baseId + NULL_VALUE_COUNT_OFFSET, + "null_value_count", + Types.LongType.get(), + "Number of null values")); } - @Override - public Types.NestedField variant(Types.VariantType variant) { - return null; + if (isFloatingPoint(type)) { + fields.add( + optional( + baseId + NAN_VALUE_COUNT_OFFSET, + "nan_value_count", + Types.LongType.get(), + "Number of NaN values")); } + + if (isVariableLength(type)) { + fields.add( + optional( + baseId + AVG_VALUE_SIZE_OFFSET, "avg_value_size_in_bytes", Types.IntegerType.get())); + } + + return Types.StructType.of(fields); } } diff --git a/core/src/main/java/org/apache/iceberg/TrackedFileStruct.java b/core/src/main/java/org/apache/iceberg/TrackedFileStruct.java index 4c7e8cda3c50..7ff9e391b668 100644 --- a/core/src/main/java/org/apache/iceberg/TrackedFileStruct.java +++ b/core/src/main/java/org/apache/iceberg/TrackedFileStruct.java @@ -148,8 +148,8 @@ private TrackedFileStruct(TrackedFileStruct toCopy, boolean withStats, Set BaseContentStats.builder().build()) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("Either stats struct or table schema must be set"); - - assertThatThrownBy( - () -> - BaseContentStats.builder() - .withTableSchema(new Schema()) - .withStatsStruct(new Schema().asStruct()) - .build()) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("Cannot set stats struct and table schema"); - } - - @Test - public void emptyContentStats() { - BaseContentStats stats = BaseContentStats.builder().withTableSchema(new Schema()).build(); - assertThat(stats).isNotNull(); - assertThat(stats.fieldStats()).isEmpty(); - } - - @Test - public void validContentStats() { - BaseFieldStats fieldStatsOne = BaseFieldStats.builder().fieldId(1).build(); - BaseFieldStats fieldStatsTwo = BaseFieldStats.builder().fieldId(2).build(); - BaseContentStats stats = - BaseContentStats.builder() - .withTableSchema( - new Schema( - optional(1, "id", Types.IntegerType.get()), - optional(2, "id2", Types.IntegerType.get()))) - .withFieldStats(fieldStatsOne) - .withFieldStats(fieldStatsTwo) - .build(); - - assertThat(stats.fieldStats()).containsExactly(fieldStatsOne, fieldStatsTwo); - assertThat(stats.size()).isEqualTo(stats.fieldStats().size()).isEqualTo(2); - } - - @Test - public void buildFromExistingStats() { - BaseFieldStats fieldStatsOne = BaseFieldStats.builder().fieldId(1).build(); - BaseFieldStats fieldStatsTwo = BaseFieldStats.builder().fieldId(2).build(); - BaseFieldStats fieldStatsThree = BaseFieldStats.builder().fieldId(3).build(); - - BaseContentStats stats = - BaseContentStats.buildFrom( - BaseContentStats.builder() - .withTableSchema( - new Schema( - optional(1, "id", Types.IntegerType.get()), - optional(2, "id2", Types.IntegerType.get()), - optional(3, "id3", Types.IntegerType.get()))) - .withFieldStats(fieldStatsOne) - .withFieldStats(fieldStatsTwo) - .build()) - .withFieldStats(fieldStatsThree) - .build(); - assertThat(stats.fieldStats()).containsExactly(fieldStatsOne, fieldStatsTwo, fieldStatsThree); - } - - @Test - public void buildFromExistingStatsWithRequestedIds() { - BaseFieldStats fieldStatsOne = BaseFieldStats.builder().fieldId(1).build(); - BaseFieldStats fieldStatsTwo = BaseFieldStats.builder().fieldId(2).build(); - BaseFieldStats fieldStatsThree = BaseFieldStats.builder().fieldId(3).build(); - - BaseContentStats stats = - BaseContentStats.builder() - .withTableSchema( - new Schema( - optional(1, "id", Types.IntegerType.get()), - optional(2, "id2", Types.IntegerType.get()), - optional(3, "id3", Types.IntegerType.get()))) - .withFieldStats(fieldStatsOne) - .withFieldStats(fieldStatsTwo) - .withFieldStats(fieldStatsThree) - .build(); - - assertThat(BaseContentStats.buildFrom(stats, null).build()).isEqualTo(stats); - assertThat(BaseContentStats.buildFrom(stats, ImmutableSet.of(1, 3)).build().fieldStats()) - .containsExactly(fieldStatsOne, fieldStatsThree); - assertThat(BaseContentStats.buildFrom(stats, ImmutableSet.of(2)).build().fieldStats()) - .containsExactly(fieldStatsTwo); - assertThat( - BaseContentStats.buildFrom(stats, ImmutableSet.of(2, 5, 10, 12)).build().fieldStats()) - .containsExactly(fieldStatsTwo); - assertThat(BaseContentStats.buildFrom(stats, ImmutableSet.of(5, 10, 12)).build().fieldStats()) - .isEmpty(); - } - - @Test - public void retrievalByPosition() { - BaseFieldStats fieldStatsOne = BaseFieldStats.builder().fieldId(1).build(); - BaseFieldStats fieldStatsTwo = BaseFieldStats.builder().fieldId(2).build(); - BaseContentStats stats = - BaseContentStats.builder() - .withTableSchema( - new Schema( - optional(1, "id", Types.IntegerType.get()), - optional(2, "id2", Types.IntegerType.get()))) - .withFieldStats(fieldStatsOne) - .withFieldStats(fieldStatsTwo) - .build(); - - assertThat(stats.get(0, FieldStats.class)).isEqualTo(fieldStatsOne); - assertThat(stats.get(1, FieldStats.class)).isEqualTo(fieldStatsTwo); - assertThat(stats.get(2, FieldStats.class)).isNull(); - assertThat(stats.get(10, FieldStats.class)).isNull(); - - assertThatThrownBy(() -> stats.get(0, Long.class)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining( - "Wrong class, expected java.lang.Long but was org.apache.iceberg.BaseFieldStats for object:"); - } - - @Test - public void retrievalByFieldId() { - Schema schema = - new Schema( - required(1, "id", Types.IntegerType.get()), - required(2, "id2", Types.StringType.get()), - required(3, "id3", Types.DoubleType.get()), - required(4, "id4", Types.LongType.get()), - required(5, "id5", Types.FloatType.get())); - - BaseFieldStats fieldStatsTwo = - BaseFieldStats.builder() - .fieldId(2) - .type(Types.StringType.get()) - .lowerBound("aaa") - .upperBound("zzz") - .build(); - BaseFieldStats fieldStatsFive = - BaseFieldStats.builder() - .fieldId(5) - .type(Types.FloatType.get()) - .lowerBound(1.0f) - .upperBound(5.0f) - .build(); - - // table schema has 5 columns, but we only have stats for field IDs 2 and 5 and hold the stats - // in an inverse order - BaseContentStats stats = - BaseContentStats.builder() - .withTableSchema(schema) - .withFieldStats(fieldStatsFive) - .withFieldStats(fieldStatsTwo) - .build(); - - assertThat(stats.statsFor(1)).isNull(); - assertThat(stats.statsFor(2)).isEqualTo(fieldStatsTwo); - assertThat(stats.statsFor(3)).isNull(); - assertThat(stats.statsFor(4)).isNull(); - assertThat(stats.statsFor(5)).isEqualTo(fieldStatsFive); - assertThat(stats.statsFor(100)).isNull(); - } - - @Test - public void retrievalByPositionWithPartialStats() { - Schema schema = - new Schema( - required(1, "id", Types.IntegerType.get()), - required(2, "id2", Types.StringType.get()), - required(3, "id3", Types.DoubleType.get()), - required(4, "id4", Types.LongType.get()), - required(5, "id5", Types.FloatType.get())); - - BaseFieldStats fieldStatsTwo = - BaseFieldStats.builder() - .fieldId(2) - .type(Types.StringType.get()) - .lowerBound("aaa") - .upperBound("zzz") - .build(); - BaseFieldStats fieldStatsFive = - BaseFieldStats.builder() - .fieldId(5) - .type(Types.FloatType.get()) - .lowerBound(1.0f) - .upperBound(5.0f) - .build(); - - // table schema has 5 columns, but we only have stats for field IDs 2 and 5 and hold the stats - // in an inverse order - BaseContentStats stats = - BaseContentStats.builder() - .withTableSchema(schema) - .withFieldStats(fieldStatsFive) - .withFieldStats(fieldStatsTwo) - .build(); - - assertThat(stats.get(0, FieldStats.class)).isNull(); - assertThat(stats.get(1, FieldStats.class)).isEqualTo(fieldStatsTwo); - assertThat(stats.get(2, FieldStats.class)).isNull(); - assertThat(stats.get(3, FieldStats.class)).isNull(); - assertThat(stats.get(4, FieldStats.class)).isEqualTo(fieldStatsFive); - } - - @Test - public void setByPositionOptionalString() { - Schema tableSchema = new Schema(optional(1, "s", Types.StringType.get())); - Types.StructType rootStatsStruct = StatsUtil.contentStatsFor(tableSchema).type().asStructType(); - Types.StructType statsStructForFieldId = rootStatsStruct.fields().get(0).type().asStructType(); - assertThat(statsStructForFieldId.fields()).hasSize(6); - - GenericRecord record = GenericRecord.create(statsStructForFieldId); - BaseFieldStats fieldStats = - BaseFieldStats.builder() - .type(Types.StringType.get()) - .fieldId(1) - .valueCount(10L) - .nullValueCount(2L) - .avgValueSizeInBytes(3) - .lowerBound("aa") - .upperBound("zzz") - .tightBounds() - .build(); - - record.setField(LOWER_BOUND.fieldName(), fieldStats.lowerBound()); - record.setField(UPPER_BOUND.fieldName(), fieldStats.upperBound()); - record.setField(TIGHT_BOUNDS.fieldName(), fieldStats.tightBounds()); - record.setField(VALUE_COUNT.fieldName(), fieldStats.valueCount()); - record.setField(NULL_VALUE_COUNT.fieldName(), fieldStats.nullValueCount()); - record.setField(AVG_VALUE_SIZE_IN_BYTES.fieldName(), fieldStats.avgValueSizeInBytes()); - - BaseContentStats stats = new BaseContentStats(rootStatsStruct); - stats.set(0, record); - assertThat(stats.fieldStats()).containsExactly(fieldStats); - } - - @Test - public void setByPositionOptionalDouble() { - Schema tableSchema = new Schema(optional(1, "d", Types.DoubleType.get())); - Types.StructType rootStatsStruct = StatsUtil.contentStatsFor(tableSchema).type().asStructType(); - Types.StructType statsStructForFieldId = rootStatsStruct.fields().get(0).type().asStructType(); - assertThat(statsStructForFieldId.fields()).hasSize(6); - - GenericRecord record = GenericRecord.create(statsStructForFieldId); - BaseFieldStats fieldStats = - BaseFieldStats.builder() - .type(Types.DoubleType.get()) - .fieldId(1) - .valueCount(10L) - .nullValueCount(2L) - .nanValueCount(3L) - .lowerBound(5.0) - .upperBound(20.0) - .tightBounds() - .build(); - - record.setField(LOWER_BOUND.fieldName(), fieldStats.lowerBound()); - record.setField(UPPER_BOUND.fieldName(), fieldStats.upperBound()); - record.setField(TIGHT_BOUNDS.fieldName(), fieldStats.tightBounds()); - record.setField(VALUE_COUNT.fieldName(), fieldStats.valueCount()); - record.setField(NULL_VALUE_COUNT.fieldName(), fieldStats.nullValueCount()); - record.setField(NAN_VALUE_COUNT.fieldName(), fieldStats.nanValueCount()); - - BaseContentStats stats = new BaseContentStats(rootStatsStruct); - stats.set(0, record); - assertThat(stats.fieldStats()).containsExactly(fieldStats); - } - - @Test - public void setByPositionRequiredInteger() { - Schema tableSchema = new Schema(required(1, "id", Types.IntegerType.get())); - Types.StructType rootStatsStruct = StatsUtil.contentStatsFor(tableSchema).type().asStructType(); - Types.StructType statsStructForFieldId = rootStatsStruct.fields().get(0).type().asStructType(); - assertThat(statsStructForFieldId.fields()).hasSize(4); - - GenericRecord record = GenericRecord.create(statsStructForFieldId); - BaseFieldStats fieldStats = - BaseFieldStats.builder() - .type(Types.IntegerType.get()) - .fieldId(1) - .valueCount(10L) - .lowerBound(5) - .upperBound(20) - .tightBounds() - .build(); - - record.setField(LOWER_BOUND.fieldName(), fieldStats.lowerBound()); - record.setField(UPPER_BOUND.fieldName(), fieldStats.upperBound()); - record.setField(TIGHT_BOUNDS.fieldName(), fieldStats.tightBounds()); - record.setField(VALUE_COUNT.fieldName(), fieldStats.valueCount()); - - // this is typically called by Avro reflection code - BaseContentStats stats = new BaseContentStats(rootStatsStruct); - stats.set(0, record); - assertThat(stats.fieldStats()).containsExactly(fieldStats); - } - - @Test - public void setByPositionOptionalGeometry() { - Schema tableSchema = new Schema(optional(1, "g", Types.GeometryType.crs84())); - Types.StructType rootStatsStruct = StatsUtil.contentStatsFor(tableSchema).type().asStructType(); - Types.StructType statsStructForFieldId = rootStatsStruct.fields().get(0).type().asStructType(); - // lower_bound, upper_bound, value_count, null_value_count - assertThat(statsStructForFieldId.fields()).hasSize(4); - - GenericRecord lower = - GenericRecord.create( - statsStructForFieldId.field(LOWER_BOUND.fieldName()).type().asStructType()); - lower.setField("x", -122.4); - lower.setField("y", 37.7); - lower.setField("z", null); - lower.setField("m", null); - - GenericRecord upper = - GenericRecord.create( - statsStructForFieldId.field(UPPER_BOUND.fieldName()).type().asStructType()); - upper.setField("x", -122.0); - upper.setField("y", 38.0); - upper.setField("z", null); - upper.setField("m", null); - - GenericRecord record = GenericRecord.create(statsStructForFieldId); - record.setField(LOWER_BOUND.fieldName(), lower); - record.setField(UPPER_BOUND.fieldName(), upper); - record.setField(VALUE_COUNT.fieldName(), 100L); - record.setField(NULL_VALUE_COUNT.fieldName(), 5L); - - ContentStats stats = new BaseContentStats(rootStatsStruct); - stats.set(0, record); - - FieldStats result = stats.fieldStats().get(0); - assertThat(result.valueCount()).isEqualTo(100L); - assertThat(result.nullValueCount()).isEqualTo(5L); - assertThat(result.tightBounds()).isFalse(); - assertThat(result.lowerBound()).isEqualTo(lower); - assertThat(result.upperBound()).isEqualTo(upper); - } - - @Test - public void setByPositionOptionalGeography() { - Schema tableSchema = new Schema(optional(1, "g", Types.GeographyType.crs84())); - Types.StructType rootStatsStruct = StatsUtil.contentStatsFor(tableSchema).type().asStructType(); - Types.StructType statsStructForFieldId = rootStatsStruct.fields().get(0).type().asStructType(); - // lower_bound, upper_bound, value_count, null_value_count - assertThat(statsStructForFieldId.fields()).hasSize(4); - - GenericRecord lower = - GenericRecord.create( - statsStructForFieldId.field(LOWER_BOUND.fieldName()).type().asStructType()); - lower.setField("x", 10.0); - lower.setField("y", 20.0); - lower.setField("z", 0.0); - lower.setField("m", null); - - GenericRecord upper = - GenericRecord.create( - statsStructForFieldId.field(UPPER_BOUND.fieldName()).type().asStructType()); - upper.setField("x", 30.0); - upper.setField("y", 40.0); - upper.setField("z", 100.0); - upper.setField("m", null); - - GenericRecord record = GenericRecord.create(statsStructForFieldId); - record.setField(LOWER_BOUND.fieldName(), lower); - record.setField(UPPER_BOUND.fieldName(), upper); - record.setField(VALUE_COUNT.fieldName(), 200L); - record.setField(NULL_VALUE_COUNT.fieldName(), 10L); - - ContentStats stats = new BaseContentStats(rootStatsStruct); - stats.set(0, record); - - FieldStats result = stats.fieldStats().get(0); - assertThat(result.valueCount()).isEqualTo(200L); - assertThat(result.nullValueCount()).isEqualTo(10L); - assertThat(result.tightBounds()).isFalse(); - assertThat(result.lowerBound()).isEqualTo(lower); - assertThat(result.upperBound()).isEqualTo(upper); - } - - @Test - public void setByPositionRequiredVariant() { - Schema tableSchema = new Schema(required(1, "v", Types.VariantType.get())); - Types.StructType rootStatsStruct = StatsUtil.contentStatsFor(tableSchema).type().asStructType(); - Types.StructType statsStructForFieldId = rootStatsStruct.fields().get(0).type().asStructType(); - // lower_bound, upper_bound, value_count, avg_value_size_in_bytes - assertThat(statsStructForFieldId.fields()).hasSize(4); - - VariantMetadata metadata = Variants.metadata("$['name']", "$['score']"); - ShreddedObject lower = Variants.object(metadata); - lower.put("$['name']", Variants.of("alice")); - lower.put("$['score']", Variants.of(1)); - Variant lowerVariant = Variant.of(metadata, lower); - - ShreddedObject upper = Variants.object(metadata); - upper.put("$['name']", Variants.of("zara")); - upper.put("$['score']", Variants.of(100)); - Variant upperVariant = Variant.of(metadata, upper); - - GenericRecord record = GenericRecord.create(statsStructForFieldId); - record.setField(LOWER_BOUND.fieldName(), lowerVariant); - record.setField(UPPER_BOUND.fieldName(), upperVariant); - record.setField(VALUE_COUNT.fieldName(), 50L); - record.setField(AVG_VALUE_SIZE_IN_BYTES.fieldName(), 128); - - ContentStats stats = new BaseContentStats(rootStatsStruct); - stats.set(0, record); - - FieldStats result = stats.fieldStats().get(0); - assertThat(result.valueCount()).isEqualTo(50L); - assertThat(result.avgValueSizeInBytes()).isEqualTo(128); - assertThat(result.tightBounds()).isFalse(); - assertThat(result.lowerBound()).isEqualTo(lowerVariant); - assertThat(result.upperBound()).isEqualTo(upperVariant); - } - - @Test - public void setByPositionWithInvalidLowerAndUpperBound() { - Schema tableSchema = new Schema(required(1, "id", Types.IntegerType.get())); - Types.StructType rootStatsStruct = StatsUtil.contentStatsFor(tableSchema).type().asStructType(); - Types.StructType statsStructForIdField = rootStatsStruct.fields().get(0).type().asStructType(); - - GenericRecord record = GenericRecord.create(statsStructForIdField); - // this is typically called by Avro reflection code - BaseContentStats stats = new BaseContentStats(rootStatsStruct); - - // invalid lower bound - record.setField(LOWER_BOUND.fieldName(), 5.0); - assertThatThrownBy(() -> stats.set(0, record)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage( - "Invalid lower bound type, expected a subtype of class java.lang.Integer: java.lang.Double"); - - // set valid lower bound so that upper bound is evaluated - record.setField(LOWER_BOUND.fieldName(), 5); - - // invalid upper bound - record.setField(UPPER_BOUND.fieldName(), "20"); - assertThatThrownBy(() -> stats.set(0, record)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage( - "Invalid upper bound type, expected a subtype of class java.lang.Integer: java.lang.String"); - } -} diff --git a/core/src/test/java/org/apache/iceberg/TestContentStatsStruct.java b/core/src/test/java/org/apache/iceberg/TestContentStatsStruct.java new file mode 100644 index 000000000000..51b5343494b1 --- /dev/null +++ b/core/src/test/java/org/apache/iceberg/TestContentStatsStruct.java @@ -0,0 +1,261 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg; + +import static org.apache.iceberg.types.Types.NestedField.optional; +import static org.apache.iceberg.types.Types.NestedField.required; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.io.IOException; +import java.util.Comparator; +import java.util.List; +import org.apache.iceberg.TestHelpers.RoundTripSerializer; +import org.apache.iceberg.inmemory.InMemoryOutputFile; +import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.io.FileAppender; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; +import org.apache.iceberg.relocated.com.google.common.collect.Iterables; +import org.apache.iceberg.types.Comparators; +import org.apache.iceberg.types.TypeUtil; +import org.apache.iceberg.types.Types; +import org.junit.jupiter.api.Named; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.FieldSource; +import org.mockito.Mockito; + +public class TestContentStatsStruct { + private static final Schema SCHEMA = + new Schema( + required(1, "id", Types.LongType.get()), + optional(2, "data", Types.StringType.get()), + optional(3, "score", Types.DoubleType.get())); + + private static final Types.StructType CONTENT_STATS_STRUCT = + StatsUtil.statsReadSchema(SCHEMA, ImmutableList.of(1, 2, 3)); + private static final Types.StructType UNKNOWN_FIELD_STATS_STRUCT = + StatsUtil.fieldStatsStruct(false, Types.IntegerType.get(), 10_800, MetricsModes.Full.get()); + + private static final FieldStats ID_STATS = + new FieldStatsStruct<>( + CONTENT_STATS_STRUCT.field("id").type().asStructType(), 0L, 25L, true, 26L, 0L, 0L, null); + private static final FieldStats DATA_STATS = + new FieldStatsStruct<>( + CONTENT_STATS_STRUCT.field("data").type().asStructType(), + "a", + "z", + true, + 26L, + 0L, + 0L, + null); + + @Test + public void testEmptyContentStats() { + ContentStats stats = new ContentStatsStruct(CONTENT_STATS_STRUCT); + + assertThat(stats.statsFor(1)).isNull(); + assertThat(stats.statsFor(2)).isNull(); + assertThat(stats.statsFor(3)).isNull(); + assertThat(stats.statsFor(4)).as("Should ignore unknown field IDs").isNull(); + } + + @Test + public void testSetStats() { + ContentStatsStruct stats = new ContentStatsStruct(CONTENT_STATS_STRUCT); + + stats.setStats(1, ID_STATS); + stats.setStats(2, DATA_STATS); + + assertThat(stats.statsFor(1)).isEqualTo(ID_STATS); + assertThat(stats.statsFor(2)).isEqualTo(DATA_STATS); + assertThat(stats.statsFor(3)).isNull(); + } + + @Test + public void testSetStatsWrongId() { + ContentStatsStruct stats = new ContentStatsStruct(CONTENT_STATS_STRUCT); + + assertThatThrownBy(() -> stats.setStats(2, ID_STATS)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Mismatched field stats for ID 2: actual ID 1"); + } + + @Test + public void testSetStatsUnknownField() { + ContentStatsStruct stats = new ContentStatsStruct(CONTENT_STATS_STRUCT); + + FieldStats fieldStats = + new FieldStatsStruct<>(UNKNOWN_FIELD_STATS_STRUCT, 0, 10, false, 8, 3, 0, null); + + assertThatThrownBy(() -> stats.setStats(4, fieldStats)) + .hasMessage("Cannot set stats for unknown field ID: 4") + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + public void testGetByPosition() { + // the content stats struct with a known field order + Types.StructType contentStatsStruct = + Types.StructType.of( + CONTENT_STATS_STRUCT.field("data"), + CONTENT_STATS_STRUCT.field("score"), + CONTENT_STATS_STRUCT.field("id")); + + ContentStatsStruct stats = new ContentStatsStruct(contentStatsStruct); + + stats.setStats(1, ID_STATS); + stats.setStats(2, DATA_STATS); + + assertThat(stats.get(0, FieldStats.class)).isEqualTo(DATA_STATS); + assertThat(stats.get(1, FieldStats.class)).isNull(); + assertThat(stats.get(2, FieldStats.class)).isEqualTo(ID_STATS); + } + + @Test + public void testSetByPosition() { + // the content stats struct with a known field order + Types.StructType statsStruct = + Types.StructType.of( + CONTENT_STATS_STRUCT.field("data"), + CONTENT_STATS_STRUCT.field("score"), + CONTENT_STATS_STRUCT.field("id")); + + ContentStatsStruct stats = new ContentStatsStruct(statsStruct); + + stats.set(1, null); + stats.set(2, ID_STATS); + stats.set(0, DATA_STATS); + + assertThat(stats.statsFor(SCHEMA.findField("id").fieldId())).isEqualTo(ID_STATS); + assertThat(stats.statsFor(SCHEMA.findField("data").fieldId())).isEqualTo(DATA_STATS); + assertThat(stats.statsFor(SCHEMA.findField("score").fieldId())).isNull(); + } + + @Test + public void testSize() { + ContentStatsStruct stats = new ContentStatsStruct(CONTENT_STATS_STRUCT); + + assertThat(stats.size()).isEqualTo(3); + } + + @Test + @SuppressWarnings("unchecked") + public void testCopy() { + FieldStats idStats = Mockito.mock(FieldStats.class); + FieldStats idStatsCopy = Mockito.mock(FieldStats.class); + Mockito.when(idStats.fieldId()).thenReturn(1); + Mockito.when(idStats.copy()).thenReturn(idStatsCopy); + + FieldStats dataStats = Mockito.mock(FieldStats.class); + FieldStats dataStatsCopy = Mockito.mock(FieldStats.class); + Mockito.when(dataStats.fieldId()).thenReturn(2); + Mockito.when(dataStats.copy()).thenReturn(dataStatsCopy); + + ContentStatsStruct stats = new ContentStatsStruct(CONTENT_STATS_STRUCT); + stats.setStats(1, idStats); + stats.setStats(2, dataStats); + + ContentStats copy = stats.copy(); + + assertThat(copy).isInstanceOf(ContentStatsStruct.class).isNotSameAs(stats); + assertThat(copy.type()).isEqualTo(stats.type()); + + // each field stats is deep-copied using its own copy() method + assertThat(copy.statsFor(1)).isSameAs(idStatsCopy); + assertThat(copy.statsFor(2)).isSameAs(dataStatsCopy); + assertThat(copy.statsFor(3)).isNull(); + } + + @Test + @SuppressWarnings("unchecked") + public void testFilteredCopy() { + FieldStats idStats = Mockito.mock(FieldStats.class); + FieldStats idStatsCopy = Mockito.mock(FieldStats.class); + Mockito.when(idStats.fieldId()).thenReturn(1); + Mockito.when(idStats.copy()).thenReturn(idStatsCopy); + + FieldStats dataStats = Mockito.mock(FieldStats.class); + FieldStats dataStatsCopy = Mockito.mock(FieldStats.class); + Mockito.when(dataStats.fieldId()).thenReturn(2); + Mockito.when(dataStats.copy()).thenReturn(dataStatsCopy); + + ContentStatsStruct stats = new ContentStatsStruct(CONTENT_STATS_STRUCT); + stats.setStats(1, idStats); + stats.setStats(2, dataStats); + + // copy just the stats for field ID 2 + ContentStats copy = stats.copy(ImmutableSet.of(2)); + + assertThat(copy).isInstanceOf(ContentStatsStruct.class).isNotSameAs(stats); + assertThat(copy.type()).isEqualTo(TypeUtil.select(stats.type(), ImmutableSet.of(1))); + + // each field stats is deep-copied using its own copy() method + assertThat(copy.statsFor(1)).isNull(); + assertThat(copy.statsFor(2)).isSameAs(dataStatsCopy); + assertThat(copy.statsFor(3)).isNull(); + } + + private static final List>> SERIALIZERS = + List.of( + Named.of("Java", TestHelpers::roundTripSerialize), + Named.of("Kryo", TestHelpers.KryoHelpers::roundTripSerialize), + Named.of("InternalData", TestContentStatsStruct::roundTripInternalData)); + + @ParameterizedTest + @FieldSource("SERIALIZERS") + public void testSerialization(RoundTripSerializer serializer) + throws Exception { + ContentStatsStruct stats = new ContentStatsStruct(CONTENT_STATS_STRUCT); + stats.setStats(1, ID_STATS); + stats.setStats(2, DATA_STATS); + + Comparator comparator = Comparators.forType(CONTENT_STATS_STRUCT); + + ContentStatsStruct copy = serializer.apply(stats); + assertThat(copy.type()).isEqualTo(stats.type()); + assertThat(comparator.compare(copy, stats)).isEqualTo(0); + } + + private static ContentStatsStruct roundTripInternalData(ContentStatsStruct stats) + throws IOException { + Schema schema = stats.type().asSchema(); + InMemoryOutputFile file = new InMemoryOutputFile("internal.avro"); + + try (FileAppender writer = + InternalData.write(FileFormat.AVRO, file).schema(schema).build()) { + writer.add(stats); + } + + InternalData.ReadBuilder read = + InternalData.read(FileFormat.AVRO, file.toInputFile()) + .project(schema) + .setRootType(ContentStatsStruct.class); + // the nested field stats structs are read as FieldStatsStruct + for (Types.NestedField field : stats.type().fields()) { + read = read.setCustomType(field.fieldId(), FieldStatsStruct.class); + } + + try (CloseableIterable reader = read.build()) { + return Iterables.getOnlyElement(reader); + } + } +} diff --git a/core/src/test/java/org/apache/iceberg/TestFieldStats.java b/core/src/test/java/org/apache/iceberg/TestFieldStats.java deleted file mode 100644 index 35be3ed36bcf..000000000000 --- a/core/src/test/java/org/apache/iceberg/TestFieldStats.java +++ /dev/null @@ -1,244 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iceberg; - -import static org.apache.iceberg.FieldStatistic.AVG_VALUE_SIZE_IN_BYTES; -import static org.apache.iceberg.FieldStatistic.LOWER_BOUND; -import static org.apache.iceberg.FieldStatistic.NAN_VALUE_COUNT; -import static org.apache.iceberg.FieldStatistic.NULL_VALUE_COUNT; -import static org.apache.iceberg.FieldStatistic.TIGHT_BOUNDS; -import static org.apache.iceberg.FieldStatistic.UPPER_BOUND; -import static org.apache.iceberg.FieldStatistic.VALUE_COUNT; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.util.stream.Stream; -import org.apache.iceberg.types.Type; -import org.apache.iceberg.types.Types; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; - -public class TestFieldStats { - - @Test - public void empty() { - BaseFieldStats empty = BaseFieldStats.builder().build(); - assertThat(empty.fieldId()).isEqualTo(0); - assertThat(empty.type()).isNull(); - assertThat(empty.lowerBound()).isNull(); - assertThat(empty.upperBound()).isNull(); - assertThat(empty.tightBounds()).isFalse(); - assertThat(empty.valueCount()).isNull(); - assertThat(empty.nullValueCount()).isNull(); - assertThat(empty.nanValueCount()).isNull(); - assertThat(empty.avgValueSizeInBytes()).isNull(); - } - - @Test - public void validIndividualValues() { - BaseFieldStats fieldStats = - BaseFieldStats.builder() - .type(Types.IntegerType.get()) - .fieldId(23) - .valueCount(10L) - .nullValueCount(2L) - .nanValueCount(3L) - .avgValueSizeInBytes(30) - .lowerBound(5) - .upperBound(20) - .tightBounds() - .build(); - - assertThat(fieldStats.type()).isEqualTo(Types.IntegerType.get()); - assertThat(fieldStats.fieldId()).isEqualTo(23); - assertThat(fieldStats.valueCount()).isEqualTo(10L); - assertThat(fieldStats.nullValueCount()).isEqualTo(2L); - assertThat(fieldStats.nanValueCount()).isEqualTo(3L); - assertThat(fieldStats.avgValueSizeInBytes()).isEqualTo(30); - assertThat(fieldStats.lowerBound()).isEqualTo(5); - assertThat(fieldStats.upperBound()).isEqualTo(20); - assertThat(fieldStats.tightBounds()).isTrue(); - } - - @Test - public void buildFromExistingStats() { - BaseFieldStats fieldStats = - BaseFieldStats.buildFrom( - BaseFieldStats.builder() - .type(Types.IntegerType.get()) - .fieldId(23) - .valueCount(10L) - .nullValueCount(2L) - .nanValueCount(3L) - .avgValueSizeInBytes(30) - .lowerBound(5) - .upperBound(20) - .build()) - .lowerBound(2) - .upperBound(50) - .avgValueSizeInBytes(90) - .tightBounds() - .build(); - assertThat(fieldStats.type()).isEqualTo(Types.IntegerType.get()); - assertThat(fieldStats.fieldId()).isEqualTo(23); - assertThat(fieldStats.valueCount()).isEqualTo(10L); - assertThat(fieldStats.nullValueCount()).isEqualTo(2L); - assertThat(fieldStats.nanValueCount()).isEqualTo(3L); - assertThat(fieldStats.avgValueSizeInBytes()).isEqualTo(90); - assertThat(fieldStats.lowerBound()).isEqualTo(2); - assertThat(fieldStats.upperBound()).isEqualTo(50); - assertThat(fieldStats.tightBounds()).isTrue(); - } - - @Test - public void validFieldStats() { - assertThat(BaseFieldStats.builder().build()).isNotNull(); - assertThat(BaseFieldStats.builder().fieldId(1).build()).isNotNull(); - assertThat(BaseFieldStats.builder().valueCount(3L).build()).isNotNull(); - assertThat(BaseFieldStats.builder().nullValueCount(3L).build()).isNotNull(); - assertThat(BaseFieldStats.builder().nanValueCount(3L).build()).isNotNull(); - assertThat(BaseFieldStats.builder().type(Types.IntegerType.get()).build()).isNotNull(); - assertThat(BaseFieldStats.builder().avgValueSizeInBytes(3).build()).isNotNull(); - - assertThat(BaseFieldStats.builder().type(Types.LongType.get()).lowerBound(3L).build()) - .isNotNull(); - assertThat(BaseFieldStats.builder().type(Types.LongType.get()).upperBound(10L).build()) - .isNotNull(); - assertThat( - BaseFieldStats.builder() - .type(Types.LongType.get()) - .lowerBound(3L) - .upperBound(10L) - .build()) - .isNotNull(); - assertThat( - BaseFieldStats.builder() - .type(Types.LongType.get()) - .lowerBound(3L) - .upperBound(10L) - .build()) - .isNotNull(); - } - - @Test - public void missingTypeWithUpperOrLowerBound() { - assertThatThrownBy(() -> BaseFieldStats.builder().lowerBound(3).build()) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("Invalid type (required when lower bound is set): null"); - assertThatThrownBy(() -> BaseFieldStats.builder().upperBound(3).build()) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("Invalid type (required when lower bound is set): null"); - } - - @Test - public void invalidType() { - assertThatThrownBy( - () -> BaseFieldStats.builder().type(Types.LongType.get()).lowerBound(3).build()) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage( - "Invalid lower bound type, expected a subtype of java.lang.Long: java.lang.Integer"); - assertThatThrownBy( - () -> - BaseFieldStats.builder().type(Types.LongType.get()).lowerBound(3).build()) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage( - "Invalid lower bound type, expected a subtype of java.lang.Long: java.lang.Integer"); - - assertThatThrownBy( - () -> BaseFieldStats.builder().type(Types.LongType.get()).upperBound(3).build()) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage( - "Invalid upper bound type, expected a subtype of java.lang.Long: java.lang.Integer"); - assertThatThrownBy( - () -> - BaseFieldStats.builder().type(Types.LongType.get()).upperBound(3).build()) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage( - "Invalid upper bound type, expected a subtype of java.lang.Long: java.lang.Integer"); - } - - @Test - public void retrievalByPosition() { - BaseFieldStats fieldStats = - BaseFieldStats.builder() - .type(Types.IntegerType.get()) - .fieldId(23) - .valueCount(10L) - .nullValueCount(2L) - .nanValueCount(3L) - .avgValueSizeInBytes(30) - .lowerBound(5) - .upperBound(20) - .tightBounds() - .build(); - - assertThat(fieldStats.get(LOWER_BOUND.position(), Integer.class)).isEqualTo(5); - assertThat(fieldStats.get(UPPER_BOUND.position(), Integer.class)).isEqualTo(20); - assertThat(fieldStats.get(TIGHT_BOUNDS.position(), Boolean.class)).isEqualTo(true); - assertThat(fieldStats.get(VALUE_COUNT.position(), Long.class)).isEqualTo(10L); - assertThat(fieldStats.get(NULL_VALUE_COUNT.position(), Long.class)).isEqualTo(2L); - assertThat(fieldStats.get(NAN_VALUE_COUNT.position(), Long.class)).isEqualTo(3L); - assertThat(fieldStats.get(AVG_VALUE_SIZE_IN_BYTES.position(), Integer.class)).isEqualTo(30); - - assertThatThrownBy(() -> assertThat(fieldStats.get(10, Long.class))) - .isInstanceOf(ArrayIndexOutOfBoundsException.class) - .hasMessage("Index 10 out of bounds for length 7"); - assertThatThrownBy(() -> assertThat(fieldStats.get(VALUE_COUNT.position(), Double.class))) - .isInstanceOf(ClassCastException.class) - .hasMessage("Cannot cast java.lang.Long to java.lang.Double"); - assertThatThrownBy( - () -> assertThat(fieldStats.get(AVG_VALUE_SIZE_IN_BYTES.position(), Long.class))) - .isInstanceOf(ClassCastException.class) - .hasMessage("Cannot cast java.lang.Integer to java.lang.Long"); - } - - private static Stream binaryTypes() { - return Stream.of( - Arguments.of(Types.BinaryType.get()), Arguments.of(Types.FixedType.ofLength(3))); - } - - @ParameterizedTest - @MethodSource("binaryTypes") - public void statsForBinaryTypes(Type binaryType) throws IOException, ClassNotFoundException { - BaseFieldStats statsWithByteBuffer = - BaseFieldStats.builder() - .type(binaryType) - .lowerBound(ByteBuffer.wrap("AAA".getBytes(StandardCharsets.UTF_8))) - .upperBound(ByteBuffer.wrap("ZZZ".getBytes(StandardCharsets.UTF_8))) - .build(); - assertThat(TestHelpers.roundTripSerialize(statsWithByteBuffer)).isEqualTo(statsWithByteBuffer); - - BaseFieldStats statsWithByteArray = - BaseFieldStats.builder() - .type(binaryType) - .lowerBound("AAA".getBytes(StandardCharsets.UTF_8)) - .upperBound("ZZZ".getBytes(StandardCharsets.UTF_8)) - .build(); - assertThat(TestHelpers.roundTripSerialize(statsWithByteArray)).isEqualTo(statsWithByteArray); - - assertThat(statsWithByteArray) - .as("both field stats should produce the same upper/lower bound values") - .isEqualTo(statsWithByteBuffer); - } -} diff --git a/core/src/test/java/org/apache/iceberg/TestFieldStatsStruct.java b/core/src/test/java/org/apache/iceberg/TestFieldStatsStruct.java new file mode 100644 index 000000000000..fc136a372a3a --- /dev/null +++ b/core/src/test/java/org/apache/iceberg/TestFieldStatsStruct.java @@ -0,0 +1,414 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg; + +import static org.apache.iceberg.types.Types.NestedField.optional; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.io.IOException; +import java.math.BigDecimal; +import java.nio.ByteBuffer; +import java.util.Comparator; +import java.util.List; +import java.util.UUID; +import java.util.stream.Stream; +import org.apache.iceberg.TestHelpers.RoundTripSerializer; +import org.apache.iceberg.inmemory.InMemoryOutputFile; +import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.io.FileAppender; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.Iterables; +import org.apache.iceberg.types.Comparators; +import org.apache.iceberg.types.Type; +import org.apache.iceberg.types.Types; +import org.apache.iceberg.variants.ShreddedObject; +import org.apache.iceberg.variants.Variant; +import org.apache.iceberg.variants.VariantMetadata; +import org.apache.iceberg.variants.VariantTestUtil; +import org.apache.iceberg.variants.Variants; +import org.junit.jupiter.api.Named; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.FieldSource; +import org.junit.jupiter.params.provider.MethodSource; + +public class TestFieldStatsStruct { + private static final int BASE_ID = 30_000; + + private static final Types.StructType STRING_STATS = + StatsUtil.fieldStatsStruct(true, Types.StringType.get(), BASE_ID, MetricsModes.Full.get()); + + private static final Types.StructType DOUBLE_STATS = + StatsUtil.fieldStatsStruct(true, Types.DoubleType.get(), BASE_ID, MetricsModes.Full.get()); + + @Test + public void testFieldAccess() { + FieldStats stats = new FieldStatsStruct<>(STRING_STATS, "a", "z", true, 28, 2, 0, 1); + + assertThat(stats.fieldId()).isEqualTo(100); + assertThat(stats.lowerBound()).isEqualTo("a"); + assertThat(stats.upperBound()).isEqualTo("z"); + assertThat(stats.tightBounds()).isTrue(); + assertThat(stats.valueCount()).isEqualTo(28L); + assertThat(stats.nullValueCount()).isEqualTo(2L); + assertThat(stats.nanValueCount()).isEqualTo(0L); + assertThat(stats.avgValueSizeInBytes()).isEqualTo(1); + } + + @Test + public void testStringGetByPosition() { + FieldStatsStruct stats = + new FieldStatsStruct<>(STRING_STATS, "a", "z", true, 28, 2, 0, 1); + + assertThat(stats.get(pos(STRING_STATS, "lower_bound"), String.class)).isEqualTo("a"); + assertThat(stats.get(pos(STRING_STATS, "upper_bound"), String.class)).isEqualTo("z"); + assertThat(stats.get(pos(STRING_STATS, "tight_bounds"), Boolean.class)).isTrue(); + assertThat(stats.get(pos(STRING_STATS, "value_count"), Long.class)).isEqualTo(28L); + assertThat(stats.get(pos(STRING_STATS, "null_value_count"), Long.class)).isEqualTo(2L); + assertThat(stats.get(pos(STRING_STATS, "avg_value_size_in_bytes"), Integer.class)).isEqualTo(1); + } + + @Test + public void testStringSetByPosition() { + FieldStatsStruct stats = new FieldStatsStruct<>(STRING_STATS); + + stats.set(pos(STRING_STATS, "lower_bound"), "a"); + stats.set(pos(STRING_STATS, "upper_bound"), "z"); + stats.set(pos(STRING_STATS, "tight_bounds"), false); + stats.set(pos(STRING_STATS, "value_count"), 28L); + stats.set(pos(STRING_STATS, "null_value_count"), 2L); + stats.set(pos(STRING_STATS, "avg_value_size_in_bytes"), 1); + + assertThat(stats.lowerBound()).isEqualTo("a"); + assertThat(stats.upperBound()).isEqualTo("z"); + assertThat(stats.tightBounds()).isFalse(); + assertThat(stats.valueCount()).isEqualTo(28L); + assertThat(stats.nullValueCount()).isEqualTo(2L); + assertThat(stats.avgValueSizeInBytes()).isEqualTo(1); + } + + @Test + public void testDoubleGetByPosition() { + FieldStatsStruct stats = + new FieldStatsStruct<>(DOUBLE_STATS, 0.0d, 25.0d, true, 34, 2, 6, 0); + + assertThat(stats.get(pos(DOUBLE_STATS, "lower_bound"), Double.class)).isEqualTo(0.0d); + assertThat(stats.get(pos(DOUBLE_STATS, "upper_bound"), Double.class)).isEqualTo(25.0d); + assertThat(stats.get(pos(DOUBLE_STATS, "tight_bounds"), Boolean.class)).isTrue(); + assertThat(stats.get(pos(DOUBLE_STATS, "value_count"), Long.class)).isEqualTo(34L); + assertThat(stats.get(pos(DOUBLE_STATS, "null_value_count"), Long.class)).isEqualTo(2L); + assertThat(stats.get(pos(DOUBLE_STATS, "nan_value_count"), Long.class)).isEqualTo(6L); + } + + @Test + public void testDoubleSetByPosition() { + FieldStatsStruct stats = new FieldStatsStruct<>(DOUBLE_STATS); + + stats.set(pos(DOUBLE_STATS, "lower_bound"), 0.0d); + stats.set(pos(DOUBLE_STATS, "upper_bound"), 25.0d); + stats.set(pos(DOUBLE_STATS, "tight_bounds"), false); + stats.set(pos(DOUBLE_STATS, "value_count"), 34L); + stats.set(pos(DOUBLE_STATS, "null_value_count"), 2L); + stats.set(pos(DOUBLE_STATS, "nan_value_count"), 6L); + + assertThat(stats.lowerBound()).isEqualTo(0.0d); + assertThat(stats.upperBound()).isEqualTo(25.0d); + assertThat(stats.tightBounds()).isFalse(); + assertThat(stats.valueCount()).isEqualTo(34L); + assertThat(stats.nullValueCount()).isEqualTo(2L); + assertThat(stats.nanValueCount()).isEqualTo(6L); + assertThat(stats.avgValueSizeInBytes()).isNull(); + } + + @Test + public void testFromFieldMetricsWrongField() { + FieldStatsStruct stats = new FieldStatsStruct<>(STRING_STATS); + assertThatThrownBy(() -> stats.fromFieldMetrics(new FieldMetrics<>(21, 50, 0))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Cannot store stats for field ID: 21 (expected 100)"); + } + + @Test + public void testFromFieldMetricsString() { + FieldStatsStruct stats = new FieldStatsStruct<>(STRING_STATS); + + stats.fromFieldMetrics(new FieldMetrics<>(100, 28, 2, "a", "z")); + + assertThat(stats.lowerBound()).isEqualTo("a"); + assertThat(stats.upperBound()).isEqualTo("z"); + assertThat(stats.tightBounds()).isFalse(); + assertThat(stats.valueCount()).isEqualTo(28L); + assertThat(stats.nullValueCount()).isEqualTo(2L); + assertThat(stats.avgValueSizeInBytes()).isNull(); // unknown + } + + @Test + public void testFromFieldMetricsDouble() { + FieldStatsStruct stats = new FieldStatsStruct<>(DOUBLE_STATS); + + stats.fromFieldMetrics(new FieldMetrics<>(100, 34, 2, 6, 0.0d, 25.0d)); + + assertThat(stats.lowerBound()).isEqualTo(0.0d); + assertThat(stats.upperBound()).isEqualTo(25.0d); + assertThat(stats.tightBounds()).isFalse(); + assertThat(stats.valueCount()).isEqualTo(34L); + assertThat(stats.nullValueCount()).isEqualTo(2L); + assertThat(stats.nanValueCount()).isEqualTo(6L); + assertThat(stats.avgValueSizeInBytes()).isNull(); // unknown + } + + @Test + public void testCopy() { + FieldStatsStruct stats = + new FieldStatsStruct<>(STRING_STATS, "a", "z", true, 28, 2, 0, 1); + + FieldStatsStruct copy = stats.copy(); + + assertThat(copy).isNotSameAs(stats); + assertThat(copy.fieldId()).isEqualTo(stats.fieldId()); + assertThat(copy.type()).isEqualTo(stats.type()); + assertThat(copy.lowerBound()).isEqualTo("a"); + assertThat(copy.upperBound()).isEqualTo("z"); + assertThat(copy.tightBounds()).isTrue(); + assertThat(copy.valueCount()).isEqualTo(28L); + assertThat(copy.nullValueCount()).isEqualTo(2L); + assertThat(copy.nanValueCount()).isEqualTo(0L); + assertThat(copy.avgValueSizeInBytes()).isEqualTo(1); + } + + @Test + public void testFieldStatsProjection() { + Types.StructType projection = + Types.StructType.of( + optional(BASE_ID + StatsUtil.VALUE_COUNT_OFFSET, "count", Types.StringType.get()), + optional(BASE_ID + StatsUtil.UPPER_BOUND_OFFSET, "upper", Types.StringType.get())); + + FieldStatsStruct stats = new FieldStatsStruct<>(projection); + + stats.set(0, 34L); + stats.set(1, "z"); + assertThatThrownBy(() -> stats.set(stats.size(), null)) + .hasMessage("Index 2 out of bounds for length 2") + .isInstanceOf(IndexOutOfBoundsException.class); + + assertThat(stats.upperBound()).isEqualTo("z"); + assertThat(stats.valueCount()).isEqualTo(34L); + + assertThat(stats.get(0, Long.class)).isEqualTo(34L); + assertThat(stats.get(1, String.class)).isEqualTo("z"); + assertThatThrownBy(() -> stats.get(stats.size(), Object.class)) + .hasMessage("Index 2 out of bounds for length 2") + .isInstanceOf(IndexOutOfBoundsException.class); + } + + private static final List TYPES_AND_BOUNDS = + List.of( + Arguments.of(Types.BooleanType.get(), false, true), + Arguments.of(Types.IntegerType.get(), -5, 100), + Arguments.of(Types.LongType.get(), 0L, 1_000L), + Arguments.of(Types.FloatType.get(), 1.5f, 9.5f), + Arguments.of(Types.DoubleType.get(), 0.0d, 25.0d), + Arguments.of(Types.DateType.get(), 100, 200), + Arguments.of(Types.TimeType.get(), 1_000L, 2_000L), + Arguments.of(Types.TimestampType.withZone(), 111L, 222L), + Arguments.of(Types.TimestampNanoType.withZone(), 111L, 222L), + Arguments.of(Types.StringType.get(), "a", "z"), + Arguments.of( + Types.UUIDType.get(), + UUID.fromString("07ceab48-62b2-4219-9172-856e687c92ad"), + UUID.fromString("a9a7c24d-2869-4c66-8803-b1f6a36256ed")), + Arguments.of( + Types.FixedType.ofLength(4), + ByteBuffer.wrap(new byte[] {0, 1, 2, 3}), + ByteBuffer.wrap(new byte[] {4, 5, 6, 7})), + Arguments.of( + Types.BinaryType.get(), + ByteBuffer.wrap(new byte[] {1, 2}), + ByteBuffer.wrap(new byte[] {3, 4, 5})), + Arguments.of(Types.DecimalType.of(9, 2), new BigDecimal("1.23"), new BigDecimal("9.99")), + Arguments.of(Types.UnknownType.get(), null, null)); + + private static final List>>> SERIALIZERS = + List.of( + Named.of("Java", TestHelpers::roundTripSerialize), + Named.of("Kryo", TestHelpers.KryoHelpers::roundTripSerialize), + Named.of("InternalData", TestFieldStatsStruct::roundTripInternalData)); + + private static Stream serializationCases() { + return TYPES_AND_BOUNDS.stream() + .flatMap( + typeCase -> { + Object[] typeAndBounds = typeCase.get(); + return SERIALIZERS.stream() + .map( + serializer -> + Arguments.of( + typeAndBounds[0], typeAndBounds[1], typeAndBounds[2], serializer)); + }); + } + + @ParameterizedTest + @MethodSource("serializationCases") + public void testSerialization( + Type type, + Object lowerBound, + Object upperBound, + RoundTripSerializer> serializer) + throws Exception { + Types.StructType statsStruct = + StatsUtil.fieldStatsStruct(true, type, BASE_ID, MetricsModes.Full.get()); + + boolean isFloatingPoint = + type.typeId() == Type.TypeID.FLOAT || type.typeId() == Type.TypeID.DOUBLE; + boolean isVariableLength = + type.typeId() == Type.TypeID.STRING || type.typeId() == Type.TypeID.BINARY; + + FieldStatsStruct stats = + new FieldStatsStruct<>( + statsStruct, + lowerBound, + upperBound, + true, + 28L, + 2L, + isFloatingPoint ? 6L : 0L, + isVariableLength ? 1 : null); + + Comparator comparator = Comparators.forType(statsStruct); + + FieldStatsStruct copy = serializer.apply(stats); + assertThat(copy.fieldId()).isEqualTo(stats.fieldId()); + assertThat(copy.type()).isEqualTo(stats.type()); + assertThat(comparator.compare(copy, stats)).isEqualTo(0); + } + + private static Stream geoCases() { + return ImmutableList.of(Types.GeometryType.crs84(), Types.GeographyType.crs84()).stream() + .flatMap(type -> SERIALIZERS.stream().map(serializer -> Arguments.of(type, serializer))); + } + + @ParameterizedTest + @MethodSource("geoCases") + public void testGeoSerialization( + Type geoType, RoundTripSerializer> serializer) throws Exception { + Types.StructType statsStruct = + StatsUtil.fieldStatsStruct(true, geoType, BASE_ID, MetricsModes.Full.get()); + + // geometry and geography use bounding-box structs (x, y, z, m) for their bounds + PartitionData lowerBound = + new PartitionData(statsStruct.field("lower_bound").type().asStructType()); + lowerBound.set(0, 1.0d); + lowerBound.set(1, 2.0d); + lowerBound.set(2, 3.0d); + lowerBound.set(3, 4.0d); + PartitionData upperBound = + new PartitionData(statsStruct.field("upper_bound").type().asStructType()); + upperBound.set(0, 5.0d); + upperBound.set(1, 6.0d); + upperBound.set(2, 7.0d); + upperBound.set(3, 8.0d); + + FieldStatsStruct stats = + new FieldStatsStruct<>(statsStruct, lowerBound, upperBound, false, 28L, 2L, 0L, null); + + Comparator comparator = Comparators.forType(statsStruct); + + FieldStatsStruct copy = serializer.apply(stats); + assertThat(copy.fieldId()).isEqualTo(stats.fieldId()); + assertThat(copy.type()).isEqualTo(stats.type()); + assertThat(comparator.compare(copy, stats)).isEqualTo(0); + } + + // Variant is not Serializable so this does not test Java serialization + private static final List>>> VARIANT_SERIALIZERS = + List.of( + Named.of("Kryo", TestHelpers.KryoHelpers::roundTripSerialize), + Named.of("InternalData", TestFieldStatsStruct::roundTripInternalData)); + + @ParameterizedTest + @FieldSource("VARIANT_SERIALIZERS") + public void testVariantSerialization(RoundTripSerializer> serializer) + throws Exception { + Types.StructType statsStruct = + StatsUtil.fieldStatsStruct(true, Types.VariantType.get(), BASE_ID, MetricsModes.Full.get()); + + // variant bounds are variants keyed by JSON path expression that hold the bounds of fields + // within the variant; here the field "$['x']" ranges from 1 to 10 + VariantMetadata metadata = Variants.metadata("$['x']"); + + ShreddedObject lowerObject = Variants.object(metadata); + lowerObject.put("$['x']", Variants.of(1)); + Variant lowerBound = Variant.of(metadata, lowerObject); + + ShreddedObject upperObject = Variants.object(metadata); + upperObject.put("$['x']", Variants.of(10)); + Variant upperBound = Variant.of(metadata, upperObject); + + int size = metadata.dictionarySize() + lowerObject.sizeInBytes(); + + FieldStatsStruct stats = + new FieldStatsStruct<>(statsStruct, lowerBound, upperBound, false, 28L, 2L, 0L, size); + + FieldStatsStruct copy = serializer.apply(stats); + assertThat(copy.fieldId()).isEqualTo(stats.fieldId()); + assertThat(copy.type()).isEqualTo(stats.type()); + assertThat(copy.valueCount()).isEqualTo(28L); + assertThat(copy.nullValueCount()).isEqualTo(2L); + assertThat(copy.avgValueSizeInBytes()).isEqualTo(size); + + Variant copyLower = (Variant) copy.lowerBound(); + VariantTestUtil.assertEqual(lowerBound.metadata(), copyLower.metadata()); + VariantTestUtil.assertEqual(lowerBound.value(), copyLower.value()); + Variant copyUpper = (Variant) copy.upperBound(); + VariantTestUtil.assertEqual(upperBound.metadata(), copyUpper.metadata()); + VariantTestUtil.assertEqual(upperBound.value(), copyUpper.value()); + } + + private static FieldStatsStruct roundTripInternalData(FieldStats stats) + throws IOException { + Schema schema = stats.type().asSchema(); + InMemoryOutputFile file = new InMemoryOutputFile("internal.avro"); + + try (FileAppender> writer = + InternalData.write(FileFormat.AVRO, file).schema(schema).build()) { + writer.add(stats); + } + + try (CloseableIterable> reader = + InternalData.read(FileFormat.AVRO, file.toInputFile()) + .project(schema) + .setRootType(FieldStatsStruct.class) + .build()) { + return Iterables.getOnlyElement(reader); + } + } + + private static int pos(Types.StructType type, String fieldName) { + List fields = type.fields(); + for (int pos = 0; pos < fields.size(); pos += 1) { + if (fields.get(pos).name().equals(fieldName)) { + return pos; + } + } + + throw new IllegalArgumentException("Missing field: " + fieldName); + } +} diff --git a/core/src/test/java/org/apache/iceberg/TestStatsUtil.java b/core/src/test/java/org/apache/iceberg/TestStatsUtil.java index 4970f94a02fb..dc6e7e481c25 100644 --- a/core/src/test/java/org/apache/iceberg/TestStatsUtil.java +++ b/core/src/test/java/org/apache/iceberg/TestStatsUtil.java @@ -18,511 +18,449 @@ */ package org.apache.iceberg; -import static org.apache.iceberg.FieldStatistic.AVG_VALUE_SIZE_IN_BYTES; -import static org.apache.iceberg.FieldStatistic.LOWER_BOUND; -import static org.apache.iceberg.FieldStatistic.NAN_VALUE_COUNT; -import static org.apache.iceberg.FieldStatistic.NULL_VALUE_COUNT; -import static org.apache.iceberg.FieldStatistic.TIGHT_BOUNDS; -import static org.apache.iceberg.FieldStatistic.UPPER_BOUND; -import static org.apache.iceberg.FieldStatistic.VALUE_COUNT; -import static org.apache.iceberg.types.Types.NestedField.optional; -import static org.apache.iceberg.types.Types.NestedField.required; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import java.util.List; -import java.util.concurrent.ThreadLocalRandom; -import java.util.stream.IntStream; +import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.FieldSource; public class TestStatsUtil { - // the reserved field IDs from the reserved field ID space as defined in - // https://iceberg.apache.org/spec/#reserved-field-ids - private static final int RESERVED_FIELD_IDS_START = Integer.MAX_VALUE - 200; + @Test + public void testToBaseIdWithData() { + assertThat(StatsUtil.toBaseId(0)).isEqualTo(10_000); + assertThat(StatsUtil.toBaseId(1)).isEqualTo(10_200); + assertThat(StatsUtil.toBaseId(100)).isEqualTo(30_000); + assertThat(StatsUtil.toBaseId(999_949)).isEqualTo(199_999_800); + } @Test - public void statsIdsForTableColumns() { - int offset = 0; - for (int id = 0; id < StatsUtil.MAX_DATA_FIELD_ID; id++) { - int statsFieldId = StatsUtil.statsFieldIdForField(id); - int expected = StatsUtil.STATS_SPACE_FIELD_ID_START_FOR_DATA_FIELDS + offset; - assertThat(statsFieldId).as("at pos %s", id).isEqualTo(expected); - offset += StatsUtil.NUM_SUPPORTED_STATS_PER_COLUMN; - assertThat(StatsUtil.fieldIdForStatsField(statsFieldId)).as("at pos %s", id).isEqualTo(id); - } + public void testToBaseIdWithMetadata() { + assertThat(StatsUtil.toBaseId(MetadataColumns.LAST_UPDATED_SEQUENCE_NUMBER.fieldId())) + .isEqualTo(9_000); + assertThat(StatsUtil.toBaseId(MetadataColumns.ROW_ID.fieldId())).isEqualTo(9_200); + } - // also verify hardcoded field IDs from docs - int fieldId = 0; - int statsFieldId = 10_000; - assertThat(StatsUtil.statsFieldIdForField(fieldId)).isEqualTo(statsFieldId); - assertThat(StatsUtil.fieldIdForStatsField(statsFieldId)).isEqualTo(fieldId); - - fieldId = 1; - statsFieldId = 10_200; - assertThat(StatsUtil.statsFieldIdForField(fieldId)).isEqualTo(statsFieldId); - assertThat(StatsUtil.fieldIdForStatsField(statsFieldId)).isEqualTo(fieldId); - - fieldId = 2; - statsFieldId = 10_400; - assertThat(StatsUtil.statsFieldIdForField(fieldId)).isEqualTo(statsFieldId); - assertThat(StatsUtil.fieldIdForStatsField(statsFieldId)).isEqualTo(fieldId); - - fieldId = 5; - statsFieldId = 11_000; - assertThat(StatsUtil.statsFieldIdForField(fieldId)).isEqualTo(statsFieldId); - assertThat(StatsUtil.fieldIdForStatsField(statsFieldId)).isEqualTo(fieldId); - - fieldId = 100; - statsFieldId = 30_000; - assertThat(StatsUtil.statsFieldIdForField(fieldId)).isEqualTo(statsFieldId); - assertThat(StatsUtil.fieldIdForStatsField(statsFieldId)).isEqualTo(fieldId); - - fieldId = StatsUtil.MAX_DATA_FIELD_ID; - statsFieldId = StatsUtil.MAX_DATA_STATS_FIELD_ID; - assertThat(StatsUtil.statsFieldIdForField(fieldId)).isEqualTo(statsFieldId); - assertThat(StatsUtil.fieldIdForStatsField(statsFieldId)).isEqualTo(fieldId); - - fieldId = -1; - statsFieldId = -1; - assertThat(StatsUtil.statsFieldIdForField(fieldId)).isEqualTo(statsFieldId); - assertThat(StatsUtil.fieldIdForStatsField(statsFieldId)).isEqualTo(fieldId); + @Test + public void testBaseIdUnassigned() { + assertThat(StatsUtil.toBaseId(999_950)).isEqualTo(-1); + assertThat(StatsUtil.toBaseId(-1)).isEqualTo(-1); + assertThat(StatsUtil.toBaseId(MetadataColumns.FILE_PATH.fieldId())).isEqualTo(-1); } @Test - public void statsIdsOverflowForTableColumns() { - // pick 100 random IDs that are > MAX_FIELD_ID and < RESERVED_FIELD_IDS_START as going over - // the entire ID range takes too long - int invalidFieldId = -1; - for (int i = 0; i < 100; i++) { - int id = - ThreadLocalRandom.current() - .nextInt(StatsUtil.MAX_DATA_FIELD_ID + 1, RESERVED_FIELD_IDS_START); - assertThat(StatsUtil.statsFieldIdForField(id)).as("at pos %s", id).isEqualTo(invalidFieldId); - } + public void testToFieldIdWithData() { + assertThat(StatsUtil.toFieldId(10_000)).isEqualTo(0); + assertThat(StatsUtil.toFieldId(10_200)).isEqualTo(1); + assertThat(StatsUtil.toFieldId(30_000)).isEqualTo(100); + assertThat(StatsUtil.toFieldId(199_999_800)).isEqualTo(999_949); + } - assertThat(StatsUtil.fieldIdForStatsField(-1)).isEqualTo(invalidFieldId); - assertThat(StatsUtil.fieldIdForStatsField(0)).isEqualTo(invalidFieldId); - assertThat(StatsUtil.fieldIdForStatsField(200)).isEqualTo(invalidFieldId); - assertThat(StatsUtil.fieldIdForStatsField(5_000)).isEqualTo(invalidFieldId); - assertThat(StatsUtil.fieldIdForStatsField(8_600)).isEqualTo(invalidFieldId); - assertThat(StatsUtil.fieldIdForStatsField(10_001)).isEqualTo(invalidFieldId); - assertThat(StatsUtil.fieldIdForStatsField(10_201)).isEqualTo(invalidFieldId); - assertThat(StatsUtil.fieldIdForStatsField(10_500)).isEqualTo(invalidFieldId); - assertThat(StatsUtil.fieldIdForStatsField(10_900)).isEqualTo(invalidFieldId); - - // stats field IDs at or above the exclusive upper bound are invalid - assertThat(StatsUtil.fieldIdForStatsField(StatsUtil.STATS_SPACE_FIELD_ID_END)) - .isEqualTo(invalidFieldId); - assertThat(StatsUtil.fieldIdForStatsField(StatsUtil.STATS_SPACE_FIELD_ID_END + 200)) - .isEqualTo(invalidFieldId); - assertThat(StatsUtil.fieldIdForStatsField(Integer.MAX_VALUE)).isEqualTo(invalidFieldId); - - // field ID just past MAX_DATA_FIELD_ID is invalid - assertThat(StatsUtil.statsFieldIdForField(StatsUtil.MAX_DATA_FIELD_ID + 1)) - .isEqualTo(invalidFieldId); + @Test + public void testToFieldIdWithMetadata() { + assertThat(StatsUtil.toFieldId(9_000)) + .isEqualTo(MetadataColumns.LAST_UPDATED_SEQUENCE_NUMBER.fieldId()); + assertThat(StatsUtil.toFieldId(9_200)).isEqualTo(MetadataColumns.ROW_ID.fieldId()); } @Test - public void statsIdsForMetadataColumns() { - int fieldId = MetadataColumns.LAST_UPDATED_SEQUENCE_NUMBER.fieldId(); - int statsFieldId = 9_000; - assertThat(StatsUtil.statsFieldIdForField(fieldId)).isEqualTo(statsFieldId); - assertThat(StatsUtil.fieldIdForStatsField(statsFieldId)).isEqualTo(fieldId); - - fieldId = MetadataColumns.ROW_ID.fieldId(); - statsFieldId = 9_200; - assertThat(StatsUtil.statsFieldIdForField(fieldId)).isEqualTo(statsFieldId); - assertThat(StatsUtil.fieldIdForStatsField(statsFieldId)).isEqualTo(fieldId); - - // reserved metadata fields with IDs below/above the reserved stats range have no stats - int invalidFieldId = -1; - assertThat(StatsUtil.fieldIdForStatsField(8_800)).isEqualTo(invalidFieldId); - assertThat(StatsUtil.fieldIdForStatsField(9_400)).isEqualTo(invalidFieldId); - assertThat(StatsUtil.fieldIdForStatsField(9_600)).isEqualTo(invalidFieldId); - assertThat(StatsUtil.fieldIdForStatsField(9_800)).isEqualTo(invalidFieldId); - assertThat(IntStream.range(RESERVED_FIELD_IDS_START, Integer.MAX_VALUE)) - .filteredOn(id -> !StatsUtil.SUPPORTED_METADATA_FIELD_IDS.contains(id)) - .allSatisfy( - id -> - assertThat(StatsUtil.statsFieldIdForField(id)) - .as("at pos %s", id) - .isEqualTo(invalidFieldId)); + public void testToFieldIdOutsideRange() { + assertThatThrownBy(() -> StatsUtil.toFieldId(200_000_000)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid stats field ID: 200000000"); + assertThatThrownBy(() -> StatsUtil.toFieldId(200_000_001)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid stats field ID: 200000001"); + assertThatThrownBy(() -> StatsUtil.toFieldId(8_800)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid stats field ID: 8800"); + assertThatThrownBy(() -> StatsUtil.toFieldId(8_801)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid stats field ID: 8801"); } @Test - public void contentStatsForSimpleSchema() { - Types.NestedField intField = required(0, "i", Types.IntegerType.get()); - Types.NestedField floatField = required(2, "f", Types.FloatType.get()); - Types.NestedField stringField = required(4, "s", Types.StringType.get()); - Types.NestedField booleanField = required(6, "b", Types.BooleanType.get()); - Types.NestedField uuidField = required(StatsUtil.MAX_DATA_FIELD_ID, "u", Types.UUIDType.get()); - Schema schema = new Schema(intField, floatField, stringField, booleanField, uuidField); - Schema expectedStatsSchema = - new Schema( - optional( - 146, - "content_stats", - Types.StructType.of( - optional(10000, "i", FieldStatistic.fieldStatsFor(intField, 10000)), - optional(10400, "f", FieldStatistic.fieldStatsFor(floatField, 10400)), - optional(10800, "s", FieldStatistic.fieldStatsFor(stringField, 10800)), - optional(11200, "b", FieldStatistic.fieldStatsFor(booleanField, 11200)), - optional( - StatsUtil.MAX_DATA_STATS_FIELD_ID, - "u", - FieldStatistic.fieldStatsFor( - uuidField, StatsUtil.MAX_DATA_STATS_FIELD_ID))))); - Schema statsSchema = new Schema(StatsUtil.contentStatsFor(schema)); - assertThat(statsSchema.asStruct()).isEqualTo(expectedStatsSchema.asStruct()); + public void testToFieldIdReservedMetadataRange() { + // 9,000 to 10,000 (exclusive) is reserved for metadata column stats + assertThatThrownBy(() -> StatsUtil.toFieldId(9_400)) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessage("Unsupported metadata stats field ID: 9400"); } @Test - public void contentStatsForComplexSchema() { - Types.NestedField listElement = optional(3, "element", Types.IntegerType.get()); - Types.NestedField structInt = optional(7, "int", Types.IntegerType.get()); - Types.NestedField structString = optional(8, "string", Types.StringType.get()); - Types.NestedField mapKey = required(22, "key", Types.IntegerType.get()); - Types.NestedField mapValue = optional(24, "value", Types.StringType.get()); - Types.NestedField variantField = required(30, "variant", Types.VariantType.get()); - Types.NestedField uuidField = required(StatsUtil.MAX_DATA_FIELD_ID, "u", Types.UUIDType.get()); - Schema schema = - new Schema( - required(0, "i", Types.IntegerType.get()), - required(2, "list", Types.ListType.ofOptional(3, Types.IntegerType.get())), - required( - 6, - "simple_struct", - Types.StructType.of( - optional(7, "int", Types.IntegerType.get()), - optional(8, "string", Types.StringType.get()))), - required( - 20, - "b", - Types.MapType.ofOptional(22, 24, Types.IntegerType.get(), Types.StringType.get())), - variantField, - uuidField); - Schema expectedStatsSchema = - new Schema( - optional( - 146, - "content_stats", - Types.StructType.of( - optional( - 10000, - "i", - FieldStatistic.fieldStatsFor( - required(0, "i", Types.IntegerType.get()), 10000)), - optional( - 10600, "list.element", FieldStatistic.fieldStatsFor(listElement, 10600)), - optional( - 11400, "simple_struct.int", FieldStatistic.fieldStatsFor(structInt, 11400)), - optional( - 11600, - "simple_struct.string", - FieldStatistic.fieldStatsFor(structString, 11600)), - optional(14400, "b.key", FieldStatistic.fieldStatsFor(mapKey, 14400)), - optional(14800, "b.value", FieldStatistic.fieldStatsFor(mapValue, 14800)), - optional(16000, "variant", FieldStatistic.fieldStatsFor(variantField, 16000)), - optional( - StatsUtil.MAX_DATA_STATS_FIELD_ID, - "u", - FieldStatistic.fieldStatsFor( - uuidField, StatsUtil.MAX_DATA_STATS_FIELD_ID))))); - Schema statsSchema = new Schema(StatsUtil.contentStatsFor(schema)); - assertThat(statsSchema.asStruct()).isEqualTo(expectedStatsSchema.asStruct()); + public void testStatOffset() { + assertThat(StatsUtil.statOffset(10_000)).isEqualTo(0); + assertThat(StatsUtil.statOffset(10_201)).isEqualTo(1); + assertThat(StatsUtil.statOffset(10_211)).isEqualTo(11); + assertThat(StatsUtil.statOffset(10_399)).isEqualTo(199); + assertThat(StatsUtil.statOffset(10_400)).isEqualTo(0); + assertThat(StatsUtil.statOffset(199_999_999)).isEqualTo(199); } @Test - public void contentStatsChildNamesAreUnique() { - Schema schema = - new Schema( - required(1, "a", Types.StructType.of(required(2, "x", Types.IntegerType.get()))), - required(3, "b", Types.StructType.of(required(4, "x", Types.IntegerType.get())))); + public void testStatOffsetOutsideRange() { + assertThatThrownBy(() -> StatsUtil.statOffset(200_000_000)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid stats field ID: 200000000"); + assertThatThrownBy(() -> StatsUtil.statOffset(200_000_001)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid stats field ID: 200000001"); + assertThatThrownBy(() -> StatsUtil.statOffset(8_800)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid stats field ID: 8800"); + assertThatThrownBy(() -> StatsUtil.statOffset(8_801)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid stats field ID: 8801"); + } - Types.StructType contentStats = StatsUtil.contentStatsFor(schema).type().asStructType(); + private static final List FIXED_WIDTH_TYPES = + List.of( + Types.BooleanType.get(), + Types.IntegerType.get(), + Types.LongType.get(), + Types.DateType.get(), + Types.TimeType.get(), + Types.TimestampType.withoutZone(), + Types.TimestampType.withZone(), + Types.TimestampNanoType.withoutZone(), + Types.TimestampNanoType.withZone(), + Types.DecimalType.of(9, 2), + Types.UUIDType.get(), + Types.FixedType.ofLength(16)); + + @ParameterizedTest + @FieldSource("FIXED_WIDTH_TYPES") + public void testFixedWidthPrimitiveStruct(Type type) { + // fixed-width, non-floating-point types track bounds (including tight bounds) but have no NaN + // count or average value size + Types.StructType expected = + Types.StructType.of( + Types.NestedField.optional(30_001, "lower_bound", type), + Types.NestedField.optional(30_002, "upper_bound", type), + Types.NestedField.optional(30_003, "tight_bounds", Types.BooleanType.get()), + Types.NestedField.optional(30_004, "value_count", Types.LongType.get()), + Types.NestedField.optional(30_005, "null_value_count", Types.LongType.get())); + + Types.StructType actual = + StatsUtil.fieldStatsStruct(true, type, 30_000, MetricsModes.Full.get()); + + assertSameStructure(expected, actual); + } - assertThat(contentStats.fields().stream().map(Types.NestedField::name).toList()) - .doesNotHaveDuplicates() - .containsExactly("a.x", "b.x"); + private static final List VARIABLE_WIDTH_TYPES = + List.of(Types.StringType.get(), Types.BinaryType.get()); + + @ParameterizedTest + @FieldSource("VARIABLE_WIDTH_TYPES") + public void testVariableWidthPrimitiveStruct(Type type) { + // variable-width types track bounds (including tight bounds) and an average value size, but no + // NaN count + Types.StructType expected = + Types.StructType.of( + Types.NestedField.optional(30_001, "lower_bound", type), + Types.NestedField.optional(30_002, "upper_bound", type), + Types.NestedField.optional(30_003, "tight_bounds", Types.BooleanType.get()), + Types.NestedField.optional(30_004, "value_count", Types.LongType.get()), + Types.NestedField.optional(30_005, "null_value_count", Types.LongType.get()), + Types.NestedField.optional(30_007, "avg_value_size_in_bytes", Types.IntegerType.get())); + + Types.StructType actual = + StatsUtil.fieldStatsStruct(true, type, 30_000, MetricsModes.Full.get()); + + assertSameStructure(expected, actual); } - @Test - public void contentStatsSkipsFieldsOutsideStatsRange() { - Types.NestedField validField = required(0, "i", Types.IntegerType.get()); - Types.NestedField outOfRangeField = - required(StatsUtil.MAX_DATA_FIELD_ID + 1, "out_of_range", Types.IntegerType.get()); - Schema schema = new Schema(validField, outOfRangeField); - Schema expectedStatsSchema = - new Schema( - optional( - 146, - "content_stats", - Types.StructType.of( - optional(10000, "i", FieldStatistic.fieldStatsFor(validField, 10000))))); - Schema statsSchema = new Schema(StatsUtil.contentStatsFor(schema)); - assertThat(statsSchema.asStruct()).isEqualTo(expectedStatsSchema.asStruct()); + private static final List GEO_TYPES = + List.of( + Types.GeometryType.crs84(), + Types.GeometryType.of("srid:3857"), + Types.GeographyType.crs84(), + Types.GeographyType.of("srid:4269")); + + @ParameterizedTest + @FieldSource("GEO_TYPES") + public void testGeoStruct(Type type) { + // geometry and geography use bounding-box structs for their bounds, do not track tight bounds, + // and record an average value size + Types.StructType lowerBound = + Types.StructType.of( + Types.NestedField.required(30_010, "x", Types.DoubleType.get()), + Types.NestedField.required(30_011, "y", Types.DoubleType.get()), + Types.NestedField.optional(30_012, "z", Types.DoubleType.get()), + Types.NestedField.optional(30_013, "m", Types.DoubleType.get())); + Types.StructType upperBound = + Types.StructType.of( + Types.NestedField.required(30_014, "x", Types.DoubleType.get()), + Types.NestedField.required(30_015, "y", Types.DoubleType.get()), + Types.NestedField.optional(30_016, "z", Types.DoubleType.get()), + Types.NestedField.optional(30_017, "m", Types.DoubleType.get())); + Types.StructType expected = + Types.StructType.of( + Types.NestedField.optional(30_001, "lower_bound", lowerBound), + Types.NestedField.optional(30_002, "upper_bound", upperBound), + Types.NestedField.optional(30_004, "value_count", Types.LongType.get()), + Types.NestedField.optional(30_005, "null_value_count", Types.LongType.get()), + Types.NestedField.optional(30_007, "avg_value_size_in_bytes", Types.IntegerType.get())); + + Types.StructType actual = + StatsUtil.fieldStatsStruct(true, type, 30_000, MetricsModes.Full.get()); + + assertSameStructure(expected, actual); } - @Test - public void conditionalFieldInclusionForInteger() { - assertThat( - fieldStatsNames( - FieldStatistic.fieldStatsFor(required(1, "x", Types.IntegerType.get()), 10000))) - .containsExactly( - LOWER_BOUND.fieldName(), - UPPER_BOUND.fieldName(), - TIGHT_BOUNDS.fieldName(), - VALUE_COUNT.fieldName()) - .doesNotContain( - NULL_VALUE_COUNT.fieldName(), - NAN_VALUE_COUNT.fieldName(), - AVG_VALUE_SIZE_IN_BYTES.fieldName()); + private static final List FLOATING_POINT_TYPES = + List.of(Types.FloatType.get(), Types.DoubleType.get()); + + @ParameterizedTest + @FieldSource("FLOATING_POINT_TYPES") + public void testFloatingPointStruct(Type type) { + // floating-point types track bounds (including tight bounds) and a NaN count, but have no + // average value size + Types.StructType expected = + Types.StructType.of( + Types.NestedField.optional(30_001, "lower_bound", type), + Types.NestedField.optional(30_002, "upper_bound", type), + Types.NestedField.optional(30_003, "tight_bounds", Types.BooleanType.get()), + Types.NestedField.optional(30_004, "value_count", Types.LongType.get()), + Types.NestedField.optional(30_005, "null_value_count", Types.LongType.get()), + Types.NestedField.optional(30_006, "nan_value_count", Types.LongType.get())); + + Types.StructType actual = + StatsUtil.fieldStatsStruct(true, type, 30_000, MetricsModes.Full.get()); + + assertSameStructure(expected, actual); + } - assertThat( - fieldStatsNames( - FieldStatistic.fieldStatsFor(optional(1, "x", Types.IntegerType.get()), 10000))) - .containsExactly( - LOWER_BOUND.fieldName(), - UPPER_BOUND.fieldName(), - TIGHT_BOUNDS.fieldName(), - VALUE_COUNT.fieldName(), - NULL_VALUE_COUNT.fieldName()) - .doesNotContain(NAN_VALUE_COUNT.fieldName(), AVG_VALUE_SIZE_IN_BYTES.fieldName()); + private static final Types.StructType POINT = + Types.StructType.of( + Types.NestedField.required(1, "x", Types.FloatType.get()), + Types.NestedField.required(2, "y", Types.FloatType.get())); + + private static final List NESTED_TYPES = + List.of( + Types.ListType.ofRequired(1, Types.IntegerType.get()), + Types.ListType.ofOptional(1, POINT), + Types.MapType.ofRequired(1, 2, Types.StringType.get(), Types.StringType.get()), + Types.MapType.ofOptional(1, 2, Types.StringType.get(), POINT)); + + @ParameterizedTest + @FieldSource("NESTED_TYPES") + public void testNestedTypesHaveNoStats(Type type) { + // list and map types are not tracked and produce no stats struct + assertThat(StatsUtil.fieldStatsStruct(true, type, 30_000, MetricsModes.Full.get())).isNull(); } @Test - public void conditionalFieldInclusionForFloatAndDouble() { - assertThat( - fieldStatsNames( - FieldStatistic.fieldStatsFor(required(1, "x", Types.FloatType.get()), 10000))) - .containsExactly( - LOWER_BOUND.fieldName(), - UPPER_BOUND.fieldName(), - TIGHT_BOUNDS.fieldName(), - VALUE_COUNT.fieldName(), - NAN_VALUE_COUNT.fieldName()) - .doesNotContain(NULL_VALUE_COUNT.fieldName(), AVG_VALUE_SIZE_IN_BYTES.fieldName()); - - assertThat( - fieldStatsNames( - FieldStatistic.fieldStatsFor(optional(1, "x", Types.DoubleType.get()), 10000))) - .containsExactly( - LOWER_BOUND.fieldName(), - UPPER_BOUND.fieldName(), - TIGHT_BOUNDS.fieldName(), - VALUE_COUNT.fieldName(), - NULL_VALUE_COUNT.fieldName(), - NAN_VALUE_COUNT.fieldName()); + public void testRequiredField() { + // a required column does not produce a null_value_count field + Type type = Types.IntegerType.get(); + Types.StructType expected = + Types.StructType.of( + Types.NestedField.optional(30_001, "lower_bound", type), + Types.NestedField.optional(30_002, "upper_bound", type), + Types.NestedField.optional(30_003, "tight_bounds", Types.BooleanType.get()), + Types.NestedField.optional(30_004, "value_count", Types.LongType.get())); + + Types.StructType actual = + StatsUtil.fieldStatsStruct(false, type, 30_000, MetricsModes.Full.get()); + + assertSameStructure(expected, actual); } @Test - public void conditionalFieldInclusionForString() { - assertThat( - fieldStatsNames( - FieldStatistic.fieldStatsFor(required(1, "x", Types.StringType.get()), 10000))) - .containsExactly( - LOWER_BOUND.fieldName(), - UPPER_BOUND.fieldName(), - TIGHT_BOUNDS.fieldName(), - VALUE_COUNT.fieldName(), - AVG_VALUE_SIZE_IN_BYTES.fieldName()) - .doesNotContain(NULL_VALUE_COUNT.fieldName(), NAN_VALUE_COUNT.fieldName()); - + public void testStringNoneMode() { + // none mode produces no stats struct assertThat( - fieldStatsNames( - FieldStatistic.fieldStatsFor(optional(1, "x", Types.StringType.get()), 10000))) - .containsExactly( - LOWER_BOUND.fieldName(), - UPPER_BOUND.fieldName(), - TIGHT_BOUNDS.fieldName(), - VALUE_COUNT.fieldName(), - NULL_VALUE_COUNT.fieldName(), - AVG_VALUE_SIZE_IN_BYTES.fieldName()); + StatsUtil.fieldStatsStruct( + true, Types.StringType.get(), 30_000, MetricsModes.None.get())) + .isNull(); } @Test - public void conditionalFieldInclusionForBinary() { - assertThat( - fieldStatsNames( - FieldStatistic.fieldStatsFor(optional(1, "x", Types.BinaryType.get()), 10000))) - .containsExactly( - LOWER_BOUND.fieldName(), - UPPER_BOUND.fieldName(), - TIGHT_BOUNDS.fieldName(), - VALUE_COUNT.fieldName(), - NULL_VALUE_COUNT.fieldName(), - AVG_VALUE_SIZE_IN_BYTES.fieldName()) - .doesNotContain(NAN_VALUE_COUNT.fieldName()); + public void testStringCountsMode() { + // counts mode drops the bounds (and tight_bounds) but keeps counts and the value size + Types.StructType expected = + Types.StructType.of( + Types.NestedField.optional(30_004, "value_count", Types.LongType.get()), + Types.NestedField.optional(30_005, "null_value_count", Types.LongType.get()), + Types.NestedField.optional(30_007, "avg_value_size_in_bytes", Types.IntegerType.get())); + + Types.StructType actual = + StatsUtil.fieldStatsStruct(true, Types.StringType.get(), 30_000, MetricsModes.Counts.get()); + + assertSameStructure(expected, actual); + } - assertThat( - fieldStatsNames( - FieldStatistic.fieldStatsFor(required(1, "x", Types.BinaryType.get()), 10000))) - .containsExactly( - LOWER_BOUND.fieldName(), - UPPER_BOUND.fieldName(), - TIGHT_BOUNDS.fieldName(), - VALUE_COUNT.fieldName(), - AVG_VALUE_SIZE_IN_BYTES.fieldName()) - .doesNotContain(NULL_VALUE_COUNT.fieldName(), NAN_VALUE_COUNT.fieldName()); + private static final List BOUNDS_MODES = + List.of(MetricsModes.Truncate.withLength(16), MetricsModes.Full.get()); + + @ParameterizedTest + @FieldSource("BOUNDS_MODES") + public void testStringBoundsModes(MetricsModes.MetricsMode mode) { + // truncate and full both retain the full bounds struct + Type string = Types.StringType.get(); + Types.StructType expected = + Types.StructType.of( + Types.NestedField.optional(30_001, "lower_bound", string), + Types.NestedField.optional(30_002, "upper_bound", string), + Types.NestedField.optional(30_003, "tight_bounds", Types.BooleanType.get()), + Types.NestedField.optional(30_004, "value_count", Types.LongType.get()), + Types.NestedField.optional(30_005, "null_value_count", Types.LongType.get()), + Types.NestedField.optional(30_007, "avg_value_size_in_bytes", Types.IntegerType.get())); + + Types.StructType actual = StatsUtil.fieldStatsStruct(true, string, 30_000, mode); + + assertSameStructure(expected, actual); } @Test - public void conditionalFieldInclusionForGeometry() { - Types.StructType requiredStats = - FieldStatistic.fieldStatsFor(required(1, "x", Types.GeometryType.crs84()), 10000); - assertThat(fieldStatsNames(requiredStats)) - .containsExactly(LOWER_BOUND.fieldName(), UPPER_BOUND.fieldName(), VALUE_COUNT.fieldName()) - .doesNotContain( - TIGHT_BOUNDS.fieldName(), - NULL_VALUE_COUNT.fieldName(), - NAN_VALUE_COUNT.fieldName(), - AVG_VALUE_SIZE_IN_BYTES.fieldName()); - assertGeoBoundStructs(requiredStats, 10000); - - Types.StructType optionalStats = - FieldStatistic.fieldStatsFor(optional(1, "x", Types.GeometryType.crs84()), 10000); - assertThat(fieldStatsNames(optionalStats)) - .containsExactly( - LOWER_BOUND.fieldName(), - UPPER_BOUND.fieldName(), - VALUE_COUNT.fieldName(), - NULL_VALUE_COUNT.fieldName()) - .doesNotContain( - TIGHT_BOUNDS.fieldName(), - NAN_VALUE_COUNT.fieldName(), - AVG_VALUE_SIZE_IN_BYTES.fieldName()); - assertGeoBoundStructs(optionalStats, 10000); + public void testContentStatsReadSchema() { + Schema schema = + new Schema( + Types.NestedField.required(1, "id", Types.LongType.get()), + Types.NestedField.optional(2, "data", Types.StringType.get())); + + Types.StructType idStats = + Types.StructType.of( + Types.NestedField.optional(10_201, "lower_bound", Types.LongType.get()), + Types.NestedField.optional(10_202, "upper_bound", Types.LongType.get()), + Types.NestedField.optional(10_203, "tight_bounds", Types.BooleanType.get()), + Types.NestedField.optional(10_204, "value_count", Types.LongType.get())); + Types.StructType dataStats = + Types.StructType.of( + Types.NestedField.optional(10_401, "lower_bound", Types.StringType.get()), + Types.NestedField.optional(10_402, "upper_bound", Types.StringType.get()), + Types.NestedField.optional(10_403, "tight_bounds", Types.BooleanType.get()), + Types.NestedField.optional(10_404, "value_count", Types.LongType.get()), + Types.NestedField.optional(10_405, "null_value_count", Types.LongType.get()), + Types.NestedField.optional(10_407, "avg_value_size_in_bytes", Types.IntegerType.get())); + Types.StructType expected = + Types.StructType.of( + Types.NestedField.optional(10_200, "id", idStats), + Types.NestedField.optional(10_400, "data", dataStats)); + + Types.StructType actual = StatsUtil.statsReadSchema(schema, List.of(1, 2)); + + assertSameStructure(expected, actual); } @Test - public void conditionalFieldInclusionForGeography() { - Types.StructType requiredStats = - FieldStatistic.fieldStatsFor(required(1, "x", Types.GeographyType.crs84()), 10000); - assertThat(fieldStatsNames(requiredStats)) - .containsExactly(LOWER_BOUND.fieldName(), UPPER_BOUND.fieldName(), VALUE_COUNT.fieldName()) - .doesNotContain( - TIGHT_BOUNDS.fieldName(), - NULL_VALUE_COUNT.fieldName(), - NAN_VALUE_COUNT.fieldName(), - AVG_VALUE_SIZE_IN_BYTES.fieldName()); - assertGeoBoundStructs(requiredStats, 10000); - - Types.StructType optionalStats = - FieldStatistic.fieldStatsFor(optional(1, "x", Types.GeographyType.crs84()), 10000); - assertThat(fieldStatsNames(optionalStats)) - .containsExactly( - LOWER_BOUND.fieldName(), - UPPER_BOUND.fieldName(), - VALUE_COUNT.fieldName(), - NULL_VALUE_COUNT.fieldName()) - .doesNotContain( - TIGHT_BOUNDS.fieldName(), - NAN_VALUE_COUNT.fieldName(), - AVG_VALUE_SIZE_IN_BYTES.fieldName()); - assertGeoBoundStructs(optionalStats, 10000); + public void testContentStatsReadSchemaSubset() { + // a filter that only reads id and category produces a read schema for just those columns + Schema schema = + new Schema( + Types.NestedField.required(1, "id", Types.LongType.get()), + Types.NestedField.optional(2, "data", Types.StringType.get()), + Types.NestedField.optional(3, "category", Types.StringType.get())); + + Types.StructType idStats = + Types.StructType.of( + Types.NestedField.optional(10_201, "lower_bound", Types.LongType.get()), + Types.NestedField.optional(10_202, "upper_bound", Types.LongType.get()), + Types.NestedField.optional(10_203, "tight_bounds", Types.BooleanType.get()), + Types.NestedField.optional(10_204, "value_count", Types.LongType.get())); + Types.StructType categoryStats = + Types.StructType.of( + Types.NestedField.optional(10_601, "lower_bound", Types.StringType.get()), + Types.NestedField.optional(10_602, "upper_bound", Types.StringType.get()), + Types.NestedField.optional(10_603, "tight_bounds", Types.BooleanType.get()), + Types.NestedField.optional(10_604, "value_count", Types.LongType.get()), + Types.NestedField.optional(10_605, "null_value_count", Types.LongType.get()), + Types.NestedField.optional(10_607, "avg_value_size_in_bytes", Types.IntegerType.get())); + Types.StructType expected = + Types.StructType.of( + Types.NestedField.optional(10_200, "id", idStats), + Types.NestedField.optional(10_600, "category", categoryStats)); + + Types.StructType actual = StatsUtil.statsReadSchema(schema, List.of(1, 3)); + + assertSameStructure(expected, actual); } @Test - public void conditionalFieldInclusionForVariant() { - Types.StructType requiredStats = - FieldStatistic.fieldStatsFor(required(1, "x", Types.VariantType.get()), 10000); - assertThat(fieldStatsNames(requiredStats)) - .containsExactly( - LOWER_BOUND.fieldName(), - UPPER_BOUND.fieldName(), - VALUE_COUNT.fieldName(), - AVG_VALUE_SIZE_IN_BYTES.fieldName()) - .doesNotContain( - TIGHT_BOUNDS.fieldName(), NULL_VALUE_COUNT.fieldName(), NAN_VALUE_COUNT.fieldName()); - assertVariantBoundTypes(requiredStats); - - Types.StructType optionalStats = - FieldStatistic.fieldStatsFor(optional(1, "x", Types.VariantType.get()), 10000); - assertThat(fieldStatsNames(optionalStats)) - .containsExactly( - LOWER_BOUND.fieldName(), - UPPER_BOUND.fieldName(), - VALUE_COUNT.fieldName(), - NULL_VALUE_COUNT.fieldName(), - AVG_VALUE_SIZE_IN_BYTES.fieldName()) - .doesNotContain(TIGHT_BOUNDS.fieldName(), NAN_VALUE_COUNT.fieldName()); - assertVariantBoundTypes(optionalStats); + public void testContentStatsReadSchemaNestedStruct() { + // fields nested in a struct are represented by their flattened name and nested base ID + Schema schema = + new Schema( + Types.NestedField.required(1, "id", Types.LongType.get()), + Types.NestedField.optional( + 2, + "location", + Types.StructType.of( + Types.NestedField.required(3, "lat", Types.DoubleType.get()), + Types.NestedField.optional(4, "lon", Types.DoubleType.get())))); + + Types.StructType latStats = + Types.StructType.of( + Types.NestedField.optional(10_601, "lower_bound", Types.DoubleType.get()), + Types.NestedField.optional(10_602, "upper_bound", Types.DoubleType.get()), + Types.NestedField.optional(10_603, "tight_bounds", Types.BooleanType.get()), + Types.NestedField.optional(10_604, "value_count", Types.LongType.get()), + Types.NestedField.optional(10_606, "nan_value_count", Types.LongType.get())); + Types.StructType lonStats = + Types.StructType.of( + Types.NestedField.optional(10_801, "lower_bound", Types.DoubleType.get()), + Types.NestedField.optional(10_802, "upper_bound", Types.DoubleType.get()), + Types.NestedField.optional(10_803, "tight_bounds", Types.BooleanType.get()), + Types.NestedField.optional(10_804, "value_count", Types.LongType.get()), + Types.NestedField.optional(10_805, "null_value_count", Types.LongType.get()), + Types.NestedField.optional(10_806, "nan_value_count", Types.LongType.get())); + Types.StructType expected = + Types.StructType.of( + Types.NestedField.optional(10_600, "location_lat", latStats), + Types.NestedField.optional(10_800, "location_lon", lonStats)); + + Types.StructType actual = StatsUtil.statsReadSchema(schema, List.of(3, 4)); + + assertSameStructure(expected, actual); } - private void assertGeoBoundStructs(Types.StructType stats, int baseFieldId) { - Types.NestedField lowerBound = stats.field(LOWER_BOUND.fieldName()); - assertThat(lowerBound.type().isStructType()).isTrue(); - Types.StructType geoLower = lowerBound.type().asStructType(); - assertThat(geoLower.fields()).hasSize(4); - assertThat(geoLower.field("x")) - .satisfies( - f -> { - assertThat(f.fieldId()).isEqualTo(baseFieldId + 10); - assertThat(f.type()).isEqualTo(Types.DoubleType.get()); - assertThat(f.isRequired()).isTrue(); - }); - assertThat(geoLower.field("y")) - .satisfies( - f -> { - assertThat(f.fieldId()).isEqualTo(baseFieldId + 11); - assertThat(f.type()).isEqualTo(Types.DoubleType.get()); - assertThat(f.isRequired()).isTrue(); - }); - assertThat(geoLower.field("z")) - .satisfies( - f -> { - assertThat(f.fieldId()).isEqualTo(baseFieldId + 12); - assertThat(f.type()).isEqualTo(Types.DoubleType.get()); - assertThat(f.isOptional()).isTrue(); - }); - assertThat(geoLower.field("m")) - .satisfies( - f -> { - assertThat(f.fieldId()).isEqualTo(baseFieldId + 13); - assertThat(f.type()).isEqualTo(Types.DoubleType.get()); - assertThat(f.isOptional()).isTrue(); - }); - - Types.NestedField upperBound = stats.field(UPPER_BOUND.fieldName()); - assertThat(upperBound.type().isStructType()).isTrue(); - Types.StructType geoUpper = upperBound.type().asStructType(); - assertThat(geoUpper.fields()).hasSize(4); - assertThat(geoUpper.field("x")) - .satisfies( - f -> { - assertThat(f.fieldId()).isEqualTo(baseFieldId + 14); - assertThat(f.type()).isEqualTo(Types.DoubleType.get()); - assertThat(f.isRequired()).isTrue(); - }); - assertThat(geoUpper.field("y")) - .satisfies( - f -> { - assertThat(f.fieldId()).isEqualTo(baseFieldId + 15); - assertThat(f.type()).isEqualTo(Types.DoubleType.get()); - assertThat(f.isRequired()).isTrue(); - }); - assertThat(geoUpper.field("z")) - .satisfies( - f -> { - assertThat(f.fieldId()).isEqualTo(baseFieldId + 16); - assertThat(f.type()).isEqualTo(Types.DoubleType.get()); - assertThat(f.isOptional()).isTrue(); - }); - assertThat(geoUpper.field("m")) - .satisfies( - f -> { - assertThat(f.fieldId()).isEqualTo(baseFieldId + 17); - assertThat(f.type()).isEqualTo(Types.DoubleType.get()); - assertThat(f.isOptional()).isTrue(); - }); - } + @Test + public void testContentStatsReadSchemaOmitsListAndMap() { + // list and map columns do not track stats, so requesting them yields an empty read schema + Schema schema = + new Schema( + Types.NestedField.optional( + 1, "tags", Types.ListType.ofRequired(2, Types.StringType.get())), + Types.NestedField.optional( + 3, + "props", + Types.MapType.ofOptional(4, 5, Types.StringType.get(), Types.StringType.get()))); - private void assertVariantBoundTypes(Types.StructType stats) { - assertThat(stats.field(LOWER_BOUND.fieldName()).type()).isEqualTo(Types.VariantType.get()); - assertThat(stats.field(UPPER_BOUND.fieldName()).type()).isEqualTo(Types.VariantType.get()); + Types.StructType actual = StatsUtil.statsReadSchema(schema, List.of(1, 3)); + + assertThat(actual.fields()).isEmpty(); } - private List fieldStatsNames(Types.StructType structType) { - return structType.fields().stream().map(Types.NestedField::name).toList(); + /** + * Assert that two struct types match in field IDs, names, optionality, and types, ignoring docs. + */ + private static void assertSameStructure(Types.StructType expected, Types.StructType actual) { + assertThat(actual.fields()).as("Number of fields").hasSameSizeAs(expected.fields()); + + for (int i = 0; i < expected.fields().size(); i += 1) { + Types.NestedField expectedField = expected.fields().get(i); + Types.NestedField actualField = actual.fields().get(i); + + assertThat(actualField.fieldId()).as("Field ID").isEqualTo(expectedField.fieldId()); + assertThat(actualField.name()).as("Field name").isEqualTo(expectedField.name()); + assertThat(actualField.isOptional()) + .as("Field optionality for %s", expectedField.name()) + .isEqualTo(expectedField.isOptional()); + + if (expectedField.type().isStructType()) { + assertThat(actualField.type().isStructType()) + .as("Field %s should be a struct", expectedField.name()) + .isTrue(); + assertSameStructure(expectedField.type().asStructType(), actualField.type().asStructType()); + } else { + assertThat(actualField.type()) + .as("Field type for %s", expectedField.name()) + .isEqualTo(expectedField.type()); + } + } } } diff --git a/core/src/test/java/org/apache/iceberg/TestTrackedFile.java b/core/src/test/java/org/apache/iceberg/TestTrackedFile.java index e5c3ba8a247c..5840a1cd5f3d 100644 --- a/core/src/test/java/org/apache/iceberg/TestTrackedFile.java +++ b/core/src/test/java/org/apache/iceberg/TestTrackedFile.java @@ -22,6 +22,7 @@ import static org.assertj.core.api.Assertions.assertThat; import java.util.List; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.types.Types; import org.junit.jupiter.api.Test; @@ -32,7 +33,7 @@ public class TestTrackedFile { optional(1, "id", Types.IntegerType.get()), optional(2, "data", Types.StringType.get())); private static final Types.StructType CONTENT_STATS_TYPE = - StatsUtil.contentStatsFor(TABLE_SCHEMA).type().asStructType(); + StatsUtil.statsReadSchema(TABLE_SCHEMA, ImmutableList.of(1, 2)); private static final Types.StructType PARTITION_TYPE = PartitionSpec.builderFor(TABLE_SCHEMA).identity("id").build().partitionType(); @@ -93,8 +94,8 @@ public void schemaWithContentStatsReflectsInput() { optional(2, "data", Types.StringType.get()), optional(3, "ts", Types.TimestampType.withoutZone())); - Types.StructType smallStats = StatsUtil.contentStatsFor(smallSchema).type().asStructType(); - Types.StructType largeStats = StatsUtil.contentStatsFor(largeSchema).type().asStructType(); + Types.StructType smallStats = StatsUtil.statsReadSchema(smallSchema, ImmutableList.of(1)); + Types.StructType largeStats = StatsUtil.statsReadSchema(largeSchema, ImmutableList.of(1, 3)); Types.StructType smallType = TrackedFile.schemaWithContentStats(PARTITION_TYPE, smallStats); Types.StructType largeType = TrackedFile.schemaWithContentStats(PARTITION_TYPE, largeStats); @@ -105,7 +106,7 @@ public void schemaWithContentStatsReflectsInput() { largeType.field(TrackedFile.CONTENT_STATS_ID).type().asStructType(); assertThat(smallResult.fields()).hasSize(1); - assertThat(largeResult.fields()).hasSize(3); + assertThat(largeResult.fields()).hasSize(2); } @Test diff --git a/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java b/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java index f9b04f3de45b..540cd40050f4 100644 --- a/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java +++ b/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java @@ -18,6 +18,7 @@ */ package org.apache.iceberg; +import static org.apache.iceberg.types.Types.NestedField.optional; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -76,6 +77,32 @@ class TestTrackedFileAdapters { private static final int DELETION_VECTOR_ORDINAL = ordinalOf(TRACKED_FILE_SCHEMA, "deletion_vector"); + private static final Schema TABLE_SCHEMA = + new Schema( + optional(1, "id", Types.IntegerType.get()), optional(2, "score", Types.FloatType.get())); + private static final Types.StructType CONTENT_STATS_TYPE = + StatsUtil.statsReadSchema(TABLE_SCHEMA, ImmutableList.of(1, 2)); + private static final FieldStats ID_STATS = + new FieldStatsStruct<>( + CONTENT_STATS_TYPE.fieldType("id").asStructType(), 1, 1000, true, 100L, 5L, 0L, null); + private static final FieldStats SCORE_STATS = + new FieldStatsStruct<>( + CONTENT_STATS_TYPE.fieldType("score").asStructType(), + 1.0f, + 100.0f, + true, + 100L, + 10L, + 3L, + null); + private static final ContentStatsStruct CONTENT_STATS = + new ContentStatsStruct(CONTENT_STATS_TYPE); + + static { + CONTENT_STATS.setStats(1, ID_STATS); + CONTENT_STATS.setStats(2, SCORE_STATS); + } + @Test void testDataFileAdapterDelegation() { TrackedFile file = @@ -87,7 +114,7 @@ void testDataFileAdapterDelegation() { .recordCount(100L) .fileSizeInBytes(1024L) .specId(PARTITIONED_SPEC_ID) - .contentStats(createContentStats()) + .contentStats(CONTENT_STATS) .sortOrderId(3) .keyMetadata(ByteBuffer.wrap(new byte[] {1, 2, 3})) .splitOffsets(ImmutableList.of(50L, 100L)) @@ -113,7 +140,7 @@ void testDataFileAdapterDelegation() { assertThat(dataFile.manifestLocation()).isEqualTo(MANIFEST_LOCATION); assertThat(dataFile.equalityFieldIds()).isNull(); assertThat(dataFile.columnSizes()).isNull(); - assertThat(dataFile.valueCounts()).containsOnly(Map.entry(1, 100L), Map.entry(2, 200L)); + assertThat(dataFile.valueCounts()).containsOnly(Map.entry(1, 100L), Map.entry(2, 100L)); assertThat(dataFile.nullValueCounts()).containsOnly(Map.entry(1, 5L), Map.entry(2, 10L)); assertThat(dataFile.nanValueCounts()).containsOnly(Map.entry(2, 3L)); assertThat(dataFile.lowerBounds()) @@ -147,7 +174,7 @@ void testEqualityDeleteFileAdapterDelegation() { .recordCount(50L) .fileSizeInBytes(512L) .specId(PARTITIONED_SPEC_ID) - .contentStats(createContentStats()) + .contentStats(CONTENT_STATS) .sortOrderId(5) .keyMetadata(ByteBuffer.wrap(new byte[] {4, 5})) .splitOffsets(ImmutableList.of(200L)) @@ -175,7 +202,7 @@ void testEqualityDeleteFileAdapterDelegation() { assertThat(deleteFile.manifestLocation()).isEqualTo(MANIFEST_LOCATION); assertThat(deleteFile.equalityFieldIds()).containsExactly(1, 2, 3); assertThat(deleteFile.columnSizes()).isNull(); - assertThat(deleteFile.valueCounts()).containsOnly(Map.entry(1, 100L), Map.entry(2, 200L)); + assertThat(deleteFile.valueCounts()).containsOnly(Map.entry(1, 100L), Map.entry(2, 100L)); assertThat(deleteFile.nullValueCounts()).containsOnly(Map.entry(1, 5L), Map.entry(2, 10L)); assertThat(deleteFile.nanValueCounts()).containsOnly(Map.entry(2, 3L)); assertThat(deleteFile.lowerBounds()) @@ -410,54 +437,6 @@ private static DeletionVector deletionVector() { .build(); } - private static ContentStats createContentStats() { - Types.StructType statsStruct = - Types.StructType.of( - Types.NestedField.optional( - 10000, - "1", - Types.StructType.of( - Types.NestedField.optional(10001, "value_count", Types.LongType.get()), - Types.NestedField.optional(10002, "null_value_count", Types.LongType.get()), - Types.NestedField.optional(10003, "nan_value_count", Types.LongType.get()), - Types.NestedField.optional(10006, "lower_bound", Types.IntegerType.get()), - Types.NestedField.optional(10007, "upper_bound", Types.IntegerType.get()))), - Types.NestedField.optional( - 20000, - "2", - Types.StructType.of( - Types.NestedField.optional(20001, "value_count", Types.LongType.get()), - Types.NestedField.optional(20002, "null_value_count", Types.LongType.get()), - Types.NestedField.optional(20003, "nan_value_count", Types.LongType.get()), - Types.NestedField.optional(20006, "lower_bound", Types.FloatType.get()), - Types.NestedField.optional(20007, "upper_bound", Types.FloatType.get())))); - - List> fieldStatsList = - ImmutableList.of( - BaseFieldStats.builder() - .fieldId(1) - .type(Types.IntegerType.get()) - .valueCount(100L) - .nullValueCount(5L) - .lowerBound(1) - .upperBound(1000) - .build(), - BaseFieldStats.builder() - .fieldId(2) - .type(Types.FloatType.get()) - .valueCount(200L) - .nullValueCount(10L) - .nanValueCount(3L) - .lowerBound(1.0f) - .upperBound(100.0f) - .build()); - - return BaseContentStats.builder() - .withStatsStruct(statsStruct) - .withFieldStats(fieldStatsList) - .build(); - } - private static int ordinalOf(Types.StructType schema, String fieldName) { List fields = schema.fields(); for (int i = 0; i < fields.size(); i += 1) { diff --git a/core/src/test/java/org/apache/iceberg/TestTrackedFileBuilder.java b/core/src/test/java/org/apache/iceberg/TestTrackedFileBuilder.java index d6b0701fdfd7..3aa56660e615 100644 --- a/core/src/test/java/org/apache/iceberg/TestTrackedFileBuilder.java +++ b/core/src/test/java/org/apache/iceberg/TestTrackedFileBuilder.java @@ -30,6 +30,7 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; +import org.mockito.Mockito; public class TestTrackedFileBuilder { private static final int FORMAT_VERSION_V4 = 4; @@ -58,28 +59,7 @@ public class TestTrackedFileBuilder { .sizeInBytes(128L) .cardinality(10L) .build(); - private static final ContentStats CONTENT_STATS = - BaseContentStats.builder() - .withTableSchema(TABLE_SCHEMA) - .withFieldStats( - BaseFieldStats.builder() - .fieldId(1) - .type(Types.IntegerType.get()) - .valueCount(2000L) - .nullValueCount(0L) - .lowerBound(1) - .upperBound(1000) - .build()) - .withFieldStats( - BaseFieldStats.builder() - .fieldId(2) - .type(Types.StringType.get()) - .valueCount(2000L) - .nullValueCount(5L) - .lowerBound("a") - .upperBound("z") - .build()) - .build(); + private static final ContentStats CONTENT_STATS = Mockito.mock(ContentStats.class); private static final ByteBuffer KEY_METADATA = ByteBuffer.wrap(new byte[] {1, 2, 3}); private static final ImmutableList SPLIT_OFFSETS = ImmutableList.of(0L, 4096L, 8192L); private static final ByteBuffer DELETED_POSITIONS = ByteBuffer.wrap(new byte[] {10, 11, 12}); diff --git a/core/src/test/java/org/apache/iceberg/TestTrackedFileStruct.java b/core/src/test/java/org/apache/iceberg/TestTrackedFileStruct.java index 92453ff28f7c..9d5ae3419d9f 100644 --- a/core/src/test/java/org/apache/iceberg/TestTrackedFileStruct.java +++ b/core/src/test/java/org/apache/iceberg/TestTrackedFileStruct.java @@ -24,6 +24,7 @@ import java.util.List; import org.apache.iceberg.TestHelpers.RoundTripSerializer; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; import org.apache.iceberg.types.Types; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -60,6 +61,11 @@ class TestTrackedFileStruct { } private static final ContentStats CONTENT_STATS = Mockito.mock(ContentStats.class); + private static final ContentStats CONTENT_STATS_COPY = Mockito.mock(ContentStats.class); + + static { + Mockito.when(CONTENT_STATS.copy()).thenReturn(CONTENT_STATS_COPY); + } @Test void fieldAccess() { @@ -192,7 +198,7 @@ void copy() { 50L, 512L, 1, - null, + CONTENT_STATS, 5, DELETION_VECTOR, MANIFEST_INFO, @@ -211,6 +217,103 @@ void copy() { assertThat(copy.recordCount()).isEqualTo(50L); assertThat(copy.fileSizeInBytes()).isEqualTo(512L); assertThat(copy.specId()).isEqualTo(1); + assertThat(copy.contentStats()).isSameAs(CONTENT_STATS_COPY); + assertThat(copy.sortOrderId()).isEqualTo(5); + assertThat(copy.deletionVector()).isSameAs(DELETION_VECTOR_COPY); + assertThat(copy.manifestInfo()).isSameAs(MANIFEST_INFO_COPY); + assertThat(copy.keyMetadata()).isEqualTo(ByteBuffer.wrap(new byte[] {1, 2, 3})); + assertThat(copy.splitOffsets()).containsExactly(100L, 200L); + assertThat(copy.equalityIds()).containsExactly(1, 2, 3); + assertThat(copy.partition()).isSameAs(PARTITION_COPY); + + // mutable fields are deep-copied, not shared with the original + assertThat(copy.keyMetadata()).isNotSameAs(file.keyMetadata()); + } + + @Test + void copyWithStats() { + ContentStats stats = Mockito.mock(ContentStats.class); + ContentStats statsCopy = Mockito.mock(ContentStats.class); + Mockito.when(stats.copy(ImmutableSet.of(1))).thenReturn(statsCopy); + + TrackedFileStruct file = + new TrackedFileStruct( + TRACKING, + FileContent.DATA, + FORMAT_VERSION_V4, + "s3://bucket/data/00000-0-file.parquet", + FileFormat.PARQUET, + PARTITION, + 50L, + 512L, + 1, + stats, + 5, + DELETION_VECTOR, + MANIFEST_INFO, + ByteBuffer.wrap(new byte[] {1, 2, 3}), + ImmutableList.of(100L, 200L), + ImmutableList.of(1, 2, 3)); + + TrackedFile copy = file.copyWithStats(ImmutableSet.of(1)); + + assertThat(copy).isInstanceOf(TrackedFileStruct.class); + assertThat(copy.tracking()).isSameAs(TRACKING_COPY); + assertThat(copy.contentType()).isEqualTo(FileContent.DATA); + assertThat(copy.formatVersion()).isEqualTo(FORMAT_VERSION_V4); + assertThat(copy.location()).isEqualTo("s3://bucket/data/00000-0-file.parquet"); + assertThat(copy.fileFormat()).isEqualTo(FileFormat.PARQUET); + assertThat(copy.recordCount()).isEqualTo(50L); + assertThat(copy.fileSizeInBytes()).isEqualTo(512L); + assertThat(copy.specId()).isEqualTo(1); + assertThat(copy.contentStats()).isSameAs(statsCopy); + assertThat(copy.sortOrderId()).isEqualTo(5); + assertThat(copy.deletionVector()).isSameAs(DELETION_VECTOR_COPY); + assertThat(copy.manifestInfo()).isSameAs(MANIFEST_INFO_COPY); + assertThat(copy.keyMetadata()).isEqualTo(ByteBuffer.wrap(new byte[] {1, 2, 3})); + assertThat(copy.splitOffsets()).containsExactly(100L, 200L); + assertThat(copy.equalityIds()).containsExactly(1, 2, 3); + assertThat(copy.partition()).isSameAs(PARTITION_COPY); + + // mutable fields are deep-copied, not shared with the original + assertThat(copy.keyMetadata()).isNotSameAs(file.keyMetadata()); + } + + @Test + void copyWithoutStats() { + ContentStats stats = Mockito.mock(ContentStats.class); + + TrackedFileStruct file = + new TrackedFileStruct( + TRACKING, + FileContent.DATA, + FORMAT_VERSION_V4, + "s3://bucket/data/00000-0-file.parquet", + FileFormat.PARQUET, + PARTITION, + 50L, + 512L, + 1, + stats, + 5, + DELETION_VECTOR, + MANIFEST_INFO, + ByteBuffer.wrap(new byte[] {1, 2, 3}), + ImmutableList.of(100L, 200L), + ImmutableList.of(1, 2, 3)); + + TrackedFile copy = file.copyWithStats(ImmutableSet.of(1)); + + assertThat(copy).isInstanceOf(TrackedFileStruct.class); + assertThat(copy.tracking()).isSameAs(TRACKING_COPY); + assertThat(copy.contentType()).isEqualTo(FileContent.DATA); + assertThat(copy.formatVersion()).isEqualTo(FORMAT_VERSION_V4); + assertThat(copy.location()).isEqualTo("s3://bucket/data/00000-0-file.parquet"); + assertThat(copy.fileFormat()).isEqualTo(FileFormat.PARQUET); + assertThat(copy.recordCount()).isEqualTo(50L); + assertThat(copy.fileSizeInBytes()).isEqualTo(512L); + assertThat(copy.specId()).isEqualTo(1); + assertThat(copy.contentStats()).isNull(); assertThat(copy.sortOrderId()).isEqualTo(5); assertThat(copy.deletionVector()).isSameAs(DELETION_VECTOR_COPY); assertThat(copy.manifestInfo()).isSameAs(MANIFEST_INFO_COPY);