diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java index 41ac6334030b..e6939c5226a0 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBDeletionTableIT.java @@ -33,8 +33,11 @@ import org.apache.iotdb.rpc.IoTDBConnectionException; import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.RowRecord; import org.apache.tsfile.read.common.TimeRange; +import org.apache.tsfile.write.record.Tablet; import org.awaitility.Awaitility; import org.junit.After; import org.junit.AfterClass; @@ -60,6 +63,7 @@ import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Locale; import java.util.Random; @@ -74,6 +78,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import static org.apache.iotdb.relational.it.session.IoTDBSessionRelationalIT.genValue; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -2289,6 +2294,159 @@ public void testMultiDeviceCompletelyDeleteTable() throws SQLException { cleanData(testNum); } + @Test + public void testDeleteDataByTag() throws IoTDBConnectionException, StatementExecutionException { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnectionWithDB("test")) { + session.executeNonQueryStatement( + "CREATE TABLE IF NOT EXISTS delete_by_tag (deviceId STRING TAG, s1 INT32 FIELD)"); + + session.executeNonQueryStatement( + "insert into delete_by_tag (time, deviceId, s1) values (1, 'sensor', 1)"); + session.executeNonQueryStatement( + "insert into delete_by_tag (time, deviceId, s1) values (2, 'sensor', 2)"); + session.executeNonQueryStatement( + "insert into delete_by_tag (time, deviceId, s1) values (3, 'sensor', 3)"); + session.executeNonQueryStatement( + "insert into delete_by_tag (time, deviceId, s1) values (4, 'sensor', 4)"); + + session.executeNonQueryStatement("DELETE FROM delete_by_tag WHERE deviceId = 'sensor'"); + + SessionDataSet dataSet = + session.executeQueryStatement("select * from delete_by_tag order by time"); + assertFalse(dataSet.hasNext()); + + session.executeNonQueryStatement( + "insert into delete_by_tag (time, deviceId, s1) values (1, 'sensor', 1)"); + session.executeNonQueryStatement( + "insert into delete_by_tag (time, deviceId, s1) values (2, 'sensor', 2)"); + session.executeNonQueryStatement( + "insert into delete_by_tag (time, deviceId, s1) values (3, 'sensor', 3)"); + session.executeNonQueryStatement( + "insert into delete_by_tag (time, deviceId, s1) values (4, 'sensor', 4)"); + session.executeNonQueryStatement("FLUSH"); + + session.executeNonQueryStatement("DELETE FROM delete_by_tag WHERE deviceId = 'sensor'"); + + dataSet = session.executeQueryStatement("select * from delete_by_tag order by time"); + assertFalse(dataSet.hasNext()); + } finally { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnectionWithDB("test")) { + session.executeNonQueryStatement("DROP TABLE IF EXISTS delete_by_tag"); + } + } + } + + @Test + public void testDropAndAlter() throws IoTDBConnectionException, StatementExecutionException { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnectionWithDB("test")) { + session.executeNonQueryStatement("CREATE TABLE IF NOT EXISTS drop_and_alter (s1 int32)"); + + // time=1 and time=2 are INT32 and deleted by drop column + Tablet tablet = + new Tablet( + "drop_and_alter", + Collections.singletonList("s1"), + Collections.singletonList(TSDataType.INT32), + Collections.singletonList(ColumnCategory.FIELD)); + tablet.addTimestamp(0, 1); + tablet.addValue("s1", 0, genValue(TSDataType.INT32, 1)); + session.insert(tablet); + tablet.reset(); + + session.executeNonQueryStatement("FLUSH"); + + tablet = + new Tablet( + "drop_and_alter", + Collections.singletonList("s1"), + Collections.singletonList(TSDataType.INT32), + Collections.singletonList(ColumnCategory.FIELD)); + tablet.addTimestamp(0, 2); + tablet.addValue("s1", 0, genValue(TSDataType.INT32, 2)); + session.insert(tablet); + tablet.reset(); + + session.executeNonQueryStatement("ALTER TABLE drop_and_alter DROP COLUMN s1"); + + // time=3 and time=4 are STRING + tablet = + new Tablet( + "drop_and_alter", + Collections.singletonList("s1"), + Collections.singletonList(TSDataType.STRING), + Collections.singletonList(ColumnCategory.FIELD)); + tablet.addTimestamp(0, 3); + tablet.addValue("s1", 0, genValue(TSDataType.STRING, 3)); + session.insert(tablet); + tablet.reset(); + + session.executeNonQueryStatement("FLUSH"); + + tablet = + new Tablet( + "drop_and_alter", + Collections.singletonList("s1"), + Collections.singletonList(TSDataType.STRING), + Collections.singletonList(ColumnCategory.FIELD)); + tablet.addTimestamp(0, 4); + tablet.addValue("s1", 0, genValue(TSDataType.STRING, 4)); + session.insert(tablet); + tablet.reset(); + + session.executeNonQueryStatement("ALTER TABLE drop_and_alter DROP COLUMN s1"); + session.executeNonQueryStatement("ALTER TABLE drop_and_alter ADD COLUMN s1 TEXT"); + + // time=5 and time=6 are TEXT + tablet = + new Tablet( + "drop_and_alter", + Collections.singletonList("s1"), + Collections.singletonList(TSDataType.TEXT), + Collections.singletonList(ColumnCategory.FIELD)); + tablet.addTimestamp(0, 5); + tablet.addValue("s1", 0, genValue(TSDataType.STRING, 5)); + session.insert(tablet); + tablet.reset(); + + session.executeNonQueryStatement("FLUSH"); + + tablet = + new Tablet( + "drop_and_alter", + Collections.singletonList("s1"), + Collections.singletonList(TSDataType.TEXT), + Collections.singletonList(ColumnCategory.FIELD)); + tablet.addTimestamp(0, 6); + tablet.addValue("s1", 0, genValue(TSDataType.STRING, 6)); + session.insert(tablet); + tablet.reset(); + + SessionDataSet dataSet = + session.executeQueryStatement("select * from drop_and_alter order by time"); + // s1 is dropped but the time should remain + RowRecord rec; + int cnt = 0; + for (int i = 1; i < 7; i++) { + rec = dataSet.next(); + assertEquals(i, rec.getFields().get(0).getLongV()); + LOGGER.error( + "time is {}, value is {}, value type is {}", + rec.getFields().get(0).getLongV(), + rec.getFields().get(1), + rec.getFields().get(1).getDataType()); + // assertNull(rec.getFields().get(1).getDataType()); + // Assert.assertEquals(TSDataType.TEXT, rec.getFields().get(1).getDataType()); + cnt++; + } + Assert.assertEquals(6, cnt); + assertFalse(dataSet.hasNext()); + } finally { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnectionWithDB("test")) { + session.executeNonQueryStatement("DROP TABLE IF EXISTS drop_and_alter"); + } + } + } + private static void prepareDatabase() { try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT); Statement statement = connection.createStatement()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java index 188c4ec65298..ab6fbd4547a8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/ResourceByPathUtils.java @@ -280,7 +280,8 @@ public AbstractAlignedTimeSeriesMetadata generateTimeSeriesMetadata( if (!useFakeStatistics) { timeStatistics.mergeStatistics(alignedChunkMetadata.getTimeChunkMetadata().getStatistics()); for (int i = 0; i < valueTimeSeriesMetadataList.size(); i++) { - if (alignedChunkMetadata.getValueChunkMetadataList().get(i) != null) { + if (!alignedChunkMetadata.getValueChunkMetadataList().isEmpty() + && alignedChunkMetadata.getValueChunkMetadataList().get(i) != null) { exist[i] = true; valueTimeSeriesMetadataList .get(i) @@ -542,8 +543,18 @@ public ITimeSeriesMetadata generateTimeSeriesMetadata( boolean isModified = false; for (IChunkMetadata chunkMetadata : chunkMetadataList) { isModified = (isModified || chunkMetadata.isModified()); + TSDataType targetDataType = fullPath.getMeasurementSchema().getType(); + if (targetDataType.equals(TSDataType.STRING) + && (chunkMetadata.getDataType() != targetDataType)) { + // create new statistics object via new data type, and merge statistics information + SchemaUtils.rewriteNonAlignedChunkMetadataStatistics( + (ChunkMetadata) chunkMetadata, targetDataType); + chunkMetadata.setModified(true); + } if (!useFakeStatistics) { - seriesStatistics.mergeStatistics(chunkMetadata.getStatistics()); + if (targetDataType.isCompatible(chunkMetadata.getDataType())) { + seriesStatistics.mergeStatistics(chunkMetadata.getStatistics()); + } continue; } startTime = Math.min(startTime, chunkMetadata.getStartTime()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index e0b9ab91cd9b..746e042ab14b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -116,6 +116,7 @@ import org.apache.iotdb.db.storageengine.dataregion.memtable.IMemTable; import org.apache.iotdb.db.storageengine.dataregion.memtable.TsFileProcessor; import org.apache.iotdb.db.storageengine.dataregion.memtable.TsFileProcessorInfo; +import org.apache.iotdb.db.storageengine.dataregion.modification.IDPredicate; import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; import org.apache.iotdb.db.storageengine.dataregion.modification.TableDeletionEntry; @@ -3235,18 +3236,14 @@ private void deleteDataInSealedFiles(Collection sealedTsFiles, M List deletedByMods = new ArrayList<>(); List deletedByFiles = new ArrayList<>(); boolean isDropMeasurementExist = false; - boolean isDropTagExist = false; + IDPredicate.IDPredicateType idPredicateType = null; if (deletion instanceof TableDeletionEntry) { - TableDeletionEntry entry = (TableDeletionEntry) deletion; - isDropMeasurementExist = !entry.getPredicate().getMeasurementNames().isEmpty(); - } else { - TreeDeletionEntry entry = (TreeDeletionEntry) deletion; - if (entry.getPathPattern() instanceof MeasurementPath) { - Map tagMap = ((MeasurementPath) entry.getPathPattern()).getTagMap(); - isDropTagExist = (tagMap != null) && !tagMap.isEmpty(); - } + TableDeletionEntry tableDeletionEntry = (TableDeletionEntry) deletion; + isDropMeasurementExist = !tableDeletionEntry.getPredicate().getMeasurementNames().isEmpty(); + idPredicateType = tableDeletionEntry.getPredicate().getIdPredicateType(); } + for (TsFileResource sealedTsFile : sealedTsFiles) { if (canSkipDelete(sealedTsFile, deletion)) { continue; @@ -3310,7 +3307,9 @@ private void deleteDataInSealedFiles(Collection sealedTsFiles, M fileStartTime, fileEndTime); } - if (isFileFullyMatchedByTime(deletion, fileStartTime, fileEndTime)) { + if (isFileFullyMatchedByTime(deletion, fileStartTime, fileEndTime) + && idPredicateType.equals(IDPredicate.IDPredicateType.NOP) + && !isDropMeasurementExist) { ++matchSize; } else { deletedByMods.add(sealedTsFile); @@ -3343,7 +3342,7 @@ private void deleteDataInSealedFiles(Collection sealedTsFiles, M } // else do nothing } - if (!deletedByFiles.isEmpty() && !isDropMeasurementExist && !isDropTagExist) { + if (!deletedByFiles.isEmpty()) { deleteTsFileCompletely(deletedByFiles); if (logger.isDebugEnabled()) { logger.debug( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/DeletionPredicate.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/DeletionPredicate.java index 0a22da2a90a9..877c94f7081c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/DeletionPredicate.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/DeletionPredicate.java @@ -76,6 +76,10 @@ public IDPredicate getIdPredicate() { return idPredicate; } + public IDPredicate.IDPredicateType getIdPredicateType() { + return this.idPredicate.type; + } + public String getTableName() { return tableName; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java index c16f5339ee88..7345241bf784 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java @@ -55,6 +55,7 @@ public class SchemaUtils { private static final Map dataTypeColumnClassMap; public static final Logger logger = LoggerFactory.getLogger(SchemaUtils.class); + private static final Binary EMPTY_BINARY = new Binary("", StandardCharsets.UTF_8); static { dataTypeColumnClassMap = new HashMap<>(); @@ -403,123 +404,152 @@ public static AbstractAlignedChunkMetadata rewriteAlignedChunkMetadataStatistics AbstractAlignedChunkMetadata alignedChunkMetadata, TSDataType targetDataType) { List newValueChunkMetadataList = new ArrayList<>(); for (IChunkMetadata valueChunkMetadata : alignedChunkMetadata.getValueChunkMetadataList()) { - Statistics statistics = Statistics.getStatsByType(targetDataType); - switch (valueChunkMetadata.getDataType()) { - case INT32: - case DATE: - case INT64: - case TIMESTAMP: - case FLOAT: - case DOUBLE: - case BOOLEAN: - if (targetDataType == TSDataType.STRING) { - Binary[] binaryValues = new Binary[4]; - binaryValues[0] = - new Binary( - valueChunkMetadata.getStatistics().getFirstValue().toString(), - StandardCharsets.UTF_8); - binaryValues[1] = - new Binary( - valueChunkMetadata.getStatistics().getLastValue().toString(), - StandardCharsets.UTF_8); - if (valueChunkMetadata.getDataType() == TSDataType.BOOLEAN) { - binaryValues[2] = new Binary(Boolean.FALSE.toString(), StandardCharsets.UTF_8); - binaryValues[3] = new Binary(Boolean.TRUE.toString(), StandardCharsets.UTF_8); - } else { - binaryValues[2] = - new Binary( - valueChunkMetadata.getStatistics().getMinValue().toString(), - StandardCharsets.UTF_8); - binaryValues[3] = - new Binary( - valueChunkMetadata.getStatistics().getMaxValue().toString(), - StandardCharsets.UTF_8); - } - long[] longValues = new long[4]; - longValues[0] = valueChunkMetadata.getStatistics().getStartTime(); - longValues[1] = valueChunkMetadata.getStatistics().getEndTime(); - longValues[2] = longValues[1]; - longValues[3] = longValues[1]; - statistics.update(longValues, binaryValues, binaryValues.length); - } else if (targetDataType == TSDataType.TEXT) { - Binary[] binaryValues = new Binary[2]; - if (valueChunkMetadata.getDataType() == TSDataType.BOOLEAN) { - binaryValues[0] = new Binary(Boolean.FALSE.toString(), StandardCharsets.UTF_8); - binaryValues[1] = new Binary(Boolean.TRUE.toString(), StandardCharsets.UTF_8); - } else { - binaryValues[0] = - new Binary( - valueChunkMetadata.getStatistics().getMinValue().toString(), - StandardCharsets.UTF_8); - binaryValues[1] = - new Binary( - valueChunkMetadata.getStatistics().getMaxValue().toString(), - StandardCharsets.UTF_8); - } - long[] longValues = new long[2]; - longValues[0] = valueChunkMetadata.getStatistics().getStartTime(); - longValues[1] = valueChunkMetadata.getStatistics().getEndTime(); - statistics.update(longValues, binaryValues, binaryValues.length); - } else { - statistics = valueChunkMetadata.getStatistics(); - } - break; - case STRING: - if (targetDataType == TSDataType.TEXT) { - Binary[] binaryValues = new Binary[2]; - binaryValues[0] = - new Binary( - Arrays.asList(TSDataType.TEXT, TSDataType.BLOB) - .contains(valueChunkMetadata.getDataType()) - ? "" - : valueChunkMetadata.getStatistics().getMinValue().toString(), - StandardCharsets.UTF_8); - binaryValues[1] = - new Binary( - Arrays.asList(TSDataType.TEXT, TSDataType.BLOB) - .contains(valueChunkMetadata.getDataType()) - ? "" - : valueChunkMetadata.getStatistics().getMaxValue().toString(), - StandardCharsets.UTF_8); - long[] longValues = new long[2]; - longValues[0] = valueChunkMetadata.getStatistics().getStartTime(); - longValues[1] = valueChunkMetadata.getStatistics().getEndTime(); - statistics.update(longValues, binaryValues, binaryValues.length); - } else { - statistics = valueChunkMetadata.getStatistics(); - } - break; - case TEXT: - case BLOB: - if (targetDataType == TSDataType.STRING) { - Binary[] binaryValues = new Binary[2]; - binaryValues[0] = new Binary("", StandardCharsets.UTF_8); - binaryValues[1] = new Binary("", StandardCharsets.UTF_8); - long[] longValues = new long[2]; - longValues[0] = valueChunkMetadata.getStatistics().getStartTime(); - longValues[1] = valueChunkMetadata.getStatistics().getEndTime(); - statistics.update(longValues, binaryValues, binaryValues.length); - } else { - statistics = valueChunkMetadata.getStatistics(); - } - break; - default: - break; - } + if (targetDataType.isCompatible(valueChunkMetadata.getDataType())) { + Statistics statistics = Statistics.getStatsByType(targetDataType); + statistics = getNewStatistics(valueChunkMetadata, targetDataType, statistics); - ChunkMetadata newChunkMetadata = (ChunkMetadata) valueChunkMetadata; - newChunkMetadata.setTsDataType(targetDataType); - newChunkMetadata.setStatistics(statistics); - newValueChunkMetadataList.add(newChunkMetadata); + ChunkMetadata newChunkMetadata = (ChunkMetadata) valueChunkMetadata; + newChunkMetadata.setTsDataType(targetDataType); + newChunkMetadata.setStatistics(statistics); + newValueChunkMetadataList.add(newChunkMetadata); + } } return new AlignedChunkMetadata( alignedChunkMetadata.getTimeChunkMetadata(), newValueChunkMetadataList); } + public static void rewriteNonAlignedChunkMetadataStatistics( + ChunkMetadata chunkMetadata, TSDataType targetDataType) { + if (targetDataType.isCompatible(chunkMetadata.getDataType())) { + Statistics statistics = Statistics.getStatsByType(targetDataType); + statistics = getNewStatistics(chunkMetadata, targetDataType, statistics); + + chunkMetadata.setTsDataType(targetDataType); + chunkMetadata.setStatistics(statistics); + } + } + public static TSEncoding getDataTypeCompatibleEncoding(TSDataType dataType, TSEncoding encoding) { if (!encoding.isSupported(dataType)) { return EncodingInferenceUtils.getDefaultEncoding(dataType); } return encoding; } + + public static Statistics getNewStatistics( + IChunkMetadata chunkMetadata, TSDataType targetDataType, Statistics statistics) { + switch (chunkMetadata.getDataType()) { + case INT32: + case DATE: + case INT64: + case TIMESTAMP: + case FLOAT: + case DOUBLE: + case BOOLEAN: + if (targetDataType == TSDataType.STRING) { + Binary[] binaryValues = new Binary[4]; + binaryValues[0] = + new Binary( + chunkMetadata.getStatistics().getFirstValue().toString(), StandardCharsets.UTF_8); + binaryValues[1] = + new Binary( + chunkMetadata.getStatistics().getLastValue().toString(), StandardCharsets.UTF_8); + if (chunkMetadata.getDataType() == TSDataType.BOOLEAN) { + binaryValues[2] = new Binary(Boolean.FALSE.toString(), StandardCharsets.UTF_8); + binaryValues[3] = new Binary(Boolean.TRUE.toString(), StandardCharsets.UTF_8); + } else { + binaryValues[2] = + new Binary( + chunkMetadata.getStatistics().getMinValue().toString(), StandardCharsets.UTF_8); + binaryValues[3] = + new Binary( + chunkMetadata.getStatistics().getMaxValue().toString(), StandardCharsets.UTF_8); + } + long[] longValues = new long[4]; + longValues[0] = chunkMetadata.getStatistics().getStartTime(); + longValues[1] = chunkMetadata.getStatistics().getEndTime(); + longValues[2] = longValues[1]; + longValues[3] = longValues[1]; + statistics.update(longValues, binaryValues, binaryValues.length); + } else if (targetDataType == TSDataType.TEXT) { + Binary[] binaryValues = new Binary[2]; + if (chunkMetadata.getDataType() == TSDataType.BOOLEAN) { + binaryValues[0] = new Binary(Boolean.FALSE.toString(), StandardCharsets.UTF_8); + binaryValues[1] = new Binary(Boolean.TRUE.toString(), StandardCharsets.UTF_8); + } else { + binaryValues[0] = + new Binary( + chunkMetadata.getStatistics().getMinValue().toString(), StandardCharsets.UTF_8); + binaryValues[1] = + new Binary( + chunkMetadata.getStatistics().getMaxValue().toString(), StandardCharsets.UTF_8); + } + long[] longValues = new long[2]; + longValues[0] = chunkMetadata.getStatistics().getStartTime(); + longValues[1] = chunkMetadata.getStatistics().getEndTime(); + statistics.update(longValues, binaryValues, binaryValues.length); + } else { + statistics = chunkMetadata.getStatistics(); + } + break; + case STRING: + if (targetDataType == TSDataType.TEXT) { + Binary[] binaryValues = new Binary[2]; + binaryValues[0] = + new Binary( + chunkMetadata.getStatistics().getMinValue().toString(), StandardCharsets.UTF_8); + binaryValues[1] = + new Binary( + chunkMetadata.getStatistics().getMaxValue().toString(), StandardCharsets.UTF_8); + long[] longValues = new long[2]; + longValues[0] = chunkMetadata.getStatistics().getStartTime(); + longValues[1] = chunkMetadata.getStatistics().getEndTime(); + statistics.update(longValues, binaryValues, binaryValues.length); + } else if (targetDataType == TSDataType.BLOB) { + statistics.update( + chunkMetadata.getStatistics().getStartTime(), + new Binary( + chunkMetadata.getStatistics().getMinValue().toString(), StandardCharsets.UTF_8)); + statistics.update( + chunkMetadata.getStatistics().getEndTime(), + new Binary( + chunkMetadata.getStatistics().getMaxValue().toString(), StandardCharsets.UTF_8)); + } else { + statistics = chunkMetadata.getStatistics(); + } + break; + case TEXT: + if (targetDataType == TSDataType.STRING) { + Binary[] binaryValues = new Binary[2]; + binaryValues[0] = EMPTY_BINARY; + binaryValues[1] = EMPTY_BINARY; + long[] longValues = new long[2]; + longValues[0] = chunkMetadata.getStatistics().getStartTime(); + longValues[1] = chunkMetadata.getStatistics().getEndTime(); + statistics.update(longValues, binaryValues, binaryValues.length); + } else if (targetDataType == TSDataType.BLOB) { + statistics.update(chunkMetadata.getStatistics().getStartTime(), EMPTY_BINARY); + statistics.update(chunkMetadata.getStatistics().getEndTime(), EMPTY_BINARY); + } else { + statistics = chunkMetadata.getStatistics(); + } + break; + case BLOB: + if (targetDataType == TSDataType.STRING || targetDataType == TSDataType.TEXT) { + Binary[] binaryValues = new Binary[2]; + binaryValues[0] = EMPTY_BINARY; + binaryValues[1] = EMPTY_BINARY; + long[] longValues = new long[2]; + longValues[0] = chunkMetadata.getStatistics().getStartTime(); + longValues[1] = chunkMetadata.getStatistics().getEndTime(); + statistics.update(longValues, binaryValues, binaryValues.length); + } else { + statistics = chunkMetadata.getStatistics(); + } + break; + default: + break; + } + return statistics; + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/SchemaUtilsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/SchemaUtilsTest.java index 17dffd697850..0d9875a5b2db 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/SchemaUtilsTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/SchemaUtilsTest.java @@ -29,13 +29,19 @@ import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.utils.Binary; import org.junit.Assert; import org.junit.Test; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Set; + +import static org.apache.tsfile.file.metadata.statistics.Statistics.canMerge; public class SchemaUtilsTest { @Test @@ -103,13 +109,145 @@ public void rewriteAlignedChunkMetadataStatistics() { AbstractAlignedChunkMetadata abstractAlignedChunkMetadata = SchemaUtils.rewriteAlignedChunkMetadataStatistics( alignedChunkMetadata, targetDataType); - Assert.assertEquals( - targetDataType, - abstractAlignedChunkMetadata.getValueChunkMetadataList().get(0).getDataType()); + if (!abstractAlignedChunkMetadata.getValueChunkMetadataList().isEmpty()) { + Assert.assertEquals( + targetDataType, + abstractAlignedChunkMetadata.getValueChunkMetadataList().get(0).getDataType()); + } } catch (ClassCastException e) { Assert.fail(e.getMessage()); } } } } + + @Test + public void mergeMetadataStatistics() throws Exception { + Set unsupportTsDataType = new HashSet<>(); + unsupportTsDataType.add(TSDataType.UNKNOWN); + unsupportTsDataType.add(TSDataType.VECTOR); + for (TSDataType sourceDataType : Arrays.asList(TSDataType.DOUBLE)) { + for (TSDataType targetDataType : Arrays.asList(TSDataType.TEXT, TSDataType.BLOB)) { + + if (sourceDataType.equals(targetDataType)) { + continue; + } + if (unsupportTsDataType.contains(sourceDataType) + || unsupportTsDataType.contains(targetDataType)) { + continue; + } + + System.out.println("from " + sourceDataType + " to " + targetDataType); + + // Aligned series + Statistics s1 = Statistics.getStatsByType(sourceDataType); + s1.update(new long[] {1, 2}, new double[] {1.0, 2.0}, 2); + Statistics s2 = Statistics.getStatsByType(TSDataType.DOUBLE); + s2.update(new long[] {1, 2}, new double[] {1.0, 2.0}, 2); + List valueChunkMetadatas = + Arrays.asList( + new ChunkMetadata( + "s0", + sourceDataType, + SchemaUtils.getDataTypeCompatibleEncoding(sourceDataType, TSEncoding.RLE), + CompressionType.LZ4, + 0, + s1), + new ChunkMetadata( + "s1", + TSDataType.DOUBLE, + SchemaUtils.getDataTypeCompatibleEncoding(TSDataType.DOUBLE, TSEncoding.RLE), + CompressionType.LZ4, + 0, + s2)); + IChunkMetadata alignedChunkMetadata = + new AlignedChunkMetadata(new ChunkMetadata(), valueChunkMetadatas); + + Statistics s3 = Statistics.getStatsByType(targetDataType); + if (targetDataType == TSDataType.BLOB) { + s3.update(3, new Binary("3", StandardCharsets.UTF_8)); + s3.update(4, new Binary("4", StandardCharsets.UTF_8)); + } else { + s3.update( + new long[] {1, 2}, + new Binary[] { + new Binary("3", StandardCharsets.UTF_8), + new Binary("4", StandardCharsets.UTF_8), + new Binary("3", StandardCharsets.UTF_8), + new Binary("4", StandardCharsets.UTF_8) + }, + 2); + } + Statistics s4 = Statistics.getStatsByType(targetDataType); + if (targetDataType == TSDataType.BLOB) { + s3.update(4, new Binary("4", StandardCharsets.UTF_8)); + } else { + s4.update( + new long[] {1, 2}, + new Binary[] { + new Binary("5", StandardCharsets.UTF_8), + new Binary("6", StandardCharsets.UTF_8), + new Binary("5", StandardCharsets.UTF_8), + new Binary("6", StandardCharsets.UTF_8) + }, + 2); + } + List targetChunkMetadatas = + Arrays.asList( + new ChunkMetadata( + "s0", + targetDataType, + SchemaUtils.getDataTypeCompatibleEncoding(targetDataType, TSEncoding.RLE), + CompressionType.LZ4, + 0, + s3), + new ChunkMetadata( + "s1", + targetDataType, + SchemaUtils.getDataTypeCompatibleEncoding(targetDataType, TSEncoding.RLE), + CompressionType.LZ4, + 0, + s4)); + AbstractAlignedChunkMetadata abstractAlignedChunkMetadata = + (AbstractAlignedChunkMetadata) alignedChunkMetadata; + try { + abstractAlignedChunkMetadata = + SchemaUtils.rewriteAlignedChunkMetadataStatistics( + abstractAlignedChunkMetadata, targetDataType); + } catch (ClassCastException e) { + Assert.fail(e.getMessage()); + } + + for (int i = 0; i < targetChunkMetadatas.size(); i++) { + if (!abstractAlignedChunkMetadata.getValueChunkMetadataList().isEmpty() + && abstractAlignedChunkMetadata.getValueChunkMetadataList().get(i) != null) { + if (targetChunkMetadatas.get(i).getStatistics().getClass() + == abstractAlignedChunkMetadata + .getValueChunkMetadataList() + .get(i) + .getStatistics() + .getClass() + || canMerge( + abstractAlignedChunkMetadata + .getValueChunkMetadataList() + .get(i) + .getStatistics() + .getType(), + targetChunkMetadatas.get(i).getStatistics().getType())) { + targetChunkMetadatas + .get(i) + .getStatistics() + .mergeStatistics( + abstractAlignedChunkMetadata + .getValueChunkMetadataList() + .get(i) + .getStatistics()); + } else { + throw new Exception("unsupported"); + } + } + } + } + } + } }