From af79e1948e8b46c3c822b9a52ec8c893daa255b5 Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Thu, 29 May 2025 11:20:12 +0800 Subject: [PATCH 1/2] Pipe: Fixed the problem of null pointer and incorrect Tablet format being modified during TSFile construction --- .../util/builder/PipeTableModelTsFileBuilderV2.java | 11 ++++++++--- .../util/builder/PipeTreeModelTsFileBuilderV2.java | 11 +++++++---- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/builder/PipeTableModelTsFileBuilderV2.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/builder/PipeTableModelTsFileBuilderV2.java index a7400370a3876..9b349d39a459c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/builder/PipeTableModelTsFileBuilderV2.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/builder/PipeTableModelTsFileBuilderV2.java @@ -44,6 +44,7 @@ import java.io.IOException; import java.time.LocalDate; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -163,10 +164,12 @@ private void writeTabletsIntoOneFile( List aggregatedSchemas = tablets.stream() .flatMap(tablet -> tablet.getSchemas().stream()) + .filter(Objects::nonNull) .collect(Collectors.toList()); List aggregatedColumnCategories = tablets.stream() .flatMap(tablet -> tablet.getColumnTypes().stream()) + .filter(Objects::nonNull) .collect(Collectors.toList()); final Set seen = new HashSet<>(); @@ -193,10 +196,9 @@ private void writeTabletsIntoOneFile( for (int i = 0, size = tabletList.size(); i < size; ++i) { final Tablet tablet = tabletList.get(i); - // convert date value to int - // refer to + // convert date value to int refer to // org.apache.iotdb.db.storageengine.dataregion.memtable.WritableMemChunk.writeNonAlignedTablet - final Object[] values = tablet.getValues(); + final Object[] values = Arrays.copyOf(tablet.getValues(), tablet.getValues().length); for (int j = 0; j < tablet.getSchemas().size(); ++j) { final IMeasurementSchema schema = tablet.getSchemas().get(j); if (Objects.nonNull(schema) @@ -220,6 +222,7 @@ private void writeTabletsIntoOneFile( // the data of the table model is aligned true, tablet.getSchemas().stream() + .filter(Objects::nonNull) .map(IMeasurementSchema::getMeasurementName) .toArray(String[]::new), tablet.getSchemas().stream() @@ -227,6 +230,7 @@ private void writeTabletsIntoOneFile( .toArray(TSDataType[]::new), // TODO: cast tablet.getSchemas().stream() + .filter(Objects::nonNull) .map(schema -> (MeasurementSchema) schema) .toArray(MeasurementSchema[]::new), tablet.getTimestamps(), @@ -234,6 +238,7 @@ private void writeTabletsIntoOneFile( tablet.getValues(), tablet.getRowSize(), tablet.getColumnTypes().stream() + .filter(Objects::nonNull) .map(TsTableColumnCategory::fromTsFileColumnCategory) .toArray(TsTableColumnCategory[]::new)); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/builder/PipeTreeModelTsFileBuilderV2.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/builder/PipeTreeModelTsFileBuilderV2.java index 0347ec6706542..9e8855a53b834 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/builder/PipeTreeModelTsFileBuilderV2.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/builder/PipeTreeModelTsFileBuilderV2.java @@ -41,6 +41,7 @@ import java.io.IOException; import java.time.LocalDate; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Objects; import java.util.concurrent.atomic.AtomicLong; @@ -138,10 +139,9 @@ private void writeTabletsIntoOneFile( for (int i = 0, size = tabletList.size(); i < size; ++i) { final Tablet tablet = tabletList.get(i); - // convert date value to int - // refer to + // convert date value to int refer to // org.apache.iotdb.db.storageengine.dataregion.memtable.WritableMemChunk.writeNonAlignedTablet - final Object[] values = tablet.getValues(); + final Object[] values = Arrays.copyOf(tablet.getValues(), tablet.getValues().length); for (int j = 0; j < tablet.getSchemas().size(); ++j) { final IMeasurementSchema schema = tablet.getSchemas().get(j); if (Objects.nonNull(schema) @@ -162,18 +162,21 @@ private void writeTabletsIntoOneFile( new PartialPath(tablet.getDeviceId()), isTabletAlignedList.get(i), tablet.getSchemas().stream() + .filter(Objects::nonNull) .map(IMeasurementSchema::getMeasurementName) .toArray(String[]::new), tablet.getSchemas().stream() + .filter(Objects::nonNull) .map(IMeasurementSchema::getType) .toArray(TSDataType[]::new), // TODO: cast tablet.getSchemas().stream() + .filter(Objects::nonNull) .map(schema -> (MeasurementSchema) schema) .toArray(MeasurementSchema[]::new), tablet.getTimestamps(), tablet.getBitMaps(), - tablet.getValues(), + values, tablet.getRowSize()); final int start = 0; From 3dd00f44d7e22b3f840f021a2ecf82523a85ee61 Mon Sep 17 00:00:00 2001 From: luoluoyuyu Date: Thu, 29 May 2025 11:22:44 +0800 Subject: [PATCH 2/2] fix --- .../connector/util/builder/PipeTableModelTsFileBuilderV2.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/builder/PipeTableModelTsFileBuilderV2.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/builder/PipeTableModelTsFileBuilderV2.java index 9b349d39a459c..7e6886deebb37 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/builder/PipeTableModelTsFileBuilderV2.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/util/builder/PipeTableModelTsFileBuilderV2.java @@ -235,7 +235,7 @@ private void writeTabletsIntoOneFile( .toArray(MeasurementSchema[]::new), tablet.getTimestamps(), tablet.getBitMaps(), - tablet.getValues(), + values, tablet.getRowSize(), tablet.getColumnTypes().stream() .filter(Objects::nonNull)