diff --git a/extensions-contrib/druid-deltalake-extensions/src/main/java/org/apache/druid/delta/input/DeltaInputRow.java b/extensions-contrib/druid-deltalake-extensions/src/main/java/org/apache/druid/delta/input/DeltaInputRow.java index 442412dd154f..a8cd0d5020e2 100644 --- a/extensions-contrib/druid-deltalake-extensions/src/main/java/org/apache/druid/delta/input/DeltaInputRow.java +++ b/extensions-contrib/druid-deltalake-extensions/src/main/java/org/apache/druid/delta/input/DeltaInputRow.java @@ -201,7 +201,7 @@ private static Object getValue(DataType dataType, io.delta.kernel.data.Row dataR } return String.valueOf(charArray); } else if (dataType instanceof DecimalType) { - return dataRow.getDecimal(columnOrdinal).longValue(); + return dataRow.getDecimal(columnOrdinal); } else if (dataType instanceof StructType) { final io.delta.kernel.data.Row structRow = dataRow.getStruct(columnOrdinal); return RowSerde.convertRowToJsonObject(structRow); diff --git a/extensions-contrib/druid-deltalake-extensions/src/main/java/org/apache/druid/delta/input/RowSerde.java b/extensions-contrib/druid-deltalake-extensions/src/main/java/org/apache/druid/delta/input/RowSerde.java index bad6191496d7..d7c6fcccdba2 100644 --- a/extensions-contrib/druid-deltalake-extensions/src/main/java/org/apache/druid/delta/input/RowSerde.java +++ b/extensions-contrib/druid-deltalake-extensions/src/main/java/org/apache/druid/delta/input/RowSerde.java @@ -32,6 +32,7 @@ import io.delta.kernel.types.ByteType; import io.delta.kernel.types.DataType; import io.delta.kernel.types.DateType; +import io.delta.kernel.types.DecimalType; import io.delta.kernel.types.DoubleType; import io.delta.kernel.types.FloatType; import io.delta.kernel.types.IntegerType; @@ -126,6 +127,8 @@ public static Map convertRowToJsonObject(Row row) value = row.getFloat(fieldId); } else if (fieldType instanceof DoubleType) { value = row.getDouble(fieldId); + } else if (fieldType instanceof DecimalType) { + value = row.getDecimal(fieldId); } else if (fieldType instanceof DateType) { value = DeltaTimeUtils.getSecondsFromDate(row.getInt(fieldId)); } else if (fieldType instanceof TimestampType) { diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/java/org/apache/druid/delta/input/ComplexTypesDeltaTable.java b/extensions-contrib/druid-deltalake-extensions/src/test/java/org/apache/druid/delta/input/ComplexTypesDeltaTable.java index 7fdffc03041c..81edba927770 100644 --- a/extensions-contrib/druid-deltalake-extensions/src/test/java/org/apache/druid/delta/input/ComplexTypesDeltaTable.java +++ b/extensions-contrib/druid-deltalake-extensions/src/test/java/org/apache/druid/delta/input/ComplexTypesDeltaTable.java @@ -28,6 +28,7 @@ import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.segment.AutoTypeColumnSchema; +import java.math.BigDecimal; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -66,35 +67,35 @@ public class ComplexTypesDeltaTable "id", 0L, "array_info", ImmutableList.of(0, 1, 2, 3), "struct_info", ImmutableMap.of("id", 0L, "name", "0"), - "nested_struct_info", ImmutableMap.of("id", 0L, "name", "0", "nested", ImmutableMap.of("nested_int", 0, "nested_double", 1.0)), + "nested_struct_info", ImmutableMap.of("id", 0L, "name", "0", "nested", ImmutableMap.of("nested_int", 0, "nested_double", 1.0, "nested_decimal", BigDecimal.valueOf(0.23))), "map_info", ImmutableMap.of("key1", 1.0f, "key2", 1.0f) ), ImmutableMap.of( "id", 1L, "array_info", ImmutableList.of(1, 2, 3, 4), "struct_info", ImmutableMap.of("id", 1L, "name", "1"), - "nested_struct_info", ImmutableMap.of("id", 1L, "name", "1", "nested", ImmutableMap.of("nested_int", 1, "nested_double", 2.0)), + "nested_struct_info", ImmutableMap.of("id", 1L, "name", "1", "nested", ImmutableMap.of("nested_int", 1, "nested_double", 2.0, "nested_decimal", BigDecimal.valueOf(1.23))), "map_info", ImmutableMap.of("key1", 2.0f, "key2", 2.0f) ), ImmutableMap.of( "id", 2L, "array_info", ImmutableList.of(2, 3, 4, 5), "struct_info", ImmutableMap.of("id", 2L, "name", "2"), - "nested_struct_info", ImmutableMap.of("id", 2L, "name", "2", "nested", ImmutableMap.of("nested_int", 2, "nested_double", 3.0)), + "nested_struct_info", ImmutableMap.of("id", 2L, "name", "2", "nested", ImmutableMap.of("nested_int", 2, "nested_double", 3.0, "nested_decimal", BigDecimal.valueOf(2.23))), "map_info", ImmutableMap.of("key1", 3.0f, "key2", 3.0f) ), ImmutableMap.of( "id", 3L, "array_info", ImmutableList.of(3, 4, 5, 6), "struct_info", ImmutableMap.of("id", 3L, "name", "3"), - "nested_struct_info", ImmutableMap.of("id", 3L, "name", "3", "nested", ImmutableMap.of("nested_int", 3, "nested_double", 4.0)), + "nested_struct_info", ImmutableMap.of("id", 3L, "name", "3", "nested", ImmutableMap.of("nested_int", 3, "nested_double", 4.0, "nested_decimal", BigDecimal.valueOf(3.23))), "map_info", ImmutableMap.of("key1", 4.0f, "key2", 4.0f) ), ImmutableMap.of( "id", 4L, "array_info", ImmutableList.of(4, 5, 6, 7), "struct_info", ImmutableMap.of("id", 4L, "name", "4"), - "nested_struct_info", ImmutableMap.of("id", 4L, "name", "4", "nested", ImmutableMap.of("nested_int", 4, "nested_double", 5.0)), + "nested_struct_info", ImmutableMap.of("id", 4L, "name", "4", "nested", ImmutableMap.of("nested_int", 4, "nested_double", 5.0, "nested_decimal", BigDecimal.valueOf(4.23))), "map_info", ImmutableMap.of("key1", 5.0f, "key2", 5.0f) ) ) diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00000-ba651ee3-3de4-4d3e-8bff-04182c482c2f-c000.snappy.parquet.crc b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00000-ba651ee3-3de4-4d3e-8bff-04182c482c2f-c000.snappy.parquet.crc new file mode 100644 index 000000000000..92c3e2d8fba8 Binary files /dev/null and b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00000-ba651ee3-3de4-4d3e-8bff-04182c482c2f-c000.snappy.parquet.crc differ diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00000-f4353008-5e85-4a53-9b74-0cc7b853103a-c000.snappy.parquet.crc b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00000-f4353008-5e85-4a53-9b74-0cc7b853103a-c000.snappy.parquet.crc deleted file mode 100644 index a56f68f447ba..000000000000 Binary files a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00000-f4353008-5e85-4a53-9b74-0cc7b853103a-c000.snappy.parquet.crc and /dev/null differ diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00001-01efecb8-5771-4e91-834e-2a1cb6601eb8-c000.snappy.parquet.crc b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00001-01efecb8-5771-4e91-834e-2a1cb6601eb8-c000.snappy.parquet.crc deleted file mode 100644 index 6b7e86bcf54a..000000000000 Binary files a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00001-01efecb8-5771-4e91-834e-2a1cb6601eb8-c000.snappy.parquet.crc and /dev/null differ diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00001-72cd58ed-7979-4b19-bce2-303feb8f9c66-c000.snappy.parquet.crc b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00001-72cd58ed-7979-4b19-bce2-303feb8f9c66-c000.snappy.parquet.crc new file mode 100644 index 000000000000..7bba84c69e67 Binary files /dev/null and b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00001-72cd58ed-7979-4b19-bce2-303feb8f9c66-c000.snappy.parquet.crc differ diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00003-0a39e5f0-e30e-4b08-abf3-e4f79a9062f5-c000.snappy.parquet.crc b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00003-0a39e5f0-e30e-4b08-abf3-e4f79a9062f5-c000.snappy.parquet.crc new file mode 100644 index 000000000000..4b77b095336a Binary files /dev/null and b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00003-0a39e5f0-e30e-4b08-abf3-e4f79a9062f5-c000.snappy.parquet.crc differ diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00003-383f5a97-c624-4ef3-82a4-f3f273308e53-c000.snappy.parquet.crc b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00003-383f5a97-c624-4ef3-82a4-f3f273308e53-c000.snappy.parquet.crc deleted file mode 100644 index 88b089e95034..000000000000 Binary files a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00003-383f5a97-c624-4ef3-82a4-f3f273308e53-c000.snappy.parquet.crc and /dev/null differ diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00005-2f71a6be-840a-427d-ad22-7036bb052433-c000.snappy.parquet.crc b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00005-2f71a6be-840a-427d-ad22-7036bb052433-c000.snappy.parquet.crc new file mode 100644 index 000000000000..e669a893a30a Binary files /dev/null and b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00005-2f71a6be-840a-427d-ad22-7036bb052433-c000.snappy.parquet.crc differ diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00005-febee455-5e89-404a-bb38-f627c47eb20b-c000.snappy.parquet.crc b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00005-febee455-5e89-404a-bb38-f627c47eb20b-c000.snappy.parquet.crc deleted file mode 100644 index 7f4972520056..000000000000 Binary files a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00005-febee455-5e89-404a-bb38-f627c47eb20b-c000.snappy.parquet.crc and /dev/null differ diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00007-07d88387-16f9-4141-bc77-0106e7f28f7a-c000.snappy.parquet.crc b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00007-07d88387-16f9-4141-bc77-0106e7f28f7a-c000.snappy.parquet.crc deleted file mode 100644 index cd8fc7a087f6..000000000000 Binary files a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00007-07d88387-16f9-4141-bc77-0106e7f28f7a-c000.snappy.parquet.crc and /dev/null differ diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00007-35186f7d-8e26-4b92-a61f-b9a3e5d8b986-c000.snappy.parquet.crc b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00007-35186f7d-8e26-4b92-a61f-b9a3e5d8b986-c000.snappy.parquet.crc new file mode 100644 index 000000000000..d464223f06fe Binary files /dev/null and b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00007-35186f7d-8e26-4b92-a61f-b9a3e5d8b986-c000.snappy.parquet.crc differ diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00009-73760316-7ace-43fe-b605-506c942cd969-c000.snappy.parquet.crc b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00009-73760316-7ace-43fe-b605-506c942cd969-c000.snappy.parquet.crc deleted file mode 100644 index 038082a933b7..000000000000 Binary files a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00009-73760316-7ace-43fe-b605-506c942cd969-c000.snappy.parquet.crc and /dev/null differ diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00009-cecabfb6-53c7-4526-abc6-69042b415d87-c000.snappy.parquet.crc b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00009-cecabfb6-53c7-4526-abc6-69042b415d87-c000.snappy.parquet.crc new file mode 100644 index 000000000000..92345f4f98a2 Binary files /dev/null and b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/.part-00009-cecabfb6-53c7-4526-abc6-69042b415d87-c000.snappy.parquet.crc differ diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/_delta_log/.00000000000000000000.json.crc b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/_delta_log/.00000000000000000000.json.crc index 311d2a22b04d..fca90d03dc1b 100644 Binary files a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/_delta_log/.00000000000000000000.json.crc and b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/_delta_log/.00000000000000000000.json.crc differ diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/_delta_log/00000000000000000000.json b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/_delta_log/00000000000000000000.json index 84803f9483ca..5383b51e0b72 100644 --- a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/_delta_log/00000000000000000000.json +++ b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/_delta_log/00000000000000000000.json @@ -1,8 +1,8 @@ -{"commitInfo":{"timestamp":1723511561738,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"6","numOutputRows":"5","numOutputBytes":"17937"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.1.0","txnId":"b9eae5f4-d55b-4c38-b365-8228ec09248e"}} -{"metaData":{"id":"ce998219-9bde-4831-b78c-14b11f919fbe","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"array_info\",\"type\":{\"type\":\"array\",\"elementType\":\"integer\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"struct_info\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"name\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"nested_struct_info\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"name\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"nested\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"nested_int\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"nested_double\",\"type\":\"double\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"map_info\",\"type\":{\"type\":\"map\",\"keyType\":\"string\",\"valueType\":\"float\",\"valueContainsNull\":true},\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1723511559184}} +{"commitInfo":{"timestamp":1729202194177,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"6","numOutputRows":"5","numOutputBytes":"19553"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.1.0","txnId":"ff724e89-210e-47dd-a31f-edec6f2c6f6f"}} +{"metaData":{"id":"68614ef0-e5ca-4caa-bbd9-b3427eecadc9","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"array_info\",\"type\":{\"type\":\"array\",\"elementType\":\"integer\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"struct_info\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"name\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"nested_struct_info\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"name\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"nested\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"nested_int\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"nested_double\",\"type\":\"double\",\"nullable\":true,\"metadata\":{}},{\"name\":\"nested_decimal\",\"type\":\"decimal(4,2)\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"map_info\",\"type\":{\"type\":\"map\",\"keyType\":\"string\",\"valueType\":\"float\",\"valueContainsNull\":true},\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1729202191427}} {"protocol":{"minReaderVersion":1,"minWriterVersion":2}} -{"add":{"path":"part-00001-01efecb8-5771-4e91-834e-2a1cb6601eb8-c000.snappy.parquet","partitionValues":{},"size":3288,"modificationTime":1723511561689,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":0,\"struct_info\":{\"id\":0,\"name\":\"0\"},\"nested_struct_info\":{\"id\":0,\"name\":\"0\",\"nested\":{\"nested_int\":0,\"nested_double\":1.0}}},\"maxValues\":{\"id\":0,\"struct_info\":{\"id\":0,\"name\":\"0\"},\"nested_struct_info\":{\"id\":0,\"name\":\"0\",\"nested\":{\"nested_int\":0,\"nested_double\":1.0}}},\"nullCount\":{\"id\":0,\"array_info\":0,\"struct_info\":{\"id\":0,\"name\":0},\"nested_struct_info\":{\"id\":0,\"name\":0,\"nested\":{\"nested_int\":0,\"nested_double\":0}},\"map_info\":0}}"}} -{"add":{"path":"part-00003-383f5a97-c624-4ef3-82a4-f3f273308e53-c000.snappy.parquet","partitionValues":{},"size":3291,"modificationTime":1723511561689,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":1,\"struct_info\":{\"id\":1,\"name\":\"1\"},\"nested_struct_info\":{\"id\":1,\"name\":\"1\",\"nested\":{\"nested_int\":1,\"nested_double\":2.0}}},\"maxValues\":{\"id\":1,\"struct_info\":{\"id\":1,\"name\":\"1\"},\"nested_struct_info\":{\"id\":1,\"name\":\"1\",\"nested\":{\"nested_int\":1,\"nested_double\":2.0}}},\"nullCount\":{\"id\":0,\"array_info\":0,\"struct_info\":{\"id\":0,\"name\":0},\"nested_struct_info\":{\"id\":0,\"name\":0,\"nested\":{\"nested_int\":0,\"nested_double\":0}},\"map_info\":0}}"}} -{"add":{"path":"part-00005-febee455-5e89-404a-bb38-f627c47eb20b-c000.snappy.parquet","partitionValues":{},"size":3289,"modificationTime":1723511561689,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":2,\"struct_info\":{\"id\":2,\"name\":\"2\"},\"nested_struct_info\":{\"id\":2,\"name\":\"2\",\"nested\":{\"nested_int\":2,\"nested_double\":3.0}}},\"maxValues\":{\"id\":2,\"struct_info\":{\"id\":2,\"name\":\"2\"},\"nested_struct_info\":{\"id\":2,\"name\":\"2\",\"nested\":{\"nested_int\":2,\"nested_double\":3.0}}},\"nullCount\":{\"id\":0,\"array_info\":0,\"struct_info\":{\"id\":0,\"name\":0},\"nested_struct_info\":{\"id\":0,\"name\":0,\"nested\":{\"nested_int\":0,\"nested_double\":0}},\"map_info\":0}}"}} -{"add":{"path":"part-00007-07d88387-16f9-4141-bc77-0106e7f28f7a-c000.snappy.parquet","partitionValues":{},"size":3290,"modificationTime":1723511561689,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":3,\"struct_info\":{\"id\":3,\"name\":\"3\"},\"nested_struct_info\":{\"id\":3,\"name\":\"3\",\"nested\":{\"nested_int\":3,\"nested_double\":4.0}}},\"maxValues\":{\"id\":3,\"struct_info\":{\"id\":3,\"name\":\"3\"},\"nested_struct_info\":{\"id\":3,\"name\":\"3\",\"nested\":{\"nested_int\":3,\"nested_double\":4.0}}},\"nullCount\":{\"id\":0,\"array_info\":0,\"struct_info\":{\"id\":0,\"name\":0},\"nested_struct_info\":{\"id\":0,\"name\":0,\"nested\":{\"nested_int\":0,\"nested_double\":0}},\"map_info\":0}}"}} -{"add":{"path":"part-00009-73760316-7ace-43fe-b605-506c942cd969-c000.snappy.parquet","partitionValues":{},"size":3291,"modificationTime":1723511561689,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":4,\"struct_info\":{\"id\":4,\"name\":\"4\"},\"nested_struct_info\":{\"id\":4,\"name\":\"4\",\"nested\":{\"nested_int\":4,\"nested_double\":5.0}}},\"maxValues\":{\"id\":4,\"struct_info\":{\"id\":4,\"name\":\"4\"},\"nested_struct_info\":{\"id\":4,\"name\":\"4\",\"nested\":{\"nested_int\":4,\"nested_double\":5.0}}},\"nullCount\":{\"id\":0,\"array_info\":0,\"struct_info\":{\"id\":0,\"name\":0},\"nested_struct_info\":{\"id\":0,\"name\":0,\"nested\":{\"nested_int\":0,\"nested_double\":0}},\"map_info\":0}}"}} +{"add":{"path":"part-00001-72cd58ed-7979-4b19-bce2-303feb8f9c66-c000.snappy.parquet","partitionValues":{},"size":3588,"modificationTime":1729202194127,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":0,\"struct_info\":{\"id\":0,\"name\":\"0\"},\"nested_struct_info\":{\"id\":0,\"name\":\"0\",\"nested\":{\"nested_int\":0,\"nested_double\":1.0,\"nested_decimal\":0.23}}},\"maxValues\":{\"id\":0,\"struct_info\":{\"id\":0,\"name\":\"0\"},\"nested_struct_info\":{\"id\":0,\"name\":\"0\",\"nested\":{\"nested_int\":0,\"nested_double\":1.0,\"nested_decimal\":0.23}}},\"nullCount\":{\"id\":0,\"array_info\":0,\"struct_info\":{\"id\":0,\"name\":0},\"nested_struct_info\":{\"id\":0,\"name\":0,\"nested\":{\"nested_int\":0,\"nested_double\":0,\"nested_decimal\":0}},\"map_info\":0}}"}} +{"add":{"path":"part-00003-0a39e5f0-e30e-4b08-abf3-e4f79a9062f5-c000.snappy.parquet","partitionValues":{},"size":3591,"modificationTime":1729202194127,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":1,\"struct_info\":{\"id\":1,\"name\":\"1\"},\"nested_struct_info\":{\"id\":1,\"name\":\"1\",\"nested\":{\"nested_int\":1,\"nested_double\":2.0,\"nested_decimal\":1.23}}},\"maxValues\":{\"id\":1,\"struct_info\":{\"id\":1,\"name\":\"1\"},\"nested_struct_info\":{\"id\":1,\"name\":\"1\",\"nested\":{\"nested_int\":1,\"nested_double\":2.0,\"nested_decimal\":1.23}}},\"nullCount\":{\"id\":0,\"array_info\":0,\"struct_info\":{\"id\":0,\"name\":0},\"nested_struct_info\":{\"id\":0,\"name\":0,\"nested\":{\"nested_int\":0,\"nested_double\":0,\"nested_decimal\":0}},\"map_info\":0}}"}} +{"add":{"path":"part-00005-2f71a6be-840a-427d-ad22-7036bb052433-c000.snappy.parquet","partitionValues":{},"size":3589,"modificationTime":1729202194127,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":2,\"struct_info\":{\"id\":2,\"name\":\"2\"},\"nested_struct_info\":{\"id\":2,\"name\":\"2\",\"nested\":{\"nested_int\":2,\"nested_double\":3.0,\"nested_decimal\":2.23}}},\"maxValues\":{\"id\":2,\"struct_info\":{\"id\":2,\"name\":\"2\"},\"nested_struct_info\":{\"id\":2,\"name\":\"2\",\"nested\":{\"nested_int\":2,\"nested_double\":3.0,\"nested_decimal\":2.23}}},\"nullCount\":{\"id\":0,\"array_info\":0,\"struct_info\":{\"id\":0,\"name\":0},\"nested_struct_info\":{\"id\":0,\"name\":0,\"nested\":{\"nested_int\":0,\"nested_double\":0,\"nested_decimal\":0}},\"map_info\":0}}"}} +{"add":{"path":"part-00007-35186f7d-8e26-4b92-a61f-b9a3e5d8b986-c000.snappy.parquet","partitionValues":{},"size":3590,"modificationTime":1729202194127,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":3,\"struct_info\":{\"id\":3,\"name\":\"3\"},\"nested_struct_info\":{\"id\":3,\"name\":\"3\",\"nested\":{\"nested_int\":3,\"nested_double\":4.0,\"nested_decimal\":3.23}}},\"maxValues\":{\"id\":3,\"struct_info\":{\"id\":3,\"name\":\"3\"},\"nested_struct_info\":{\"id\":3,\"name\":\"3\",\"nested\":{\"nested_int\":3,\"nested_double\":4.0,\"nested_decimal\":3.23}}},\"nullCount\":{\"id\":0,\"array_info\":0,\"struct_info\":{\"id\":0,\"name\":0},\"nested_struct_info\":{\"id\":0,\"name\":0,\"nested\":{\"nested_int\":0,\"nested_double\":0,\"nested_decimal\":0}},\"map_info\":0}}"}} +{"add":{"path":"part-00009-cecabfb6-53c7-4526-abc6-69042b415d87-c000.snappy.parquet","partitionValues":{},"size":3591,"modificationTime":1729202194127,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":4,\"struct_info\":{\"id\":4,\"name\":\"4\"},\"nested_struct_info\":{\"id\":4,\"name\":\"4\",\"nested\":{\"nested_int\":4,\"nested_double\":5.0,\"nested_decimal\":4.23}}},\"maxValues\":{\"id\":4,\"struct_info\":{\"id\":4,\"name\":\"4\"},\"nested_struct_info\":{\"id\":4,\"name\":\"4\",\"nested\":{\"nested_int\":4,\"nested_double\":5.0,\"nested_decimal\":4.23}}},\"nullCount\":{\"id\":0,\"array_info\":0,\"struct_info\":{\"id\":0,\"name\":0},\"nested_struct_info\":{\"id\":0,\"name\":0,\"nested\":{\"nested_int\":0,\"nested_double\":0,\"nested_decimal\":0}},\"map_info\":0}}"}} diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00000-ba651ee3-3de4-4d3e-8bff-04182c482c2f-c000.snappy.parquet b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00000-ba651ee3-3de4-4d3e-8bff-04182c482c2f-c000.snappy.parquet new file mode 100644 index 000000000000..fdd3237e37cd Binary files /dev/null and b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00000-ba651ee3-3de4-4d3e-8bff-04182c482c2f-c000.snappy.parquet differ diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00000-f4353008-5e85-4a53-9b74-0cc7b853103a-c000.snappy.parquet b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00000-f4353008-5e85-4a53-9b74-0cc7b853103a-c000.snappy.parquet deleted file mode 100644 index 7a105c0f74ff..000000000000 Binary files a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00000-f4353008-5e85-4a53-9b74-0cc7b853103a-c000.snappy.parquet and /dev/null differ diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00001-01efecb8-5771-4e91-834e-2a1cb6601eb8-c000.snappy.parquet b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00001-01efecb8-5771-4e91-834e-2a1cb6601eb8-c000.snappy.parquet deleted file mode 100644 index bb2fb67389d2..000000000000 Binary files a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00001-01efecb8-5771-4e91-834e-2a1cb6601eb8-c000.snappy.parquet and /dev/null differ diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00001-72cd58ed-7979-4b19-bce2-303feb8f9c66-c000.snappy.parquet b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00001-72cd58ed-7979-4b19-bce2-303feb8f9c66-c000.snappy.parquet new file mode 100644 index 000000000000..8b6fdcfab647 Binary files /dev/null and b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00001-72cd58ed-7979-4b19-bce2-303feb8f9c66-c000.snappy.parquet differ diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00003-0a39e5f0-e30e-4b08-abf3-e4f79a9062f5-c000.snappy.parquet b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00003-0a39e5f0-e30e-4b08-abf3-e4f79a9062f5-c000.snappy.parquet new file mode 100644 index 000000000000..c349328307e7 Binary files /dev/null and b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00003-0a39e5f0-e30e-4b08-abf3-e4f79a9062f5-c000.snappy.parquet differ diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00003-383f5a97-c624-4ef3-82a4-f3f273308e53-c000.snappy.parquet b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00003-383f5a97-c624-4ef3-82a4-f3f273308e53-c000.snappy.parquet deleted file mode 100644 index 641396cb6f7c..000000000000 Binary files a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00003-383f5a97-c624-4ef3-82a4-f3f273308e53-c000.snappy.parquet and /dev/null differ diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00005-2f71a6be-840a-427d-ad22-7036bb052433-c000.snappy.parquet b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00005-2f71a6be-840a-427d-ad22-7036bb052433-c000.snappy.parquet new file mode 100644 index 000000000000..dcfcdd89bc0f Binary files /dev/null and b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00005-2f71a6be-840a-427d-ad22-7036bb052433-c000.snappy.parquet differ diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00005-febee455-5e89-404a-bb38-f627c47eb20b-c000.snappy.parquet b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00005-febee455-5e89-404a-bb38-f627c47eb20b-c000.snappy.parquet deleted file mode 100644 index abee0ea5d0a9..000000000000 Binary files a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00005-febee455-5e89-404a-bb38-f627c47eb20b-c000.snappy.parquet and /dev/null differ diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00007-07d88387-16f9-4141-bc77-0106e7f28f7a-c000.snappy.parquet b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00007-07d88387-16f9-4141-bc77-0106e7f28f7a-c000.snappy.parquet deleted file mode 100644 index 453c3879fa35..000000000000 Binary files a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00007-07d88387-16f9-4141-bc77-0106e7f28f7a-c000.snappy.parquet and /dev/null differ diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00007-35186f7d-8e26-4b92-a61f-b9a3e5d8b986-c000.snappy.parquet b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00007-35186f7d-8e26-4b92-a61f-b9a3e5d8b986-c000.snappy.parquet new file mode 100644 index 000000000000..ab5b52c9091a Binary files /dev/null and b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00007-35186f7d-8e26-4b92-a61f-b9a3e5d8b986-c000.snappy.parquet differ diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00009-73760316-7ace-43fe-b605-506c942cd969-c000.snappy.parquet b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00009-73760316-7ace-43fe-b605-506c942cd969-c000.snappy.parquet deleted file mode 100644 index aadf1e152c87..000000000000 Binary files a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00009-73760316-7ace-43fe-b605-506c942cd969-c000.snappy.parquet and /dev/null differ diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00009-cecabfb6-53c7-4526-abc6-69042b415d87-c000.snappy.parquet b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00009-cecabfb6-53c7-4526-abc6-69042b415d87-c000.snappy.parquet new file mode 100644 index 000000000000..507b56bbfc15 Binary files /dev/null and b/extensions-contrib/druid-deltalake-extensions/src/test/resources/complex-types-table/part-00009-cecabfb6-53c7-4526-abc6-69042b415d87-c000.snappy.parquet differ diff --git a/extensions-contrib/druid-deltalake-extensions/src/test/resources/create_delta_table.py b/extensions-contrib/druid-deltalake-extensions/src/test/resources/create_delta_table.py index ffca055e73c1..52cd3ed3de55 100755 --- a/extensions-contrib/druid-deltalake-extensions/src/test/resources/create_delta_table.py +++ b/extensions-contrib/druid-deltalake-extensions/src/test/resources/create_delta_table.py @@ -17,10 +17,10 @@ import argparse from enum import Enum - +from decimal import Decimal from delta import * import pyspark -from pyspark.sql.types import MapType, StructType, StructField, ShortType, StringType, TimestampType, LongType, IntegerType, DoubleType, FloatType, DateType, BooleanType, ArrayType +from pyspark.sql.types import MapType, StructType, StructField, ShortType, StringType, TimestampType, LongType, IntegerType, DoubleType, FloatType, DateType, BooleanType, ArrayType, DecimalType from pyspark.sql.functions import expr from datetime import datetime, timedelta import random @@ -73,6 +73,7 @@ def create_dataset_with_complex_types(num_records): StructField("nested", StructType([ StructField("nested_int", IntegerType(), False), StructField("nested_double", DoubleType(), True), + StructField("nested_decimal", DecimalType(4, 2), True), ])) ])), StructField("map_info", MapType(StringType(), FloatType())) @@ -85,7 +86,7 @@ def create_dataset_with_complex_types(num_records): idx, (idx, idx + 1, idx + 2, idx + 3), (idx, f"{idx}"), - (idx, f"{idx}", (idx, idx + 1.0)), + (idx, f"{idx}", (idx, idx + 1.0, Decimal(idx + 0.23))), {"key1": idx + 1.0, "key2": idx + 1.0} ) data.append(record)