From 02b20c0c43956589dc5c48fb36cab145fbc6c57c Mon Sep 17 00:00:00 2001 From: zouxxyy Date: Thu, 9 Jan 2025 15:48:05 +0800 Subject: [PATCH] v1 --- .../paimon/types/DataTypeJsonParser.java | 32 ++++++++++--- .../java/org/apache/paimon/types/RowType.java | 6 ++- .../paimon/schema/DataTypeJsonParserTest.java | 45 +++++++++++++++++-- 3 files changed, 72 insertions(+), 11 deletions(-) diff --git a/paimon-common/src/main/java/org/apache/paimon/types/DataTypeJsonParser.java b/paimon-common/src/main/java/org/apache/paimon/types/DataTypeJsonParser.java index 19f2dbfe7b4f..40790f06fb6f 100644 --- a/paimon-common/src/main/java/org/apache/paimon/types/DataTypeJsonParser.java +++ b/paimon-common/src/main/java/org/apache/paimon/types/DataTypeJsonParser.java @@ -26,9 +26,12 @@ import java.util.Iterator; import java.util.List; import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import java.util.stream.Stream; +import static org.apache.paimon.utils.Preconditions.checkState; + /** * Parser for creating instances of {@link org.apache.paimon.types.DataType} from a serialized * string created with {@link org.apache.paimon.types.DataType#serializeJson}. @@ -36,9 +39,20 @@ public final class DataTypeJsonParser { public static DataField parseDataField(JsonNode json) { - int id = json.get("id").asInt(); + return parseDataField(json, null); + } + + private static DataField parseDataField(JsonNode json, AtomicInteger fieldId) { + int id; + JsonNode idNode = json.get("id"); + if (idNode != null) { + checkState(fieldId == null || fieldId.get() == -1, "Partial field id is not allowed."); + id = idNode.asInt(); + } else { + id = fieldId.incrementAndGet(); + } String name = json.get("name").asText(); - DataType type = parseDataType(json.get("type")); + DataType type = parseDataType(json.get("type"), fieldId); JsonNode descriptionNode = json.get("description"); String description = null; if (descriptionNode != null) { @@ -48,26 +62,30 @@ public static DataField parseDataField(JsonNode json) { } public static DataType parseDataType(JsonNode json) { + return parseDataType(json, new AtomicInteger(-1)); + } + + public static DataType parseDataType(JsonNode json, AtomicInteger fieldId) { if (json.isTextual()) { return parseAtomicTypeSQLString(json.asText()); } else if (json.isObject()) { String typeString = json.get("type").asText(); if (typeString.startsWith("ARRAY")) { - DataType element = parseDataType(json.get("element")); + DataType element = parseDataType(json.get("element"), fieldId); return new ArrayType(!typeString.contains("NOT NULL"), element); } else if (typeString.startsWith("MULTISET")) { - DataType element = parseDataType(json.get("element")); + DataType element = parseDataType(json.get("element"), fieldId); return new MultisetType(!typeString.contains("NOT NULL"), element); } else if (typeString.startsWith("MAP")) { - DataType key = parseDataType(json.get("key")); - DataType value = parseDataType(json.get("value")); + DataType key = parseDataType(json.get("key"), fieldId); + DataType value = parseDataType(json.get("value"), fieldId); return new MapType(!typeString.contains("NOT NULL"), key, value); } else if (typeString.startsWith("ROW")) { JsonNode fieldArray = json.get("fields"); Iterator iterator = fieldArray.elements(); List fields = new ArrayList<>(fieldArray.size()); while (iterator.hasNext()) { - fields.add(parseDataField(iterator.next())); + fields.add(parseDataField(iterator.next(), fieldId)); } return new RowType(!typeString.contains("NOT NULL"), fields); } diff --git a/paimon-common/src/main/java/org/apache/paimon/types/RowType.java b/paimon-common/src/main/java/org/apache/paimon/types/RowType.java index fecb5bed9ebd..681a07af584d 100644 --- a/paimon-common/src/main/java/org/apache/paimon/types/RowType.java +++ b/paimon-common/src/main/java/org/apache/paimon/types/RowType.java @@ -356,7 +356,11 @@ public static int currentHighestFieldId(List fields) { } public static Builder builder() { - return builder(true, new AtomicInteger(-1)); + return builder(new AtomicInteger(-1)); + } + + public static Builder builder(AtomicInteger fieldId) { + return builder(true, fieldId); } public static Builder builder(boolean isNullable, AtomicInteger fieldId) { diff --git a/paimon-core/src/test/java/org/apache/paimon/schema/DataTypeJsonParserTest.java b/paimon-core/src/test/java/org/apache/paimon/schema/DataTypeJsonParserTest.java index 52ecff282cac..2397af83aa2f 100644 --- a/paimon-core/src/test/java/org/apache/paimon/schema/DataTypeJsonParserTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/schema/DataTypeJsonParserTest.java @@ -26,6 +26,7 @@ import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataType; import org.apache.paimon.types.DataTypeJsonParser; +import org.apache.paimon.types.DataTypes; import org.apache.paimon.types.DateType; import org.apache.paimon.types.DecimalType; import org.apache.paimon.types.DoubleType; @@ -43,6 +44,7 @@ import org.apache.paimon.types.VarCharType; import org.apache.paimon.utils.JsonSerdeUtil; +import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; @@ -50,6 +52,7 @@ import java.util.Arrays; import java.util.Collections; +import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Stream; import static org.assertj.core.api.Assertions.assertThat; @@ -199,6 +202,44 @@ void testErrorMessage(TestSpec testSpec) { } } + @Test + void testParingRowTypeWithoutFieldId() { + String jsonString1 = + "{\"type\":\"ROW\",\"fields\":[{\"name\":\"field1\",\"type\":\"INT\"},{\"name\":\"field2\",\"type\":\"STRING\"}]}"; + RowType rowType1 = + RowType.builder() + .fields( + new DataType[] {DataTypes.INT(), DataTypes.STRING()}, + new String[] {"field1", "field2"}) + .build(); + assertThat(parse(jsonString1)).isEqualTo(rowType1); + + String jsonString2 = + "{\"type\":\"ROW\",\"fields\":[{\"name\":\"field1\",\"type\":\"INT\"},{\"name\":\"field2\",\"type\":{\"type\":\"ROW\",\"fields\":[{\"name\":\"s1\",\"type\":\"INT\"},{\"name\":\"s2\",\"type\":\"STRING\"}]}}]}"; + RowType rowType2 = + RowType.builder() + .fields( + new DataType[] { + DataTypes.INT(), + RowType.builder(new AtomicInteger(1)) + .fields( + new DataType[] { + DataTypes.INT(), DataTypes.STRING() + }, + new String[] {"s1", "s2"}) + .build() + }, + new String[] {"field1", "field2"}) + .build(); + assertThat(parse(jsonString2)).isEqualTo(rowType2); + + String jsonString3 = + "{\"type\":\"ROW\",\"fields\":[{\"name\":\"field1\",\"type\":\"INT\"},{\"id\":1, \"name\":\"field2\",\"type\":\"STRING\"}]}"; + assertThatThrownBy(() -> parse(jsonString3)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("Partial field id is not allowed."); + } + private static String toJson(DataType type) { return JsonSerdeUtil.toFlatJson(type); } @@ -207,9 +248,7 @@ private static DataType parse(String json) { if (!json.startsWith("\"") && !json.startsWith("{")) { json = "\"" + json + "\""; } - String dataFieldJson = - String.format("{\"id\": 0, \"name\": \"dummy\", \"type\": %s}", json); - return JsonSerdeUtil.fromJson(dataFieldJson, DataField.class).type(); + return JsonSerdeUtil.fromJson(json, DataType.class); } // --------------------------------------------------------------------------------------------