diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/ColumnDefinition.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/ColumnDefinition.java index 2a829e1950..aa26bfc5ad 100644 --- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/ColumnDefinition.java +++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/ColumnDefinition.java @@ -24,11 +24,6 @@ public class ColumnDefinition extends DataTypeExpr { @Expose @SerializedName("ColumnDefName") String columnName; - public ColumnDefinition(String columnName, String dataType) { - super(dataType); - this.columnName = columnName; - } - public ColumnDefinition(String columnName, DataTypeExpr dataType) { super(dataType.getTypeName()); @@ -42,15 +37,17 @@ public ColumnDefinition(String columnName, DataTypeExpr dataType) { } } - // nested records if (dataType.isRecordType()) { this.recordType = dataType.recordType; } - // map type if (dataType.isMapType()) { this.mapType = dataType.mapType; } + + if (dataType.isArrayType()) { + this.arrayType = dataType.arrayType; + } } public String getColumnName() { diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/DataTypeExpr.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/DataTypeExpr.java index d63532dbb3..bd5caac91b 100644 --- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/DataTypeExpr.java +++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/DataTypeExpr.java @@ -19,11 +19,15 @@ package org.apache.tajo.algebra; import com.google.common.base.Objects; +import com.google.common.base.Preconditions; import com.google.gson.annotations.Expose; import com.google.gson.annotations.SerializedName; +import org.apache.tajo.Assert; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.util.TUtil; +import static org.apache.tajo.Assert.assertNotNull; + public class DataTypeExpr extends Expr { @Expose @SerializedName("DataTypeName") String typeName; @@ -31,6 +35,8 @@ public class DataTypeExpr extends Expr { Integer lengthOrPrecision; @Expose @SerializedName("Scale") Integer scale; + @Expose @SerializedName("Array") + ArrayType arrayType; // not null if the type is ARRAY @Expose @SerializedName("Record") RecordType recordType; // not null if the type is RECORD @Expose @SerializedName("Map") @@ -38,18 +44,27 @@ public class DataTypeExpr extends Expr { public DataTypeExpr(String typeName) { super(OpType.DataType); + assertNotNull(typeName); this.typeName = typeName; } + public DataTypeExpr(ArrayType array) { + super(OpType.DataType); + assertNotNull(array); + this.typeName = Type.ARRAY.name(); + this.arrayType = array; + } + public DataTypeExpr(RecordType record) { super(OpType.DataType); + assertNotNull(record); this.typeName = Type.RECORD.name(); this.recordType = record; } public DataTypeExpr(MapType map) { super(OpType.DataType); - // RECORD = 51 in DataTypes.proto + assertNotNull(map); this.typeName = Type.MAP.name(); this.mapType = map; } @@ -59,15 +74,34 @@ public String getTypeName() { } public boolean isPrimitiveType() { - return !this.isRecordType() && !isMapType(); + return !isArrayType()&& !isRecordType() && !isMapType(); + } + + public boolean isArrayType() { + return arrayType != null; } public boolean isRecordType() { - return this.typeName.equals(Type.RECORD.name()); + return recordType != null; } public boolean isMapType() { - return this.typeName.equals(Type.MAP.name()); + return mapType != null; + } + + public DataTypeExpr getElementType() { + Preconditions.checkState(isArrayType()); + return arrayType.type; + } + + public DataTypeExpr getKeyType() { + Preconditions.checkState(isMapType()); + return mapType.keyType; + } + + public DataTypeExpr getValueType() { + Preconditions.checkState(isMapType()); + return mapType.valueType; } public ColumnDefinition [] getNestedRecordTypes() { @@ -125,6 +159,27 @@ public Object clone() throws CloneNotSupportedException { return dataType; } + public static class ArrayType implements JsonSerializable, Cloneable { + @Expose + @SerializedName("type") + DataTypeExpr type; + + public ArrayType(DataTypeExpr elementType) { + this.type = elementType; + } + + @Override + public String toJson() { + return JsonHelper.toJson(this); + } + + public Object clone() throws CloneNotSupportedException { + ArrayType newMap = (ArrayType) super.clone(); + newMap.type = type; + return newMap; + } + } + public static class RecordType implements JsonSerializable, Cloneable { @Expose @SerializedName("Schema") ColumnDefinition [] schema; // not null if the type is RECORD diff --git a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java index b0acbb12a4..2c4d41e72b 100644 --- a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java +++ b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java @@ -32,6 +32,7 @@ import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.NullProto; import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.ReturnState; import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.StringListResponse; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.util.ProtoUtil; import java.io.Closeable; @@ -281,7 +282,7 @@ public final TableDesc getTableDesc(final String databaseName, final String tabl @Override public TableDesc getTableDesc(String qualifiedName) throws UndefinedTableException { - String[] splitted = CatalogUtil.splitFQTableName(qualifiedName); + String[] splitted = IdentifierUtil.splitFQTableName(qualifiedName); return getTableDesc(splitted[0], splitted[1]); } @@ -596,7 +597,7 @@ public final void createTable(final TableDesc desc) public void dropTable(String tableName) throws UndefinedDatabaseException, UndefinedTableException, InsufficientPrivilegeException { - String[] splitted = CatalogUtil.splitFQTableName(tableName); + String[] splitted = IdentifierUtil.splitFQTableName(tableName); final String databaseName = splitted[0]; final String simpleName = splitted[1]; @@ -617,7 +618,7 @@ public void dropTable(String tableName) @Override public final boolean existsTable(final String databaseName, final String tableName) { - if (CatalogUtil.isFQTableName(tableName)) { + if (IdentifierUtil.isFQTableName(tableName)) { throw new IllegalArgumentException( "tableName cannot be composed of multiple parts, but it is \"" + tableName + "\""); } @@ -640,7 +641,7 @@ public final boolean existsTable(final String databaseName, final String tableNa @Override public final boolean existsTable(final String tableName) { - String[] splitted = CatalogUtil.splitFQTableName(tableName); + String[] splitted = IdentifierUtil.splitFQTableName(tableName); return existsTable(splitted[0], splitted[1]); } diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/DataTypeUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/DataTypeUtil.java index 9cbdbe0391..69206aa996 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/DataTypeUtil.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/DataTypeUtil.java @@ -87,8 +87,8 @@ public static boolean isUpperCastable(TajoDataTypes.Type define, TajoDataTypes.T */ public static org.apache.tajo.type.Type determineType(org.apache.tajo.type.Type left, org.apache.tajo.type.Type right) { - TajoDataTypes.Type rhsBaseType = right.baseType(); - switch (left.baseType()) { + TajoDataTypes.Type rhsBaseType = right.kind(); + switch (left.kind()) { case INT1: case INT2: diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogConstants.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogConstants.java index f2acf983db..6acbc34e7f 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogConstants.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogConstants.java @@ -19,10 +19,6 @@ package org.apache.tajo.catalog; public class CatalogConstants { - public final static String IDENTIFIER_DELIMITER_REGEXP = "\\."; - public final static String IDENTIFIER_DELIMITER = "."; - public final static String IDENTIFIER_QUOTE_STRING = "\""; - public final static int MAX_IDENTIFIER_LENGTH = 128; // Linux and BSD's max username length is 32. For compatibility with other systems, we should follow it. public final static int MAX_USERNAME_LENGTH = 32; public final static int MAX_STATEMENT_LENGTH = 128 * 1024; diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java index 2b20907c44..22dd5a4850 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java @@ -18,12 +18,10 @@ package org.apache.tajo.catalog; -import com.google.common.base.Preconditions; import com.google.common.collect.Maps; import org.apache.hadoop.fs.Path; import org.apache.tajo.BuiltinStorages; import org.apache.tajo.DataTypeUtil; -import org.apache.tajo.TajoConstants; import org.apache.tajo.annotation.Nullable; import org.apache.tajo.catalog.partition.PartitionDesc; import org.apache.tajo.catalog.partition.PartitionMethodDesc; @@ -35,6 +33,7 @@ import org.apache.tajo.datum.NullDatum; import org.apache.tajo.exception.InvalidOperationException; import org.apache.tajo.exception.UndefinedOperatorException; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.storage.StorageConstants; import org.apache.tajo.util.KeyValueSet; import org.apache.tajo.util.Pair; @@ -51,233 +50,6 @@ public class CatalogUtil { - /** - * Normalize an identifier. Normalization means a translation from a identifier to be a refined identifier name. - * - * Identifier can be composed of multiple parts as follows: - * 
-   *   database_name.table_name.column_name
-   *
- * - * Each regular identifier part can be composed alphabet ([a-z][A-Z]), number([0-9]), and underscore([_]). - * Also, the first letter must be an alphabet character. - * - * normalizeIdentifier normalizes each part of an identifier. - * - * In detail, for each part, it performs as follows: - * - * - * @param identifier The identifier to be normalized - * @return The normalized identifier - */ - public static String normalizeIdentifier(String identifier) { - if (identifier == null || identifier.equals("")) { - return identifier; - } - String [] splitted = identifier.split(CatalogConstants.IDENTIFIER_DELIMITER_REGEXP); - - StringBuilder sb = new StringBuilder(); - boolean first = true; - for (String part : splitted) { - if (first) { - first = false; - } else { - sb.append(CatalogConstants.IDENTIFIER_DELIMITER); - } - sb.append(normalizeIdentifierPart(part)); - } - return sb.toString(); - } - - public static String normalizeIdentifierPart(String part) { - return isDelimited(part) ? stripQuote(part) : part.toLowerCase(); - } - - /** - * Denormalize an identifier. Denormalize means a translation from a stored identifier - * to be a printable identifier name. - * - * In detail, for each part, it performs as follows: - * - * - * @param identifier The identifier to be normalized - * @return The denormalized identifier - */ - public static String denormalizeIdentifier(String identifier) { - String [] splitted = identifier.split(CatalogConstants.IDENTIFIER_DELIMITER_REGEXP); - - StringBuilder sb = new StringBuilder(); - boolean first = true; - for (String part : splitted) { - if (first) { - first = false; - } else { - sb.append(CatalogConstants.IDENTIFIER_DELIMITER); - } - sb.append(denormalizePart(part)); - } - return sb.toString(); - } - - public static String denormalizePart(String identifier) { - if (isShouldBeQuoted(identifier)) { - return StringUtils.doubleQuote(identifier); - } else { - return identifier; - } - } - - public static boolean isShouldBeQuoted(String columnName) { - for (char character : columnName.toCharArray()) { - if (Character.isUpperCase(character)) { - return true; - } - - if (!StringUtils.isPartOfAnsiSQLIdentifier(character)) { - return true; - } - - if (RESERVED_KEYWORDS_SET.contains(columnName.toUpperCase())) { - return true; - } - } - - return false; - } - - public static String stripQuote(String str) { - return str.substring(1, str.length() - 1); - } - - public static boolean isDelimited(String identifier) { - boolean openQuote = identifier.charAt(0) == '"'; - boolean closeQuote = identifier.charAt(identifier.length() - 1) == '"'; - - // if at least one quote mark exists, the identifier must be grater than equal to 2 characters, - if (openQuote ^ closeQuote && identifier.length() < 2) { - throw new IllegalArgumentException("Invalid Identifier: " + identifier); - } - - // does not allow the empty identifier (''), - if (openQuote && closeQuote && identifier.length() == 2) { - throw new IllegalArgumentException("zero-length delimited identifier: " + identifier); - } - - // Ensure the quote open and close - return openQuote && closeQuote; - } - - /** - * True if a given name is a simple identifier, meaning is not a dot-chained name. - * - * @param columnOrTableName Column or Table name to be checked - * @return True if a given name is a simple identifier. Otherwise, it will return False. - */ - public static boolean isSimpleIdentifier(String columnOrTableName) { - return columnOrTableName.split(CatalogConstants.IDENTIFIER_DELIMITER_REGEXP).length == 1; - } - - public static boolean isFQColumnName(String tableName) { - return tableName.split(CatalogConstants.IDENTIFIER_DELIMITER_REGEXP).length == 3; - } - - public static boolean isFQTableName(String tableName) { - int lastDelimiterIdx = tableName.lastIndexOf(CatalogConstants.IDENTIFIER_DELIMITER); - return lastDelimiterIdx > -1; - } - - public static String [] splitFQTableName(String qualifiedName) { - String [] splitted = CatalogUtil.splitTableName(qualifiedName); - if (splitted.length == 1) { - throw new IllegalArgumentException("Table name is expected to be qualified, but was \"" - + qualifiedName + "\"."); - } - return splitted; - } - - public static String [] splitTableName(String tableName) { - int lastDelimiterIdx = tableName.lastIndexOf(CatalogConstants.IDENTIFIER_DELIMITER); - if (lastDelimiterIdx > -1) { - return new String [] { - tableName.substring(0, lastDelimiterIdx), - tableName.substring(lastDelimiterIdx + 1, tableName.length()) - }; - } else { - return new String [] {tableName}; - } - } - - public static String buildFQName(String... identifiers) { - boolean first = true; - StringBuilder sb = new StringBuilder(); - for(String id : identifiers) { - if (first) { - first = false; - } else { - sb.append(CatalogConstants.IDENTIFIER_DELIMITER); - } - - sb.append(id); - } - - return sb.toString(); - } - - public static Pair separateQualifierAndName(String name) { - Preconditions.checkArgument(isFQTableName(name), "Must be a qualified name."); - return new Pair<>(extractQualifier(name), extractSimpleName(name)); - } - - /** - * Extract a qualification name from an identifier. - * - * For example, consider a table identifier like 'database1.table1'. - * In this case, this method extracts 'database1'. - * - * @param name The identifier to be extracted - * @return The extracted qualifier - */ - public static String extractQualifier(String name) { - int lastDelimiterIdx = name.lastIndexOf(CatalogConstants.IDENTIFIER_DELIMITER); - if (lastDelimiterIdx > -1) { - return name.substring(0, lastDelimiterIdx); - } else { - return TajoConstants.EMPTY_STRING; - } - } - - /** - * Extract a simple name from an identifier. - * - * For example, consider a table identifier like 'database1.table1'. - * In this case, this method extracts 'table1'. - * - * @param name The identifier to be extracted - * @return The extracted simple name - */ - public static String extractSimpleName(String name) { - int lastDelimiterIdx = name.lastIndexOf(CatalogConstants.IDENTIFIER_DELIMITER); - if (lastDelimiterIdx > -1) { - // plus one means skipping a delimiter. - return name.substring(lastDelimiterIdx + 1, name.length()); - } else { - return name; - } - } - - public static String getCanonicalTableName(String databaseName, String tableName) { - StringBuilder sb = new StringBuilder(databaseName); - sb.append(CatalogConstants.IDENTIFIER_DELIMITER); - sb.append(tableName); - return sb.toString(); - } - public static String getBackwardCompitableDataFormat(String dataFormat) { return getDataFormatAsString(asDataFormat(dataFormat)); @@ -400,7 +172,7 @@ public static DataType newDataTypeWithLen(Type type, int length) { } public static String columnToDDLString(Column column) { - StringBuilder sb = new StringBuilder(denormalizeIdentifier(column.getSimpleName())); + StringBuilder sb = new StringBuilder(IdentifierUtil.denormalizeIdentifier(column.getSimpleName())); sb.append(" ").append(column.getDataType().getType()); if (column.getDataType().hasLength()) { sb.append(" (").append(column.getDataType().getLength()).append(")"); @@ -738,32 +510,8 @@ public static void closeQuietly(Statement stmt, ResultSet res) { } } - public static final Set RESERVED_KEYWORDS_SET = new HashSet<>(); - - static final String [] RESERVED_KEYWORDS = { - "AS", "ALL", "AND", "ANY", "ASYMMETRIC", "ASC", - "BOTH", - "CASE", "CAST", "CREATE", "CROSS", "CURRENT_DATE", "CURRENT_TIME", "CURRENT_TIMESTAMP", - "DESC", "DISTINCT", - "END", "ELSE", "EXCEPT", - "FALSE", "FULL", "FROM", - "GROUP", - "HAVING", - "ILIKE", "IN", "INNER", "INTERSECT", "INTO", "IS", - "JOIN", - "LEADING", "LEFT", "LIKE", "LIMIT", - "NATURAL", "NOT", "NULL", - "ON", "OUTER", "OR", "ORDER", - "RIGHT", - "SELECT", "SOME", "SYMMETRIC", - "TABLE", "THEN", "TRAILING", "TRUE", - "OVER", - "UNION", "UNIQUE", "USING", - "WHEN", "WHERE", "WINDOW", "WITH" - }; - static { - Collections.addAll(RESERVED_KEYWORDS_SET, RESERVED_KEYWORDS); + Collections.addAll(IdentifierUtil.RESERVED_KEYWORDS_SET, IdentifierUtil.RESERVED_KEYWORDS); } public static AlterTableDesc renameColumn(String tableName, String oldColumName, String newColumName, @@ -1045,7 +793,7 @@ public static KeyValueSet newDefaultProperty(String dataFormat, TajoConf conf) { public static String getUnifiedSimpleColumnName(Schema originalSchema, String[] columnNames) { String[] simpleNames = new String[columnNames.length]; for (int i = 0; i < simpleNames.length; i++) { - String[] identifiers = columnNames[i].split(CatalogConstants.IDENTIFIER_DELIMITER_REGEXP); + String[] identifiers = columnNames[i].split(IdentifierUtil.IDENTIFIER_DELIMITER_REGEXP); simpleNames[i] = identifiers[identifiers.length-1]; } Arrays.sort(simpleNames, new ColumnPosComparator(originalSchema)); diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java index 641b674cc8..ddf0c8329c 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java @@ -26,24 +26,27 @@ import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.common.TajoDataTypes.DataType; import org.apache.tajo.json.GsonObject; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.type.Type; +import org.apache.tajo.type.TypeProtobufEncoder; /** * Describes a column. It is an immutable object. */ +@Deprecated public class Column implements ProtoObject, GsonObject { @Expose protected String name; - @Expose protected TypeDesc typeDesc; + @Expose protected Type type; /** * Column Constructor * * @param name field name - * @param typeDesc Type description + * @param type Type description */ - public Column(String name, TypeDesc typeDesc) { + public Column(String name, TypeDesc type) { this.name = name; - this.typeDesc = typeDesc; + this.type = TypeConverter.convert(type); } /** @@ -70,22 +73,13 @@ public Column(String name, TajoDataTypes.Type type) { * @param type Type */ public Column(String name, Type type) { - this(name, TypeConverter.convert(type)); - } - - /** - * - * @param name Column name - * @param type Data Type - * @param typeLength The length of type - */ - public Column(String name, TajoDataTypes.Type type, int typeLength) { - this(name, CatalogUtil.newDataTypeWithLen(type, typeLength)); + this.name = name; + this.type = type; } public Column(ColumnProto proto) { name = proto.getName(); - typeDesc = new TypeDesc(proto.getDataType()); + type = TypeProtobufEncoder.decode(proto.getType()); } /** @@ -93,7 +87,7 @@ public Column(ColumnProto proto) { * @return True if a column includes a table name. Otherwise, it returns False. */ public boolean hasQualifier() { - return name.split(CatalogConstants.IDENTIFIER_DELIMITER_REGEXP).length > 1; + return name.split(IdentifierUtil.IDENTIFIER_DELIMITER_REGEXP).length > 1; } /** @@ -109,7 +103,7 @@ public String getQualifiedName() { * @return The qualifier */ public String getQualifier() { - return CatalogUtil.extractQualifier(name); + return IdentifierUtil.extractQualifier(name); } /** @@ -117,7 +111,7 @@ public String getQualifier() { * @return The simple name without qualifications */ public String getSimpleName() { - return CatalogUtil.extractSimpleName(name); + return IdentifierUtil.extractSimpleName(name); } /** @@ -126,7 +120,14 @@ public String getSimpleName() { * @return TypeDesc */ public TypeDesc getTypeDesc() { - return this.typeDesc; + return TypeConverter.convert(this.type); + } + + /** + * @return Type which includes domain type and scale. + */ + public Type getType() { + return this.type; } /** @@ -134,21 +135,20 @@ public TypeDesc getTypeDesc() { * @return DataType which includes domain type and scale. */ public DataType getDataType() { - return this.typeDesc.dataType; + return TypeConverter.convert(this.type).getDataType(); } @Override public boolean equals(Object o) { if (o instanceof Column) { Column another = (Column)o; - return name.equals(another.name) && typeDesc.equals(another.typeDesc); + return name.equals(another.name) && type.equals(another.type); } return false; } public int hashCode() { - return Objects.hashCode(name, typeDesc); - + return Objects.hashCode(name, type); } /** @@ -160,13 +160,13 @@ public ColumnProto getProto() { ColumnProto.Builder builder = ColumnProto.newBuilder(); builder .setName(this.name) - .setDataType(this.typeDesc.getDataType()); + .setType(this.type.getProto()); return builder.build(); } public String toString() { StringBuilder sb = new StringBuilder(getQualifiedName()); - sb.append(" (").append(typeDesc.toString()).append(")"); + sb.append(" (").append(type.toString()).append(")"); return sb.toString(); } diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java index e2250eb9ef..387c669b5e 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java @@ -21,6 +21,7 @@ import com.google.common.base.Function; import org.apache.tajo.catalog.partition.PartitionMethodDesc; import org.apache.tajo.catalog.proto.CatalogProtos.PartitionDescProto; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.util.KeyValueSet; import org.apache.tajo.util.StringUtils; @@ -36,11 +37,11 @@ public static String buildDDLForExternalTable(TableDesc desc) { StringBuilder sb = new StringBuilder(); sb.append("--\n") - .append("-- Name: ").append(CatalogUtil.denormalizeIdentifier(desc.getName())).append("; Type: TABLE;") + .append("-- Name: ").append(IdentifierUtil.denormalizeIdentifier(desc.getName())).append("; Type: TABLE;") .append(" Storage: ").append(desc.getMeta().getDataFormat()); sb.append("\n-- Path: ").append(desc.getUri()); sb.append("\n--\n"); - sb.append("CREATE EXTERNAL TABLE ").append(CatalogUtil.denormalizeIdentifier(desc.getName())); + sb.append("CREATE EXTERNAL TABLE ").append(IdentifierUtil.denormalizeIdentifier(desc.getName())); buildSchema(sb, desc.getSchema()); buildUsingClause(sb, desc.getMeta()); buildWithClause(sb, desc.getMeta()); @@ -59,10 +60,10 @@ public static String buildDDLForBaseTable(TableDesc desc) { StringBuilder sb = new StringBuilder(); sb.append("--\n") - .append("-- Name: ").append(CatalogUtil.denormalizeIdentifier(desc.getName())).append("; Type: TABLE;") + .append("-- Name: ").append(IdentifierUtil.denormalizeIdentifier(desc.getName())).append("; Type: TABLE;") .append(" Storage: ").append(desc.getMeta().getDataFormat()); sb.append("\n--\n"); - sb.append("CREATE TABLE ").append(CatalogUtil.denormalizeIdentifier(desc.getName())); + sb.append("CREATE TABLE ").append(IdentifierUtil.denormalizeIdentifier(desc.getName())); buildSchema(sb, desc.getSchema()); buildUsingClause(sb, desc.getMeta()); buildWithClause(sb, desc.getMeta()); @@ -79,11 +80,11 @@ public static String buildDDLForIndex(IndexDesc desc) { StringBuilder sb = new StringBuilder(); sb.append("--\n") - .append("-- Name: ").append(CatalogUtil.denormalizeIdentifier(desc.getName())).append("; Type: INDEX;") + .append("-- Name: ").append(IdentifierUtil.denormalizeIdentifier(desc.getName())).append("; Type: INDEX;") .append(" Index Method: ").append(desc.getIndexMethod()); sb.append("\n--\n"); - sb.append("CREATE INDEX ").append(CatalogUtil.denormalizeIdentifier(desc.getName())); - sb.append(" on ").append(CatalogUtil.denormalizeIdentifier(desc.getTableName())).append(" ( "); + sb.append("CREATE INDEX ").append(IdentifierUtil.denormalizeIdentifier(desc.getName())); + sb.append(" on ").append(IdentifierUtil.denormalizeIdentifier(desc.getTableName())).append(" ( "); for (SortSpec sortSpec : desc.getKeySortSpecs()) { sb.append(sortSpec.getSortKey().getQualifiedName()).append(" "); @@ -108,7 +109,7 @@ public static void buildSchema(StringBuilder sb, Schema schema) { sb.append(", "); } - sb.append(CatalogUtil.denormalizeIdentifier(column.getSimpleName())).append(" "); + sb.append(IdentifierUtil.denormalizeIdentifier(column.getSimpleName())).append(" "); TypeDesc typeDesc = column.getTypeDesc(); sb.append(typeDesc); } @@ -177,7 +178,7 @@ private static void buildPartitionClause(StringBuilder sb, TableDesc desc) { */ public static String buildDDLForAddPartition(TableDesc table, PartitionDescProto partition) { StringBuilder sb = new StringBuilder(); - sb.append("ALTER TABLE ").append(CatalogUtil.denormalizeIdentifier(table.getName())) + sb.append("ALTER TABLE ").append(IdentifierUtil.denormalizeIdentifier(table.getName())) .append(" ADD IF NOT EXISTS PARTITION ("); diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FieldConverter.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FieldConverter.java index 616b44e5bd..1b34429b78 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FieldConverter.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FieldConverter.java @@ -21,16 +21,12 @@ import com.google.common.base.Function; import com.google.common.collect.Collections2; import com.google.common.collect.ImmutableList; -import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.exception.NotImplementedException; import org.apache.tajo.exception.TajoRuntimeException; +import org.apache.tajo.schema.Field; import org.apache.tajo.schema.Identifier; -import org.apache.tajo.schema.IdentifierPolicy; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.schema.QualifiedIdentifier; -import org.apache.tajo.schema.Schema; -import org.apache.tajo.schema.Schema.NamedPrimitiveType; -import org.apache.tajo.schema.Schema.NamedStructType; -import org.apache.tajo.type.*; import javax.annotation.Nullable; import java.util.Collection; @@ -42,58 +38,20 @@ public static QualifiedIdentifier toQualifiedIdentifier(String name) { final Collection identifiers = Collections2.transform(elems, new Function() { @Override public Identifier apply(@Nullable String input) { - boolean needQuote = CatalogUtil.isShouldBeQuoted(input); - return Identifier._(input, needQuote); + return Identifier._(input, IdentifierUtil.isShouldBeQuoted(input)); } }); return QualifiedIdentifier.$(identifiers); } - public static TypeDesc convert(Schema.NamedType src) { - if (src instanceof NamedStructType) { - NamedStructType structType = (NamedStructType) src; - - ImmutableList.Builder fields = ImmutableList.builder(); - for (Schema.NamedType t: structType.fields()) { - fields.add(new Column(t.name().raw(IdentifierPolicy.DefaultPolicy()), convert(t))); - } - - return new TypeDesc(SchemaBuilder.builder().addAll(fields.build()).build()); - } else { - final NamedPrimitiveType namedType = (NamedPrimitiveType) src; - final Type type = namedType.type(); - if (type instanceof Char) { - Char charType = (Char) type; - return new TypeDesc(CatalogUtil.newDataTypeWithLen(TajoDataTypes.Type.CHAR, charType.length())); - } else if (type instanceof Varchar) { - Varchar varcharType = (Varchar) type; - return new TypeDesc(CatalogUtil.newDataTypeWithLen(TajoDataTypes.Type.VARCHAR, varcharType.length())); - } else if (type instanceof Numeric) { - Numeric numericType = (Numeric) type; - return new TypeDesc(CatalogUtil.newDataTypeWithLen(TajoDataTypes.Type.NUMERIC, numericType.precision())); - } else if (type instanceof Protobuf) { - Protobuf protobuf = (Protobuf) type; - return new TypeDesc(CatalogUtil.newDataType(TajoDataTypes.Type.PROTOBUF, protobuf.getMessageName())); - } else { - return new TypeDesc(TypeConverter.convert(namedType.type())); - } + public static Field convert(Column column) { + if (column.type.isStruct() && column.getTypeDesc().getNestedSchema() == null) { + throw new TajoRuntimeException(new NotImplementedException("record type projection")); } + return new Field(toQualifiedIdentifier(column.getQualifiedName()), column.type); } - public static Schema.NamedType convert(Column column) { - if (column.getTypeDesc().getDataType().getType() == TajoDataTypes.Type.RECORD) { - - if (column.getTypeDesc().getNestedSchema() == null) { - throw new TajoRuntimeException(new NotImplementedException("record type projection")); - } - - return new NamedStructType(toQualifiedIdentifier(column.getQualifiedName()), - TypeConverter.convert(column.getTypeDesc())); - - } else { - return new NamedPrimitiveType(toQualifiedIdentifier(column.getQualifiedName()), - TypeConverter.convert(column.getDataType()) - ); - } + public static Column convert(Field field) { + return new Column(field.name().interned(), field.type()); } } diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/ListSchemaBuilder.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/ListSchemaBuilder.java index e0c9d9e6b8..8f77489145 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/ListSchemaBuilder.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/ListSchemaBuilder.java @@ -20,30 +20,30 @@ import com.google.common.collect.ImmutableCollection; import com.google.common.collect.ImmutableList; -import org.apache.tajo.schema.Schema.NamedType; +import org.apache.tajo.schema.Field; import java.util.Iterator; public class ListSchemaBuilder implements SchemaBuilder.SchemaCollector { - private final ImmutableList.Builder fields = new ImmutableList.Builder(); + private final ImmutableList.Builder fields = new ImmutableList.Builder(); @Override - public void add(NamedType namedType) { - fields.add(namedType); + public void add(Field field) { + fields.add(field); } @Override - public void addAll(Iterator fields) { + public void addAll(Iterator fields) { this.fields.addAll(fields); } @Override - public void addAll(Iterable fields) { + public void addAll(Iterable fields) { this.fields.addAll(fields); } @Override - public ImmutableCollection build() { + public ImmutableCollection build() { return fields.build(); } } diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/NestedPathUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/NestedPathUtil.java index bf874b95b0..afe7e3112d 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/NestedPathUtil.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/NestedPathUtil.java @@ -73,7 +73,7 @@ private static Column lookupColumnInternal(Column currentColumn, String [] paths Column found = null; if (currentColumn.getDataType().getType() == Type.RECORD) { - found = currentColumn.typeDesc.nestedRecordSchema.getColumn(paths[depth]); + found = TypeConverter.convert(currentColumn.type).nestedRecordSchema.getColumn(paths[depth]); } if (found != null) { diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaBuilder.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaBuilder.java index 35e38b53f7..582b5ee300 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaBuilder.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaBuilder.java @@ -22,18 +22,14 @@ import com.google.common.collect.ImmutableCollection; import com.google.common.collect.ImmutableList; import org.apache.tajo.common.TajoDataTypes; +import org.apache.tajo.schema.Field; import org.apache.tajo.schema.QualifiedIdentifier; -import org.apache.tajo.schema.Schema.NamedPrimitiveType; -import org.apache.tajo.schema.Schema.NamedStructType; -import org.apache.tajo.schema.Schema.NamedType; import org.apache.tajo.type.Type; import javax.annotation.Nullable; -import java.util.Collection; import java.util.Iterator; import static org.apache.tajo.catalog.FieldConverter.toQualifiedIdentifier; -import static org.apache.tajo.schema.IdentifierPolicy.DefaultPolicy; /** * Builder for Schema @@ -42,10 +38,10 @@ public class SchemaBuilder { private final SchemaCollector fields; public interface SchemaCollector { - void add(NamedType field); - void addAll(Iterator fields); - void addAll(Iterable fields); - ImmutableCollection build(); + void add(Field field); + void addAll(Iterator fields); + void addAll(Iterable fields); + ImmutableCollection build(); } public static SchemaLegacy empty() { @@ -64,28 +60,19 @@ public static SchemaBuilder uniqueNameBuilder() { this.fields = collector; } - public SchemaBuilder add(NamedType namedType) { - fields.add(namedType); + public SchemaBuilder add(Field field) { + fields.add(field); return this; } public SchemaBuilder add(QualifiedIdentifier id, Type type) { - add(new NamedPrimitiveType(id, type)); - return this; - } - - public SchemaBuilder addStruct(QualifiedIdentifier id, Collection fields) { - add(new NamedStructType(id, fields)); + add(new Field(id, type)); return this; } @Deprecated public SchemaBuilder add(String name, TypeDesc legacyType) { - if (legacyType.getDataType().getType() == TajoDataTypes.Type.RECORD) { - addStruct(toQualifiedIdentifier(name), TypeConverter.convert(legacyType)); - } else { - add(toQualifiedIdentifier(name), TypeConverter.convert(legacyType.getDataType())); - } + add(toQualifiedIdentifier(name), TypeConverter.convert(legacyType)); return this; } @@ -109,9 +96,9 @@ public SchemaBuilder add(Column column) { @Deprecated public SchemaBuilder addAll(Iterable columns) { - return addAll2(columns, new Function() { + return addAll2(columns, new Function() { @Override - public NamedType apply(@Nullable Column input) { + public Field apply(@Nullable Column input) { return FieldConverter.convert(input); } }); @@ -119,9 +106,9 @@ public NamedType apply(@Nullable Column input) { @Deprecated public SchemaBuilder addAll(Column [] columns) { - return addAll2(columns, new Function() { + return addAll2(columns, new Function() { @Override - public NamedType apply(@Nullable Column input) { + public Field apply(@Nullable Column input) { return FieldConverter.convert(input); } }); @@ -152,21 +139,26 @@ public SchemaBuilder addAll(Iterator fields, Function fn) { return this; } - public SchemaBuilder addAll2(T [] fields, Function fn) { + public SchemaBuilder addAll2(Iterable fields) { + this.fields.addAll(fields); + return this; + } + + public SchemaBuilder addAll2(T [] fields, Function fn) { for (T t : fields) { add(fn.apply(t)); } return this; } - public SchemaBuilder addAll2(Iterable fields, Function fn) { + public SchemaBuilder addAll2(Iterable fields, Function fn) { for (T t : fields) { add(fn.apply(t)); } return this; } - public SchemaBuilder addAll2(Iterator fields, Function fn) { + public SchemaBuilder addAll2(Iterator fields, Function fn) { while(fields.hasNext()) { T t = fields.next(); add(fn.apply(t)); @@ -177,8 +169,8 @@ public SchemaBuilder addAll2(Iterator fields, Function fn) @Deprecated public SchemaLegacy build() { ImmutableList.Builder columns = new ImmutableList.Builder(); - for (NamedType namedType : fields.build()) { - columns.add(new Column(namedType.name().raw(DefaultPolicy()), FieldConverter.convert(namedType))); + for (Field field : fields.build()) { + columns.add(new Column(field.name().interned(), field.type())); } return new SchemaLegacy(columns.build()); diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaLegacy.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaLegacy.java index 2a16fa6e83..db1e566d59 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaLegacy.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaLegacy.java @@ -18,23 +18,27 @@ package org.apache.tajo.catalog; +import com.google.common.base.Function; import com.google.common.base.Objects; import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; import com.google.gson.annotations.Expose; import org.apache.tajo.catalog.SchemaUtil.ColumnVisitor; import org.apache.tajo.catalog.json.CatalogGsonHelper; import org.apache.tajo.catalog.proto.CatalogProtos.ColumnProto; import org.apache.tajo.catalog.proto.CatalogProtos.SchemaProto; import org.apache.tajo.common.ProtoObject; -import org.apache.tajo.common.TajoDataTypes.DataType; -import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.exception.DuplicateColumnException; import org.apache.tajo.exception.TajoRuntimeException; import org.apache.tajo.json.GsonObject; import org.apache.tajo.util.StringUtils; +import javax.annotation.Nullable; import java.util.*; +import static com.google.common.collect.Collections2.transform; + +@Deprecated public class SchemaLegacy implements Schema, ProtoObject, Cloneable, GsonObject { @Expose protected List fields = null; @@ -55,47 +59,18 @@ public SchemaLegacy() { public SchemaLegacy(SchemaProto proto) { init(); - List toBeAdded = new ArrayList<>(); - for (int i = 0; i < proto.getFieldsCount(); i++) { - deserializeColumn(toBeAdded, proto.getFieldsList(), i); - } + Collection toBeAdded = transform(proto.getFieldsList(), new Function() { + @Override + public Column apply(@Nullable ColumnProto proto) { + return new Column(proto); + } + }); for (Column c : toBeAdded) { addColumn(c); } } - /** - * This method transforms a list of ColumnProtos into a schema tree. - * It assumes that protos contains a list of ColumnProtos in the depth-first order. - * - * @param tobeAdded - * @param protos - * @param serializedColumnIndex - */ - private static void deserializeColumn(List tobeAdded, List protos, int serializedColumnIndex) { - ColumnProto columnProto = protos.get(serializedColumnIndex); - if (columnProto.getDataType().getType() == Type.RECORD) { - - // Get the number of child fields - int childNum = columnProto.getDataType().getNumNestedFields(); - // where is start index of nested fields? - int childStartIndex = tobeAdded.size() - childNum; - // Extract nested fields - List nestedColumns = new ArrayList<>(tobeAdded.subList(childStartIndex, childStartIndex + childNum)); - - // Remove nested fields from the the current level - for (int i = 0; i < childNum; i++) { - tobeAdded.remove(tobeAdded.size() - 1); - } - - // Add the nested fields to the list as a single record column - tobeAdded.add(new Column(columnProto.getName(), new TypeDesc(new SchemaLegacy(nestedColumns)))); - } else { - tobeAdded.add(new Column(protos.get(serializedColumnIndex))); - } - } - public SchemaLegacy(Schema schema) { new SchemaLegacy(schema.getRootColumns()); } @@ -139,7 +114,7 @@ public void setQualifier(String qualifier) { Column newColumn; for (Column c : columns) { - newColumn = new Column(qualifier + "." + c.getSimpleName(), c.typeDesc); + newColumn = new Column(qualifier + "." + c.getSimpleName(), c.type); addColumn(newColumn); } } @@ -200,7 +175,7 @@ public Column getColumn(String name) { Column columnPath = new Column( column.getQualifiedName() + NestedPathUtil.makePath(paths, 1), - actualColumn.typeDesc); + actualColumn.type); return columnPath; } else { @@ -424,13 +399,13 @@ public boolean containsAny(Collection columns) { return false; } - private SchemaLegacy addColumn(String name, TypeDesc typeDesc) { + private SchemaLegacy addColumn(String name, org.apache.tajo.type.Type type) { String normalized = name; if(fieldsByQualifiedName.containsKey(normalized)) { throw new TajoRuntimeException(new DuplicateColumnException(normalized)); } - Column newCol = new Column(normalized, typeDesc); + Column newCol = new Column(normalized, type); fields.add(newCol); fieldsByQualifiedName.put(newCol.getQualifiedName(), fields.size() - 1); List inputList = new ArrayList<>(); @@ -441,7 +416,7 @@ private SchemaLegacy addColumn(String name, TypeDesc typeDesc) { } private synchronized void addColumn(Column column) { - addColumn(column.getQualifiedName(), column.typeDesc); + addColumn(column.getQualifiedName(), column.type); } @Override @@ -472,34 +447,15 @@ public Object clone() throws CloneNotSupportedException { @Override public SchemaProto getProto() { SchemaProto.Builder builder = SchemaProto.newBuilder(); - SchemaProtoBuilder recursiveBuilder = new SchemaProtoBuilder(builder); - SchemaUtil.visitSchema(this, recursiveBuilder); + builder.addAllFields(Iterables.transform(getRootColumns(), new Function() { + @Override + public ColumnProto apply(@Nullable Column column) { + return column.getProto(); + } + })); return builder.build(); } - private static class SchemaProtoBuilder implements ColumnVisitor { - private SchemaProto.Builder builder; - public SchemaProtoBuilder(SchemaProto.Builder builder) { - this.builder = builder; - } - - @Override - public void visit(int depth, List path, Column column) { - - if (column.getDataType().getType() == Type.RECORD) { - DataType.Builder updatedType = DataType.newBuilder(column.getDataType()); - updatedType.setNumNestedFields(column.typeDesc.nestedRecordSchema.size()); - - ColumnProto.Builder updatedColumn = ColumnProto.newBuilder(column.getProto()); - updatedColumn.setDataType(updatedType); - - builder.addFields(updatedColumn.build()); - } else { - builder.addFields(column.getProto()); - } - } - } - @Override public String toString() { StringBuilder sb = new StringBuilder(); diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java index c0b60a31bc..fc6d3538d9 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java @@ -185,7 +185,7 @@ private static void visitInDepthFirstOrder(int depth, Column column) { if (column.getDataType().getType() == Type.RECORD) { - for (Column nestedColumn : column.typeDesc.nestedRecordSchema.getRootColumns()) { + for (Column nestedColumn : TypeConverter.convert(column.type).nestedRecordSchema.getRootColumns()) { List newPath = new ArrayList<>(path); newPath.add(column.getQualifiedName()); diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SetSchemaBuilder.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SetSchemaBuilder.java index b66aac294e..4a4099ef56 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SetSchemaBuilder.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SetSchemaBuilder.java @@ -20,47 +20,41 @@ import com.google.common.collect.ImmutableCollection; import com.google.common.collect.ImmutableList; -import org.apache.commons.collections.collection.UnmodifiableCollection; -import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.schema.QualifiedIdentifier; -import org.apache.tajo.schema.Schema.NamedPrimitiveType; -import org.apache.tajo.schema.Schema.NamedStructType; -import org.apache.tajo.schema.Schema.NamedType; -import org.apache.tajo.type.Type; +import org.apache.tajo.schema.Field; -import java.util.*; - -import static org.apache.tajo.catalog.FieldConverter.toQualifiedIdentifier; -import static org.apache.tajo.schema.IdentifierPolicy.DefaultPolicy; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Set; public class SetSchemaBuilder implements SchemaBuilder.SchemaCollector { private final Set nameSet = new HashSet<>(); - private final ImmutableList.Builder fields = new ImmutableList.Builder(); + private final ImmutableList.Builder fields = new ImmutableList.Builder(); @Override - public void add(NamedType namedType) { - if (!nameSet.contains(namedType.name())) { - fields.add(namedType); - nameSet.add(namedType.name()); + public void add(Field field) { + if (!nameSet.contains(field.name())) { + fields.add(field); + nameSet.add(field.name()); } } @Override - public void addAll(Iterator fields) { + public void addAll(Iterator fields) { while (fields.hasNext()) { add(fields.next()); } } @Override - public void addAll(Iterable fields) { - for (NamedType n : fields) { + public void addAll(Iterable fields) { + for (Field n : fields) { add(n); } } @Override - public ImmutableCollection build() { + public ImmutableCollection build() { return fields.build(); } } diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TypeConverter.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TypeConverter.java index f4164d4434..387cf006f5 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TypeConverter.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TypeConverter.java @@ -20,66 +20,25 @@ import com.google.common.collect.ImmutableList; import org.apache.tajo.common.TajoDataTypes; -import org.apache.tajo.exception.NotImplementedException; -import org.apache.tajo.exception.TajoRuntimeException; -import org.apache.tajo.exception.UnsupportedException; -import org.apache.tajo.schema.Schema; +import org.apache.tajo.schema.Field; import org.apache.tajo.type.*; -import java.util.Collection; - +import static org.apache.tajo.catalog.CatalogUtil.newDataTypeWithLen; +import static org.apache.tajo.catalog.CatalogUtil.newSimpleDataType; +import static org.apache.tajo.common.TajoDataTypes.Type.*; import static org.apache.tajo.type.Type.*; public class TypeConverter { - public static Collection convert(TypeDesc type) { - ImmutableList.Builder fields = ImmutableList.builder(); - for (Column c : type.getNestedSchema().getRootColumns()) { - fields.add(FieldConverter.convert(c)); - } - return fields.build(); - } - - public static Type convert(TajoDataTypes.Type legacyBaseType) { - switch (legacyBaseType) { - case BOOLEAN: - return Bool; - case INT1: - return Int1; - case INT2: - return Int2; - case INT4: - return Int4; - case INT8: - return Int8; - case FLOAT4: - return Float4; - case FLOAT8: - return Float8; - case DATE: - return Date; - case TIME: - return Time; - case TIMESTAMP: - return Timestamp; - case INTERVAL: - return Interval; - case CHAR: - return Char(1); // default len = 1 - case TEXT: - return Text; - case BLOB: - return Blob; - case INET4: - return Inet4; - case RECORD: - throw new TajoRuntimeException(new NotImplementedException("record projection")); - case NULL_TYPE: - return Null; - case ANY: - return Any; - default: - throw new TajoRuntimeException(new UnsupportedException(legacyBaseType.name())); + public static Type convert(TypeDesc type) { + if (type.getDataType().getType() == TajoDataTypes.Type.RECORD) { + ImmutableList.Builder fields = ImmutableList.builder(); + for (Column c : type.getNestedSchema().getRootColumns()) { + fields.add(FieldConverter.convert(c)); + } + return Record(fields.build()); + } else { + return convert(type.dataType); } } @@ -96,26 +55,58 @@ public static Type convert(TajoDataTypes.DataType legacyType) { case PROTOBUF: return new Protobuf(legacyType.getCode()); default: - return convert(legacyType.getType()); + return TypeFactory.create(legacyType.getType()); } } - public static TajoDataTypes.DataType convert(Type type) { - switch (type.baseType()) { - case CHAR: - Char charType = (Char) type; - return CatalogUtil.newDataTypeWithLen(TajoDataTypes.Type.CHAR, charType.length()); - case VARCHAR: - Varchar varcharType = (Varchar) type; - return CatalogUtil.newDataTypeWithLen(TajoDataTypes.Type.VARCHAR, varcharType.length()); - case PROTOBUF: - Protobuf protobuf = (Protobuf) type; - return CatalogUtil.newDataType(TajoDataTypes.Type.PROTOBUF, protobuf.getMessageName()); - case NUMERIC: - Numeric numericType = (Numeric) type; - return CatalogUtil.newDataTypeWithLen(TajoDataTypes.Type.NUMERIC, numericType.precision()); + public static TypeDesc convert(Field src) { + return convert(src.type()); + } + + public static TypeDesc convert(Type type) { + switch (type.kind()) { + case CHAR: + Char charType = (Char) type; + return new TypeDesc(newDataTypeWithLen(TajoDataTypes.Type.CHAR, charType.length())); + case VARCHAR: + Varchar varcharType = (Varchar) type; + return new TypeDesc(newDataTypeWithLen(TajoDataTypes.Type.VARCHAR, varcharType.length())); + case NUMERIC: + Numeric numericType = (Numeric) type; + return new TypeDesc(newDataTypeWithLen(TajoDataTypes.Type.NUMERIC, numericType.precision())); + case PROTOBUF: + Protobuf protobuf = (Protobuf) type; + return new TypeDesc(CatalogUtil.newDataType(TajoDataTypes.Type.PROTOBUF, protobuf.getMessageName())); + case RECORD: + Record record = (Record) type; + ImmutableList.Builder fields = ImmutableList.builder(); + for (Field t: record.fields()) { + fields.add(new Column(t.name().interned(), convert(t))); + } + return new TypeDesc(SchemaBuilder.builder().addAll(fields.build()).build()); + + case ARRAY: + Array array = (Array) type; + Type elemType = array.elementType(); + switch (elemType.kind()) { + case INT1: + return new TypeDesc(newSimpleDataType(INT1_ARRAY)); + case INT2: + return new TypeDesc(newSimpleDataType(INT2_ARRAY)); + case INT4: + return new TypeDesc(newSimpleDataType(INT4_ARRAY)); + case INT8: + return new TypeDesc(newSimpleDataType(INT8_ARRAY)); + case FLOAT4: + return new TypeDesc(newSimpleDataType(FLOAT4_ARRAY)); + case FLOAT8: + return new TypeDesc(newSimpleDataType(FLOAT8_ARRAY)); default: - return CatalogUtil.newSimpleDataType(type.baseType()); + return new TypeDesc(newSimpleDataType(type.kind())); + } + + default: + return new TypeDesc(newSimpleDataType(type.kind())); } } } diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/CatalogGsonHelper.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/CatalogGsonHelper.java index c145ecdd3c..27c2309043 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/CatalogGsonHelper.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/CatalogGsonHelper.java @@ -48,6 +48,7 @@ private static Map> registerAdapters() { adapters.put(Datum.class, new DatumAdapter()); adapters.put(DataType.class, new DataTypeAdapter()); adapters.put(Schema.class, new SchemaAdapter()); + adapters.put(org.apache.tajo.type.Type.class, new TypeAdapter()); return adapters; } diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/TableStats.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/TableStats.java index c66500d2f3..47acb50f7c 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/TableStats.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/TableStats.java @@ -26,6 +26,7 @@ import com.google.gson.annotations.Expose; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.common.TajoDataTypes; +import org.apache.tajo.common.TajoDataTypes.TypeProto; import org.apache.tajo.json.GsonObject; import org.apache.tajo.catalog.json.CatalogGsonHelper; import org.apache.tajo.catalog.proto.CatalogProtos.TableStatsProto; @@ -35,6 +36,8 @@ import java.util.ArrayList; import java.util.List; +import static org.apache.tajo.common.TajoDataTypes.Type.PROTOBUF; + public class TableStats implements ProtoObject, Cloneable, GsonObject { @Expose private Long numRows = null; // required @Expose private Long numBytes = null; // required @@ -85,13 +88,17 @@ public TableStats(CatalogProtos.TableStatsProto proto) { this.columnStatses = new ArrayList<>(); for (CatalogProtos.ColumnStatsProto colProto : proto.getColStatList()) { - if (colProto.getColumn().getDataType().getType() == TajoDataTypes.Type.PROTOBUF) { + if (peekType(colProto.getColumn().getType()) == PROTOBUF) { continue; } columnStatses.add(new ColumnStats(colProto)); } } + private static TajoDataTypes.Type peekType(TypeProto proto) { + return proto.getElements(proto.getElementsCount() - 1).getKind(); + } + public Long getNumRows() { return this.numRows; } diff --git a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto index d064c62586..e79bc75dbb 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto +++ b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto @@ -66,7 +66,7 @@ enum AlterTableType { message ColumnProto { required string name = 1; optional int32 tid = 2; - required DataType data_type = 3; + required TypeProto type = 3; } message SchemaProto { diff --git a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestCatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestCatalogUtil.java index 0a7fd0addd..c09909a233 100644 --- a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestCatalogUtil.java +++ b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestCatalogUtil.java @@ -20,6 +20,7 @@ import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.function.FunctionUtil; +import org.apache.tajo.schema.IdentifierUtil; import org.junit.Test; import java.util.Arrays; @@ -51,7 +52,7 @@ public final void testGetCanonicalName() { @Test public final void testNormalizeIdentifier() { for (int i = 0; i < sources.length; i++) { - assertEquals(normalized[i], CatalogUtil.normalizeIdentifier(sources[i])); + assertEquals(normalized[i], IdentifierUtil.normalizeIdentifier(sources[i])); } } diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java index 6aa058561e..b855c77733 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java @@ -57,7 +57,9 @@ import org.apache.tajo.exception.*; import org.apache.tajo.plan.expr.AlgebraicUtil; import org.apache.tajo.plan.util.PartitionFilterAlgebraVisitor; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.storage.StorageConstants; +import org.apache.tajo.type.TypeProtobufEncoder; import org.apache.tajo.util.KeyValueSet; import org.apache.thrift.TException; @@ -165,8 +167,8 @@ public final CatalogProtos.TableDescProto getTable(String databaseName, final St } if (!isPartitionKey) { - String fieldName = databaseName + CatalogConstants.IDENTIFIER_DELIMITER + tableName + - CatalogConstants.IDENTIFIER_DELIMITER + eachField.getName(); + String fieldName = databaseName + IdentifierUtil.IDENTIFIER_DELIMITER + tableName + + IdentifierUtil.IDENTIFIER_DELIMITER + eachField.getName(); TajoDataTypes.Type dataType = HiveCatalogUtil.getTajoFieldType(eachField.getType()); schemaBuilder.add(fieldName, dataType); } @@ -246,8 +248,8 @@ public final CatalogProtos.TableDescProto getTable(String databaseName, final St for (int i = 0; i < partitionKeys.size(); i++) { FieldSchema fieldSchema = partitionKeys.get(i); TajoDataTypes.Type dataType = HiveCatalogUtil.getTajoFieldType(fieldSchema.getType()); - String fieldName = databaseName + CatalogConstants.IDENTIFIER_DELIMITER + tableName + - CatalogConstants.IDENTIFIER_DELIMITER + fieldSchema.getName(); + String fieldName = databaseName + IdentifierUtil.IDENTIFIER_DELIMITER + tableName + + IdentifierUtil.IDENTIFIER_DELIMITER + fieldSchema.getName(); expressionSchema.add(new Column(fieldName, dataType)); if (i > 0) { sb.append(","); @@ -421,7 +423,7 @@ public final void createTable(final CatalogProtos.TableDescProto tableDescProto) HiveCatalogStoreClientPool.HiveCatalogStoreClient client = null; TableDesc tableDesc = new TableDesc(tableDescProto); - String[] splitted = CatalogUtil.splitFQTableName(tableDesc.getName()); + String[] splitted = IdentifierUtil.splitFQTableName(tableDesc.getName()); String databaseName = splitted[0]; String tableName = splitted[1]; @@ -460,7 +462,7 @@ public final void createTable(final CatalogProtos.TableDescProto tableDescProto) for (Column eachField : columns) { cols.add(new FieldSchema(eachField.getSimpleName(), - HiveCatalogUtil.getHiveFieldType(eachField.getDataType()), "")); + HiveCatalogUtil.getHiveFieldType(eachField.getType()), "")); } sd.setCols(cols); @@ -469,7 +471,7 @@ public final void createTable(final CatalogProtos.TableDescProto tableDescProto) List partitionKeys = new ArrayList<>(); for (Column eachPartitionKey : tableDesc.getPartitionMethod().getExpressionSchema().getRootColumns()) { partitionKeys.add(new FieldSchema(eachPartitionKey.getSimpleName(), - HiveCatalogUtil.getHiveFieldType(eachPartitionKey.getDataType()), "")); + HiveCatalogUtil.getHiveFieldType(eachPartitionKey.getType()), "")); } table.setPartitionKeys(partitionKeys); } @@ -605,7 +607,7 @@ public void alterTable(final CatalogProtos.AlterTableDescProto alterTableDescPro throws DuplicateTableException, DuplicateColumnException, DuplicatePartitionException, UndefinedPartitionException { - final String[] split = CatalogUtil.splitFQTableName(alterTableDescProto.getTableName()); + final String[] split = IdentifierUtil.splitFQTableName(alterTableDescProto.getTableName()); if (split.length == 1) { throw new IllegalArgumentException("alterTable() requires a qualified table name, but it is \"" @@ -713,7 +715,7 @@ private void addNewColumn(String databaseName, String tableName, CatalogProtos.C Table table = client.getHiveClient().getTable(databaseName, tableName); List columns = table.getSd().getCols(); columns.add(new FieldSchema(columnProto.getName(), - HiveCatalogUtil.getHiveFieldType(columnProto.getDataType()), "")); + HiveCatalogUtil.getHiveFieldType(TypeProtobufEncoder.decode(columnProto.getType())), "")); client.getHiveClient().alter_table(databaseName, tableName, table); @@ -812,8 +814,8 @@ public CatalogProtos.PartitionMethodProto getPartitionMethod(String databaseName for (int i = 0; i < partitionKeys.size(); i++) { FieldSchema fieldSchema = partitionKeys.get(i); TajoDataTypes.Type dataType = HiveCatalogUtil.getTajoFieldType(fieldSchema.getType()); - String fieldName = databaseName + CatalogConstants.IDENTIFIER_DELIMITER + tableName + - CatalogConstants.IDENTIFIER_DELIMITER + fieldSchema.getName(); + String fieldName = databaseName + IdentifierUtil.IDENTIFIER_DELIMITER + tableName + + IdentifierUtil.IDENTIFIER_DELIMITER + fieldSchema.getName(); expressionSchema.add(new Column(fieldName, dataType)); if (i > 0) { sb.append(","); @@ -1041,7 +1043,7 @@ private List getPartitionsFromHiveMetaStore(String databaseN if (i > 0) { partitionName.append(File.separator); } - partitionName.append(CatalogUtil.extractSimpleName(parititonColumns.get(i).getName())); + partitionName.append(IdentifierUtil.extractSimpleName(parititonColumns.get(i).getName())); partitionName.append("="); partitionName.append(hivePartition.getValues().get(i)); } diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogUtil.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogUtil.java index 87b391ea60..faefd2808e 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogUtil.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogUtil.java @@ -39,6 +39,8 @@ import org.apache.tajo.exception.TajoRuntimeException; import org.apache.tajo.exception.UnknownDataFormatException; import org.apache.tajo.exception.UnsupportedException; +import org.apache.tajo.type.Type; +import org.apache.tajo.type.TypeStringEncoder; import org.apache.thrift.TException; public class HiveCatalogUtil { @@ -82,10 +84,10 @@ public static TajoDataTypes.Type getTajoFieldType(String dataType) throws LMDNoM } } - public static String getHiveFieldType(TajoDataTypes.DataType dataType) throws LMDNoMatchedDatatypeException { - Preconditions.checkNotNull(dataType); + public static String getHiveFieldType(Type type) throws LMDNoMatchedDatatypeException { + Preconditions.checkNotNull(type); - switch (dataType.getType()) { + switch (type.kind()) { case CHAR: return serdeConstants.CHAR_TYPE_NAME; case BOOLEAN: return serdeConstants.BOOLEAN_TYPE_NAME; case INT1: return serdeConstants.TINYINT_TYPE_NAME; @@ -104,7 +106,7 @@ public static String getHiveFieldType(TajoDataTypes.DataType dataType) throws LM case DATE: return serdeConstants.DATE_TYPE_NAME; case TIMESTAMP: return serdeConstants.TIMESTAMP_TYPE_NAME; default: - throw new LMDNoMatchedDatatypeException(dataType.getType().name()); + throw new LMDNoMatchedDatatypeException(TypeStringEncoder.encode(type)); } } diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java index fdb18530dc..1260371c77 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java @@ -36,6 +36,7 @@ import org.apache.tajo.catalog.proto.CatalogProtos.PartitionKeyProto; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.storage.StorageConstants; import org.apache.tajo.util.CommonTestingUtil; import org.apache.tajo.util.KeyValueSet; @@ -106,7 +107,7 @@ public void testTableUsingTextFile() throws Exception { .add("c_comment", TajoDataTypes.Type.TEXT) .build(); - TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, CUSTOMER), schema, meta, + TableDesc table = new TableDesc(IdentifierUtil.buildFQName(DB_NAME, CUSTOMER), schema, meta, new Path(warehousePath, new Path(DB_NAME, CUSTOMER)).toUri()); store.createTable(table.getProto()); assertTrue(store.existTable(DB_NAME, CUSTOMER)); @@ -142,7 +143,7 @@ public void testTableUsingRCFileWithBinarySerde() throws Exception { .add("r_comment", TajoDataTypes.Type.TEXT) .build(); - TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, REGION), schema, meta, + TableDesc table = new TableDesc(IdentifierUtil.buildFQName(DB_NAME, REGION), schema, meta, new Path(warehousePath, new Path(DB_NAME, REGION)).toUri()); store.createTable(table.getProto()); assertTrue(store.existTable(DB_NAME, REGION)); @@ -177,7 +178,7 @@ public void testTableUsingRCFileWithTextSerde() throws Exception { .add("r_comment", TajoDataTypes.Type.TEXT) .build(); - TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, REGION), schema, meta, + TableDesc table = new TableDesc(IdentifierUtil.buildFQName(DB_NAME, REGION), schema, meta, new Path(warehousePath, new Path(DB_NAME, REGION)).toUri()); store.createTable(table.getProto()); assertTrue(store.existTable(DB_NAME, REGION)); @@ -216,7 +217,7 @@ public void testTableWithNullValue() throws Exception { .add("s_comment", TajoDataTypes.Type.TEXT) .build(); - TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, SUPPLIER), schema, meta, + TableDesc table = new TableDesc(IdentifierUtil.buildFQName(DB_NAME, SUPPLIER), schema, meta, new Path(warehousePath, new Path(DB_NAME, SUPPLIER)).toUri()); store.createTable(table.getProto()); @@ -263,7 +264,7 @@ public void testAddTableByPartition() throws Exception { .build(); - TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, NATION), schema, meta, + TableDesc table = new TableDesc(IdentifierUtil.buildFQName(DB_NAME, NATION), schema, meta, new Path(warehousePath, new Path(DB_NAME, NATION)).toUri()); org.apache.tajo.catalog.Schema expressionSchema = SchemaBuilder.builder() @@ -515,7 +516,7 @@ public void testGetAllTableNames() throws Exception{ String[] tableNames = new String[]{"table1", "table2", "table3"}; for(String tableName : tableNames){ - TableDesc table = new TableDesc(CatalogUtil.buildFQName("default", tableName), schema, meta, + TableDesc table = new TableDesc(IdentifierUtil.buildFQName("default", tableName), schema, meta, new Path(warehousePath, new Path(DB_NAME, tableName)).toUri()); store.createTable(table.getProto()); } @@ -567,7 +568,7 @@ public void testTableUsingSequenceFileWithBinarySerde() throws Exception { .add("r_comment", TajoDataTypes.Type.TEXT) .build(); - TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, REGION), schema, meta, + TableDesc table = new TableDesc(IdentifierUtil.buildFQName(DB_NAME, REGION), schema, meta, new Path(warehousePath, new Path(DB_NAME, REGION)).toUri()); store.createTable(table.getProto()); assertTrue(store.existTable(DB_NAME, REGION)); @@ -602,7 +603,7 @@ public void testTableUsingSequenceFileWithTextSerde() throws Exception { .add("r_comment", TajoDataTypes.Type.TEXT) .build(); - TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, REGION), schema, meta, + TableDesc table = new TableDesc(IdentifierUtil.buildFQName(DB_NAME, REGION), schema, meta, new Path(warehousePath, new Path(DB_NAME, REGION)).toUri()); store.createTable(table.getProto()); assertTrue(store.existTable(DB_NAME, REGION)); @@ -640,7 +641,7 @@ public void testTableUsingParquet() throws Exception { .add("c_comment", TajoDataTypes.Type.TEXT) .build(); - TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, CUSTOMER), schema, meta, + TableDesc table = new TableDesc(IdentifierUtil.buildFQName(DB_NAME, CUSTOMER), schema, meta, new Path(warehousePath, new Path(DB_NAME, CUSTOMER)).toUri()); store.createTable(table.getProto()); assertTrue(store.existTable(DB_NAME, CUSTOMER)); @@ -663,7 +664,7 @@ public void testTableUsingParquet() throws Exception { @Test public void testDataTypeCompatibility() throws Exception { - String tableName = CatalogUtil.normalizeIdentifier("testDataTypeCompatibility"); + String tableName = IdentifierUtil.normalizeIdentifier("testDataTypeCompatibility"); TableMeta meta = new TableMeta(BuiltinStorages.TEXT, new KeyValueSet()); @@ -681,7 +682,7 @@ public void testDataTypeCompatibility() throws Exception { .add("col11", TajoDataTypes.Type.DATE) .build(); - TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, tableName), schema, meta, + TableDesc table = new TableDesc(IdentifierUtil.buildFQName(DB_NAME, tableName), schema, meta, new Path(warehousePath, new Path(DB_NAME, tableName)).toUri()); store.createTable(table.getProto()); assertTrue(store.existTable(DB_NAME, tableName)); diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java index df40b9b751..6583d4eb01 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java @@ -49,6 +49,7 @@ import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.ReturnState; import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.StringListResponse; import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.StringProto; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.util.NetUtils; import org.apache.tajo.util.Pair; import org.apache.tajo.util.TUtil; @@ -441,7 +442,7 @@ public ReturnState updateTableStats(RpcController controller, UpdateTableStatsPr @Override public ReturnState alterTable(RpcController controller, AlterTableDescProto proto) { - String [] split = CatalogUtil.splitTableName(proto.getTableName()); + String [] split = IdentifierUtil.splitTableName(proto.getTableName()); if (linkedMetadataManager.existsDatabase(split[0])) { return errInsufficientPrivilege("alter a table in database '" + split[0] + "'"); @@ -666,7 +667,7 @@ public GetFunctionsResponse getFunctions(RpcController controller, @Override public ReturnState createTable(RpcController controller, TableDescProto request) { - String [] splitted = CatalogUtil.splitFQTableName(request.getTableName()); + String [] splitted = IdentifierUtil.splitFQTableName(request.getTableName()); String dbName = splitted[0]; String tbName = splitted[1]; @@ -683,7 +684,7 @@ public ReturnState createTable(RpcController controller, TableDescProto request) try { store.createTable(request); LOG.info(String.format("relation \"%s\" is added to the catalog (%s)", - CatalogUtil.getCanonicalTableName(dbName, tbName), bindAddress)); + IdentifierUtil.getCanonicalTableName(dbName, tbName), bindAddress)); return OK; } catch (Throwable t) { @@ -713,7 +714,7 @@ public ReturnState dropTable(RpcController controller, TableIdentifierProto requ try { store.dropTable(dbName, tbName); LOG.info(String.format("relation \"%s\" is deleted from the catalog (%s)", - CatalogUtil.getCanonicalTableName(dbName, tbName), bindAddress)); + IdentifierUtil.getCanonicalTableName(dbName, tbName), bindAddress)); return OK; diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/AbstractTableDescriptor.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/AbstractTableDescriptor.java index cb071aa783..1cf3e64c09 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/AbstractTableDescriptor.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/AbstractTableDescriptor.java @@ -21,6 +21,8 @@ import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.proto.CatalogProtos.*; import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.KeyValueSetProto; +import org.apache.tajo.schema.IdentifierUtil; +import org.apache.tajo.type.TypeFactory; abstract class AbstractTableDescriptor implements TableDescriptor { @@ -40,13 +42,7 @@ protected SchemaProto getSchemaProto() { columnBuilder = ColumnProto.newBuilder(); columnBuilder.setName(columnDescriptor.getName().toLowerCase()); - if (columnDescriptor.getLength() > 0) { - columnBuilder.setDataType(CatalogUtil.newDataTypeWithLen(columnDescriptor.getType(), - columnDescriptor.getLength())); - } else { - columnBuilder.setDataType(CatalogUtil.newSimpleDataType(columnDescriptor.getType())); - } - + columnBuilder.setType(TypeFactory.create(columnDescriptor.getType()).getProto()); schemaBuilder.addFields(columnBuilder.build()); } @@ -71,7 +67,7 @@ protected TableStatsProto getTableStatsProto() { public TableDescProto getTableDescription() { TableDescProto.Builder tableBuilder = TableDescProto.newBuilder(); - tableBuilder.setTableName(CatalogUtil.buildFQName(dictionary.getSystemDatabaseName(), getTableNameString())); + tableBuilder.setTableName(IdentifierUtil.buildFQName(dictionary.getSystemDatabaseName(), getTableNameString())); tableBuilder.setPath(dictionary.getTablePath()); tableBuilder.setSchema(CatalogUtil.getQualfiedSchema( diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java index 88fabe2b8a..2e2db6f9f9 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java @@ -31,12 +31,14 @@ import org.apache.tajo.catalog.*; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.*; -import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.exception.*; import org.apache.tajo.plan.expr.AlgebraicUtil; import org.apache.tajo.plan.util.PartitionFilterAlgebraVisitor; +import org.apache.tajo.schema.IdentifierUtil; +import org.apache.tajo.type.TypeProtobufEncoder; +import org.apache.tajo.type.TypeStringEncoder; import org.apache.tajo.util.JavaResourceUtil; import org.apache.tajo.util.Pair; @@ -49,6 +51,8 @@ import static org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto.AlterTablespaceCommand; import static org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.KeyValueProto; import static org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.KeyValueSetProto; +import static org.apache.tajo.schema.IdentifierUtil.extractQualifier; +import static org.apache.tajo.schema.IdentifierUtil.extractSimpleName; public abstract class AbstractDBStore extends CatalogConstants implements CatalogStore { protected final Log LOG = LogFactory.getLog(getClass()); @@ -756,7 +760,7 @@ public void createTable(final CatalogProtos.TableDescProto table) PreparedStatement pstmt = null; ResultSet res = null; - String[] splitted = CatalogUtil.splitTableName(table.getTableName()); + final String[] splitted = IdentifierUtil.splitTableName(table.getTableName()); if (splitted.length == 1) { throw new TajoInternalError( "createTable() requires a qualified table name, but it is '" + table.getTableName() + "'"); @@ -811,9 +815,9 @@ public void createTable(final CatalogProtos.TableDescProto table) String colSql = "INSERT INTO " + TB_COLUMNS + - // 1 2 3 4 5 6 - " (TID, COLUMN_NAME, ORDINAL_POSITION, NESTED_FIELD_NUM, DATA_TYPE, TYPE_LENGTH)" + - " VALUES(?, ?, ?, ?, ?, ?) "; + // 1 2 3 4 + " (TID, COLUMN_NAME, ORDINAL_POSITION, DATA_TYPE)" + + " VALUES(?, ?, ?, ?) "; if (LOG.isDebugEnabled()) { LOG.debug(colSql); @@ -822,15 +826,12 @@ public void createTable(final CatalogProtos.TableDescProto table) pstmt = conn.prepareStatement(colSql); for (int i = 0; i < table.getSchema().getFieldsCount(); i++) { ColumnProto col = table.getSchema().getFields(i); - TajoDataTypes.DataType dataType = col.getDataType(); + org.apache.tajo.type.Type type = TypeProtobufEncoder.decode(col.getType()); pstmt.setInt(1, tableId); - pstmt.setString(2, CatalogUtil.extractSimpleName(col.getName())); + pstmt.setString(2, extractSimpleName(col.getName())); pstmt.setInt(3, i); - // the default number of nested fields is 0. - pstmt.setInt(4, dataType.hasNumNestedFields() ? dataType.getNumNestedFields() : 0); - pstmt.setString(5, dataType.getType().name()); - pstmt.setInt(6, (col.getDataType().hasLength() ? col.getDataType().getLength() : 0)); + pstmt.setString(4, TypeStringEncoder.encode(type)); pstmt.addBatch(); pstmt.clearParameters(); } @@ -911,7 +912,7 @@ public void updateTableStats(final CatalogProtos.UpdateTableStatsProto statsProt PreparedStatement pstmt = null; ResultSet res = null; - String[] splitted = CatalogUtil.splitTableName(statsProto.getTableName()); + String[] splitted = IdentifierUtil.splitTableName(statsProto.getTableName()); if (splitted.length == 1) { throw new IllegalArgumentException("updateTableStats() requires a qualified table name, but it is \"" + statsProto.getTableName() + "\"."); @@ -978,7 +979,7 @@ public void alterTable(CatalogProtos.AlterTableDescProto alterTableDescProto) DuplicatePartitionException, UndefinedPartitionException, UndefinedColumnException, UndefinedTableException, UndefinedPartitionMethodException, AmbiguousTableException { - String[] splitted = CatalogUtil.splitTableName(alterTableDescProto.getTableName()); + String[] splitted = IdentifierUtil.splitTableName(alterTableDescProto.getTableName()); if (splitted.length == 1) { throw new IllegalArgumentException("alterTable() requires a qualified table name, but it is \"" + alterTableDescProto.getTableName() + "\"."); @@ -993,7 +994,7 @@ public void alterTable(CatalogProtos.AlterTableDescProto alterTableDescProto) switch (alterTableDescProto.getAlterTableType()) { case RENAME_TABLE: - String simpleNewTableName = CatalogUtil.extractSimpleName(alterTableDescProto.getNewTableName()); + String simpleNewTableName = extractSimpleName(alterTableDescProto.getNewTableName()); if (existTable(databaseName, simpleNewTableName)) { throw new DuplicateTableException(alterTableDescProto.getNewTableName()); } @@ -1149,13 +1150,13 @@ private void renameColumn(final int tableId, final CatalogProtos.AlterColumnProt throws UndefinedColumnException, AmbiguousTableException { final String selectColumnSql = - "SELECT COLUMN_NAME, DATA_TYPE, TYPE_LENGTH, ORDINAL_POSITION, NESTED_FIELD_NUM from " + TB_COLUMNS + + "SELECT COLUMN_NAME, DATA_TYPE, ORDINAL_POSITION from " + TB_COLUMNS + " WHERE " + COL_TABLES_PK + " = ?" + " AND COLUMN_NAME = ?" ; final String deleteColumnNameSql = "DELETE FROM " + TB_COLUMNS + " WHERE TID = ? AND COLUMN_NAME = ?"; final String insertNewColumnSql = "INSERT INTO " + TB_COLUMNS + - " (TID, COLUMN_NAME, ORDINAL_POSITION, NESTED_FIELD_NUM, DATA_TYPE, TYPE_LENGTH) VALUES(?, ?, ?, ?, ?, ?) "; + " (TID, COLUMN_NAME, ORDINAL_POSITION, DATA_TYPE) VALUES(?, ?, ?, ?) "; if (LOG.isDebugEnabled()) { LOG.debug(selectColumnSql); @@ -1172,13 +1173,13 @@ private void renameColumn(final int tableId, final CatalogProtos.AlterColumnProt conn = getConnection(); conn.setAutoCommit(false); - String tableName = CatalogUtil.extractQualifier(alterColumnProto.getOldColumnName()); - String simpleOldColumnName = CatalogUtil.extractSimpleName(alterColumnProto.getOldColumnName()); - String simpleNewColumnName = CatalogUtil.extractSimpleName(alterColumnProto.getNewColumnName()); + String tableName = extractQualifier(alterColumnProto.getOldColumnName()); + String simpleOldColumnName = extractSimpleName(alterColumnProto.getOldColumnName()); + String simpleNewColumnName = extractSimpleName(alterColumnProto.getNewColumnName()); - if (!tableName.equals(CatalogUtil.extractQualifier(alterColumnProto.getNewColumnName()))) { + if (!tableName.equals(extractQualifier(alterColumnProto.getNewColumnName()))) { throw new AmbiguousTableException( - tableName + ", " + CatalogUtil.extractQualifier(alterColumnProto.getNewColumnName())); + tableName + ", " + extractQualifier(alterColumnProto.getNewColumnName())); } //SELECT COLUMN @@ -1188,15 +1189,15 @@ private void renameColumn(final int tableId, final CatalogProtos.AlterColumnProt resultSet = pstmt.executeQuery(); CatalogProtos.ColumnProto columnProto = null; + String typeStr; int ordinalPosition = 0; - int nestedFieldNum = 0; if (resultSet.next()) { columnProto = resultToColumnProto(resultSet); //NOTE ==> Setting new column Name columnProto = columnProto.toBuilder().setName(alterColumnProto.getNewColumnName()).build(); ordinalPosition = resultSet.getInt("ORDINAL_POSITION"); - nestedFieldNum = resultSet.getInt("NESTED_FIELD_NUM"); + typeStr = TypeStringEncoder.encode(TypeProtobufEncoder.decode(columnProto.getType())); } else { throw new UndefinedColumnException(alterColumnProto.getOldColumnName()); } @@ -1217,9 +1218,7 @@ private void renameColumn(final int tableId, final CatalogProtos.AlterColumnProt pstmt.setInt(1, tableId); pstmt.setString(2, simpleNewColumnName); pstmt.setInt(3, ordinalPosition); - pstmt.setInt(4, nestedFieldNum); - pstmt.setString(5, columnProto.getDataType().getType().name()); - pstmt.setInt(6, (columnProto.getDataType().hasLength() ? columnProto.getDataType().getLength() : 0)); + pstmt.setString(4, typeStr); pstmt.executeUpdate(); conn.commit(); @@ -1243,7 +1242,7 @@ private void addNewColumn(int tableId, CatalogProtos.ColumnProto columnProto) th final String insertNewColumnSql = "INSERT INTO " + TB_COLUMNS + - " (TID, COLUMN_NAME, ORDINAL_POSITION, NESTED_FIELD_NUM, DATA_TYPE, TYPE_LENGTH) VALUES(?, ?, ?, ?, ?, ?) "; + " (TID, COLUMN_NAME, ORDINAL_POSITION, DATA_TYPE) VALUES(?, ?, ?, ?) "; final String columnCountSql = "SELECT MAX(ORDINAL_POSITION) AS POSITION FROM " + TB_COLUMNS + " WHERE TID = ?"; @@ -1251,7 +1250,7 @@ private void addNewColumn(int tableId, CatalogProtos.ColumnProto columnProto) th conn = getConnection(); pstmt = conn.prepareStatement(existColumnSql); pstmt.setInt(1, tableId); - pstmt.setString(2, CatalogUtil.extractSimpleName(columnProto.getName())); + pstmt.setString(2, extractSimpleName(columnProto.getName())); resultSet = pstmt.executeQuery(); if (resultSet.next()) { @@ -1271,15 +1270,13 @@ private void addNewColumn(int tableId, CatalogProtos.ColumnProto columnProto) th pstmt.close(); resultSet = null; - TajoDataTypes.DataType dataType = columnProto.getDataType(); + org.apache.tajo.type.Type type = TypeProtobufEncoder.decode(columnProto.getType()); pstmt = conn.prepareStatement(insertNewColumnSql); pstmt.setInt(1, tableId); - pstmt.setString(2, CatalogUtil.extractSimpleName(columnProto.getName())); + pstmt.setString(2, extractSimpleName(columnProto.getName())); pstmt.setInt(3, position + 1); - pstmt.setInt(4, dataType.hasNumNestedFields() ? dataType.getNumNestedFields() : 0); - pstmt.setString(5, dataType.getType().name()); - pstmt.setInt(6, (columnProto.getDataType().hasLength() ? columnProto.getDataType().getLength() : 0)); + pstmt.setString(4, TypeStringEncoder.encode(type)); pstmt.executeUpdate(); } catch (SQLException sqlException) { @@ -1600,7 +1597,7 @@ public CatalogProtos.TableDescProto getTable(String databaseName, String tableNa } int tableId = res.getInt(1); - tableBuilder.setTableName(CatalogUtil.buildFQName(databaseName, res.getString(2).trim())); + tableBuilder.setTableName(IdentifierUtil.buildFQName(databaseName, res.getString(2).trim())); TableType tableType = TableType.valueOf(res.getString(3)); if (tableType == TableType.EXTERNAL) { tableBuilder.setIsExternal(true); @@ -1618,7 +1615,7 @@ public CatalogProtos.TableDescProto getTable(String databaseName, String tableNa // Geting Column Descriptions ////////////////////////////////////////// CatalogProtos.SchemaProto.Builder schemaBuilder = CatalogProtos.SchemaProto.newBuilder(); - sql = "SELECT COLUMN_NAME, NESTED_FIELD_NUM, DATA_TYPE, TYPE_LENGTH from " + TB_COLUMNS + + sql = "SELECT COLUMN_NAME, DATA_TYPE from " + TB_COLUMNS + " WHERE " + COL_TABLES_PK + " = ? ORDER BY ORDINAL_POSITION ASC"; if (LOG.isDebugEnabled()) { @@ -1847,7 +1844,7 @@ public List getAllColumns() { try { String sql = - "SELECT TID, COLUMN_NAME, ORDINAL_POSITION, NESTED_FIELD_NUM, DATA_TYPE, TYPE_LENGTH FROM " + TB_COLUMNS + + "SELECT TID, COLUMN_NAME, ORDINAL_POSITION, DATA_TYPE FROM " + TB_COLUMNS + " ORDER BY TID ASC, ORDINAL_POSITION ASC"; conn = getConnection(); @@ -1860,21 +1857,9 @@ public List getAllColumns() { String databaseName = getDatabaseNameOfTable(conn, tid); String tableName = getTableName(conn, tid); builder.setTid(tid); - builder.setName(CatalogUtil.buildFQName(databaseName, tableName, resultSet.getString("COLUMN_NAME"))); - - int nestedFieldNum = resultSet.getInt("NESTED_FIELD_NUM"); - - Type type = getDataType(resultSet.getString("DATA_TYPE").trim()); - int typeLength = resultSet.getInt("TYPE_LENGTH"); - - if (nestedFieldNum > 0) { - builder.setDataType(CatalogUtil.newRecordType(nestedFieldNum)); - } else if (typeLength > 0) { - builder.setDataType(CatalogUtil.newDataTypeWithLen(type, typeLength)); - } else { - builder.setDataType(CatalogUtil.newSimpleDataType(type)); - } - + builder.setName(IdentifierUtil.buildFQName(databaseName, tableName, resultSet.getString("COLUMN_NAME"))); + org.apache.tajo.type.Type type = TypeStringEncoder.decode(resultSet.getString("DATA_TYPE").trim()); + builder.setType(type.getProto()); columns.add(builder.build()); } } catch (SQLException se) { @@ -2484,7 +2469,7 @@ public void createIndex(final IndexDescProto proto) PreparedStatement pstmt = null; final String databaseName = proto.getTableIdentifier().getDatabaseName(); - final String tableName = CatalogUtil.extractSimpleName(proto.getTableIdentifier().getTableName()); + final String tableName = extractSimpleName(proto.getTableIdentifier().getTableName()); try { @@ -2529,7 +2514,7 @@ public void createIndex(final IndexDescProto proto) // Since the key columns are always sorted in order of their occurrence position in the relation schema, // the concatenated name can be uniquely identified. columnNamesBuilder.append(columnSpec.getSortKey().getSimpleName()).append(","); - dataTypesBuilder.append(columnSpec.getSortKey().getDataType().getType().name()).append(","); + dataTypesBuilder.append(columnSpec.getSortKey().getDataType().getType().name()).append("|"); ordersBuilder.append(columnSpec.isAscending()).append(","); nullOrdersBuilder.append(columnSpec.isNullsFirst()).append(","); } @@ -2576,7 +2561,7 @@ public void dropIndex(String databaseName, final String indexName) pstmt.setString(2, indexName); ResultSet res = pstmt.executeQuery(); if (!res.next()) { - throw new UndefinedIndexException(CatalogUtil.buildFQName(databaseName, indexName)); + throw new UndefinedIndexException(IdentifierUtil.buildFQName(databaseName, indexName)); } pstmt.close(); res.close(); @@ -2676,7 +2661,7 @@ public IndexDescProto getIndexByName(String databaseName, final String indexName IndexDescProto.Builder builder = IndexDescProto.newBuilder(); String tableName = getTableName(conn, res.getInt(COL_TABLES_PK)); builder.setTableIdentifier(CatalogUtil.buildTableIdentifier(databaseName, tableName)); - resultToIndexDescProtoBuilder(CatalogUtil.buildFQName(databaseName, tableName), builder, res); + resultToIndexDescProtoBuilder(IdentifierUtil.buildFQName(databaseName, tableName), builder, res); try { builder.setTargetRelationSchema(getTable(databaseName, tableName).getSchema()); @@ -2727,7 +2712,7 @@ public IndexDescProto getIndexByColumns(String databaseName, String tableName, S } IndexDescProto.Builder builder = IndexDescProto.newBuilder(); - resultToIndexDescProtoBuilder(CatalogUtil.buildFQName(databaseName, tableName), builder, res); + resultToIndexDescProtoBuilder(IdentifierUtil.buildFQName(databaseName, tableName), builder, res); builder.setTableIdentifier(CatalogUtil.buildTableIdentifier(databaseName, tableName)); builder.setTargetRelationSchema(tableDescProto.getSchema()); proto = builder.build(); @@ -2897,14 +2882,14 @@ private void resultToIndexDescProtoBuilder(final String qualifier, builder.setIndexPath(res.getString("path")); String[] columnNames, dataTypes, orders, nullOrders; columnNames = res.getString("column_names").trim().split(","); - dataTypes = res.getString("data_types").trim().split(","); + dataTypes = res.getString("data_types").trim().split("\\|"); orders = res.getString("orders").trim().split(","); nullOrders = res.getString("null_orders").trim().split(","); int columnNum = columnNames.length; for (int i = 0; i < columnNum; i++) { SortSpecProto.Builder colSpecBuilder = SortSpecProto.newBuilder(); - colSpecBuilder.setColumn(ColumnProto.newBuilder().setName(CatalogUtil.buildFQName(qualifier, columnNames[i])) - .setDataType(CatalogUtil.newSimpleDataType(getDataType(dataTypes[i]))).build()); + colSpecBuilder.setColumn(ColumnProto.newBuilder().setName(IdentifierUtil.buildFQName(qualifier, columnNames[i])) + .setType(TypeStringEncoder.decode(dataTypes[i]).getProto()).build()); colSpecBuilder.setAscending(orders[i].equals("true")); colSpecBuilder.setNullFirst(nullOrders[i].equals("true")); builder.addKeySortSpecs(colSpecBuilder.build()); @@ -2916,20 +2901,8 @@ private void resultToIndexDescProtoBuilder(final String qualifier, private ColumnProto resultToColumnProto(final ResultSet res) throws SQLException { ColumnProto.Builder builder = ColumnProto.newBuilder(); builder.setName(res.getString("column_name").trim()); - - int nestedFieldNum = res.getInt("NESTED_FIELD_NUM"); - - Type type = getDataType(res.getString("data_type").trim()); - int typeLength = res.getInt("type_length"); - - if (nestedFieldNum > 0) { - builder.setDataType(CatalogUtil.newRecordType(nestedFieldNum)); - } else if (typeLength > 0) { - builder.setDataType(CatalogUtil.newDataTypeWithLen(type, typeLength)); - } else { - builder.setDataType(CatalogUtil.newSimpleDataType(type)); - } - + org.apache.tajo.type.Type type = TypeStringEncoder.decode(res.getString("data_type").trim()); + builder.setType(type.getProto()); return builder.build(); } diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml index 96100e8430..70ef436bec 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml +++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml @@ -19,6 +19,7 @@