From 35720683e12748d35813f5bbd2a1ca3ee5f71ce7 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Thu, 21 Apr 2016 11:49:42 -0700 Subject: [PATCH 01/23] Add array type in sql parser. --- .../org/apache/tajo/algebra/DataTypeExpr.java | 43 ++++++++++++++++++- .../positive/create_table_with_array_type.sql | 6 +++ .../apache/tajo/parser/sql/SQLAnalyzer.java | 33 +++++++------- tajo-sql-parser/pom.xml | 5 +++ .../org/apache/tajo/parser/sql/SQLLexer.g4 | 7 ++- .../org/apache/tajo/parser/sql/SQLParser.g4 | 19 +++++--- 6 files changed, 85 insertions(+), 28 deletions(-) create mode 100644 tajo-core-tests/src/test/resources/queries/TestSQLAnalyzer/positive/create_table_with_array_type.sql diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/DataTypeExpr.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/DataTypeExpr.java index d63532dbb3..0c98640d2a 100644 --- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/DataTypeExpr.java +++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/DataTypeExpr.java @@ -19,6 +19,7 @@ package org.apache.tajo.algebra; import com.google.common.base.Objects; +import com.google.common.base.Preconditions; import com.google.gson.annotations.Expose; import com.google.gson.annotations.SerializedName; import org.apache.tajo.common.TajoDataTypes.Type; @@ -31,6 +32,8 @@ public class DataTypeExpr extends Expr { Integer lengthOrPrecision; @Expose @SerializedName("Scale") Integer scale; + @Expose @SerializedName("Array") + ArrayType arrayType; // not null if the type is RECORD @Expose @SerializedName("Record") RecordType recordType; // not null if the type is RECORD @Expose @SerializedName("Map") @@ -41,6 +44,12 @@ public DataTypeExpr(String typeName) { this.typeName = typeName; } + public DataTypeExpr(ArrayType array) { + super(OpType.DataType); + this.typeName = Type.ARRAY.name(); + this.arrayType = array; + } + public DataTypeExpr(RecordType record) { super(OpType.DataType); this.typeName = Type.RECORD.name(); @@ -62,12 +71,21 @@ public boolean isPrimitiveType() { return !this.isRecordType() && !isMapType(); } + public boolean isArrayType() { + return arrayType != null; + } + public boolean isRecordType() { - return this.typeName.equals(Type.RECORD.name()); + return recordType != null; } public boolean isMapType() { - return this.typeName.equals(Type.MAP.name()); + return mapType != null; + } + + public DataTypeExpr getElementType() { + Preconditions.checkState(isArrayType()); + return arrayType.type; } public ColumnDefinition [] getNestedRecordTypes() { @@ -125,6 +143,27 @@ public Object clone() throws CloneNotSupportedException { return dataType; } + public static class ArrayType implements JsonSerializable, Cloneable { + @Expose + @SerializedName("type") + DataTypeExpr type; + + public ArrayType(DataTypeExpr elementType) { + this.type = elementType; + } + + @Override + public String toJson() { + return JsonHelper.toJson(this); + } + + public Object clone() throws CloneNotSupportedException { + ArrayType newMap = (ArrayType) super.clone(); + newMap.type = type; + return newMap; + } + } + public static class RecordType implements JsonSerializable, Cloneable { @Expose @SerializedName("Schema") ColumnDefinition [] schema; // not null if the type is RECORD diff --git a/tajo-core-tests/src/test/resources/queries/TestSQLAnalyzer/positive/create_table_with_array_type.sql b/tajo-core-tests/src/test/resources/queries/TestSQLAnalyzer/positive/create_table_with_array_type.sql new file mode 100644 index 0000000000..fac25558e5 --- /dev/null +++ b/tajo-core-tests/src/test/resources/queries/TestSQLAnalyzer/positive/create_table_with_array_type.sql @@ -0,0 +1,6 @@ +create table table1 ( + col1 array, + col2 array, + col3 array, + col3 array> +); \ No newline at end of file diff --git a/tajo-core/src/main/java/org/apache/tajo/parser/sql/SQLAnalyzer.java b/tajo-core/src/main/java/org/apache/tajo/parser/sql/SQLAnalyzer.java index f00c85ee6f..5c9a9dbdfd 100644 --- a/tajo-core/src/main/java/org/apache/tajo/parser/sql/SQLAnalyzer.java +++ b/tajo-core/src/main/java/org/apache/tajo/parser/sql/SQLAnalyzer.java @@ -1007,16 +1007,6 @@ public Expr visitIn_predicate_value(In_predicate_valueContext ctx) { } } - @Override - public Expr visitArray(ArrayContext ctx) { - int size = ctx.numeric_value_expression().size(); - Expr[] exprs = new Expr[size]; - for (int i = 0; i < size; i++) { - exprs[i] = visit(ctx.numeric_value_expression(i)); - } - return new ValueListExpr(exprs); - } - @Override public Expr visitPattern_matching_predicate(Pattern_matching_predicateContext ctx) { Expr predicand = visitChildren(ctx.row_value_predicand()); @@ -1626,15 +1616,24 @@ public DataTypeExpr visitData_type(Data_typeContext ctx) { } else if (checkIfExist(predefined_type.network_type())) { typeDefinition = new DataTypeExpr(Type.INET4.name()); + } else if (checkIfExist(predefined_type.complex_type())) { + Complex_typeContext complexType = predefined_type.complex_type(); + + if (checkIfExist(complexType.array_type())) { + DataTypeExpr elementType = visitData_type(complexType.array_type().data_type()); + typeDefinition = new DataTypeExpr(new DataTypeExpr.ArrayType(elementType)); - } else if (checkIfExist(predefined_type.record_type())) { - ColumnDefinition [] nestedRecordDefine = getDefinitions(predefined_type.record_type().table_elements()); - typeDefinition = new DataTypeExpr(new DataTypeExpr.RecordType(nestedRecordDefine)); + } else if (checkIfExist(complexType.record_type())) { + ColumnDefinition[] nestedRecordDefine = getDefinitions(complexType.record_type().table_elements()); + typeDefinition = new DataTypeExpr(new DataTypeExpr.RecordType(nestedRecordDefine)); - } else if (checkIfExist(predefined_type.map_type())) { - Map_typeContext mapTypeContext = predefined_type.map_type(); - typeDefinition = new DataTypeExpr( - new MapType(visitData_type(mapTypeContext.key_type), visitData_type(mapTypeContext.value_type))); + } else if (checkIfExist(complexType.map_type())) { + Map_typeContext mapTypeContext = complexType.map_type(); + typeDefinition = new DataTypeExpr( + new MapType(visitData_type(mapTypeContext.key_type), visitData_type(mapTypeContext.value_type))); + } + } else { + throw new TajoInternalError("Reached code points that shouldn't be reached"); } return typeDefinition; diff --git a/tajo-sql-parser/pom.xml b/tajo-sql-parser/pom.xml index cba8af5934..956e410ad9 100644 --- a/tajo-sql-parser/pom.xml +++ b/tajo-sql-parser/pom.xml @@ -52,6 +52,11 @@ + + + **/*.tokens + + maven-deploy-plugin diff --git a/tajo-sql-parser/src/main/antlr4/org/apache/tajo/parser/sql/SQLLexer.g4 b/tajo-sql-parser/src/main/antlr4/org/apache/tajo/parser/sql/SQLLexer.g4 index 2bb3582d84..db2bfda12f 100644 --- a/tajo-sql-parser/src/main/antlr4/org/apache/tajo/parser/sql/SQLLexer.g4 +++ b/tajo-sql-parser/src/main/antlr4/org/apache/tajo/parser/sql/SQLLexer.g4 @@ -259,7 +259,6 @@ LESS : L E S S; LIST : L I S T; LOCATION : L O C A T I O N; -MAP : M A P; MAX : M A X; MAXVALUE : M A X V A L U E; MICROSECONDS : M I C R O S E C O N D S; @@ -288,7 +287,6 @@ QUARTER : Q U A R T E R; RANGE : R A N G E; RANK : R A N K; -RECORD : R E C O R D; REGEXP : R E G E X P; RENAME : R E N A M E; REPAIR : R E P A I R; @@ -385,6 +383,11 @@ BYTEA : B Y T E A; // alias for BLOB INET4 : I N E T '4'; +// complex types +ARRAY : A R R A Y; +MAP : M A P; +RECORD : R E C O R D; + // Operators Similar_To : '~'; Not_Similar_To : '!~'; diff --git a/tajo-sql-parser/src/main/antlr4/org/apache/tajo/parser/sql/SQLParser.g4 b/tajo-sql-parser/src/main/antlr4/org/apache/tajo/parser/sql/SQLParser.g4 index 190825863a..b76b5d9e89 100644 --- a/tajo-sql-parser/src/main/antlr4/org/apache/tajo/parser/sql/SQLParser.g4 +++ b/tajo-sql-parser/src/main/antlr4/org/apache/tajo/parser/sql/SQLParser.g4 @@ -211,7 +211,7 @@ truncate_table_statement /* =============================================================================== - 11.21 + 11.21 =============================================================================== */ @@ -442,8 +442,7 @@ predefined_type | bit_type | binary_type | network_type - | record_type - | map_type + | complex_type ; character_string_type @@ -539,6 +538,16 @@ network_type : INET4 ; +complex_type + : array_type + | record_type + | map_type + ; + +array_type + : ARRAY LTH data_type GTH + ; + record_type : RECORD table_elements ; @@ -765,10 +774,6 @@ factor : (sign)? numeric_primary ; -array - : LEFT_PAREN numeric_value_expression (COMMA numeric_value_expression )* RIGHT_PAREN - ; - numeric_primary : value_expression_primary (CAST_EXPRESSION cast_target)* | numeric_value_function From 10f38bbfdfbec34f100062e6d5ef1fca0bdf6439 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Thu, 21 Apr 2016 13:54:30 -0700 Subject: [PATCH 02/23] Move some methods in FieldConverter to TypeConverter --- .../apache/tajo/catalog/FieldConverter.java | 47 +++++++------------ .../apache/tajo/catalog/SchemaBuilder.java | 2 +- .../apache/tajo/catalog/TypeConverter.java | 41 +++++++++++++++- 3 files changed, 56 insertions(+), 34 deletions(-) diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FieldConverter.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FieldConverter.java index 616b44e5bd..fc73ca4c42 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FieldConverter.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FieldConverter.java @@ -49,37 +49,6 @@ public Identifier apply(@Nullable String input) { return QualifiedIdentifier.$(identifiers); } - public static TypeDesc convert(Schema.NamedType src) { - if (src instanceof NamedStructType) { - NamedStructType structType = (NamedStructType) src; - - ImmutableList.Builder fields = ImmutableList.builder(); - for (Schema.NamedType t: structType.fields()) { - fields.add(new Column(t.name().raw(IdentifierPolicy.DefaultPolicy()), convert(t))); - } - - return new TypeDesc(SchemaBuilder.builder().addAll(fields.build()).build()); - } else { - final NamedPrimitiveType namedType = (NamedPrimitiveType) src; - final Type type = namedType.type(); - if (type instanceof Char) { - Char charType = (Char) type; - return new TypeDesc(CatalogUtil.newDataTypeWithLen(TajoDataTypes.Type.CHAR, charType.length())); - } else if (type instanceof Varchar) { - Varchar varcharType = (Varchar) type; - return new TypeDesc(CatalogUtil.newDataTypeWithLen(TajoDataTypes.Type.VARCHAR, varcharType.length())); - } else if (type instanceof Numeric) { - Numeric numericType = (Numeric) type; - return new TypeDesc(CatalogUtil.newDataTypeWithLen(TajoDataTypes.Type.NUMERIC, numericType.precision())); - } else if (type instanceof Protobuf) { - Protobuf protobuf = (Protobuf) type; - return new TypeDesc(CatalogUtil.newDataType(TajoDataTypes.Type.PROTOBUF, protobuf.getMessageName())); - } else { - return new TypeDesc(TypeConverter.convert(namedType.type())); - } - } - } - public static Schema.NamedType convert(Column column) { if (column.getTypeDesc().getDataType().getType() == TajoDataTypes.Type.RECORD) { @@ -96,4 +65,20 @@ public static Schema.NamedType convert(Column column) { ); } } + + public static Column convert(Schema.NamedType type) { + if (type instanceof NamedStructType) { + NamedStructType structType = (NamedStructType) type; + Collection converted = Collections2 + .transform(structType.fields(), new Function() { + @Override + public Column apply(@Nullable Schema.NamedType namedType) { + return FieldConverter.convert(namedType); + } + }); + return new Column(type.name().raw(IdentifierPolicy.DefaultPolicy()), new TypeDesc(new SchemaLegacy(converted))); + } else { + return new Column(type.name().displayString(IdentifierPolicy.DefaultPolicy()), TypeConverter.convert(type)); + } + } } diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaBuilder.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaBuilder.java index 35e38b53f7..0f375108a5 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaBuilder.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaBuilder.java @@ -178,7 +178,7 @@ public SchemaBuilder addAll2(Iterator fields, Function fn) public SchemaLegacy build() { ImmutableList.Builder columns = new ImmutableList.Builder(); for (NamedType namedType : fields.build()) { - columns.add(new Column(namedType.name().raw(DefaultPolicy()), FieldConverter.convert(namedType))); + columns.add(new Column(namedType.name().raw(DefaultPolicy()), TypeConverter.convert(namedType))); } return new SchemaLegacy(columns.build()); diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TypeConverter.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TypeConverter.java index f82cc03593..badcf5d3b0 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TypeConverter.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TypeConverter.java @@ -22,9 +22,9 @@ import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.exception.TajoRuntimeException; import org.apache.tajo.exception.UnsupportedException; +import org.apache.tajo.schema.IdentifierPolicy; import org.apache.tajo.schema.Schema; -import org.apache.tajo.type.Protobuf; -import org.apache.tajo.type.Type; +import org.apache.tajo.type.*; import java.util.Collection; @@ -40,6 +40,12 @@ public static Collection convert(TypeDesc type) { return fields.build(); } + /** + * This is for base types. + * + * @param legacyBaseType legacy base type + * @return Type + */ public static Type convert(TajoDataTypes.Type legacyBaseType) { switch (legacyBaseType) { case BOOLEAN: @@ -98,4 +104,35 @@ public static Type convert(TajoDataTypes.DataType legacyType) { public static TajoDataTypes.DataType convert(Type type) { return CatalogUtil.newSimpleDataType(type.baseType()); } + + public static TypeDesc convert(Schema.NamedType src) { + if (src instanceof Schema.NamedStructType) { + Schema.NamedStructType structType = (Schema.NamedStructType) src; + + ImmutableList.Builder fields = ImmutableList.builder(); + for (Schema.NamedType t: structType.fields()) { + fields.add(new Column(t.name().raw(IdentifierPolicy.DefaultPolicy()), convert(t))); + } + + return new TypeDesc(SchemaBuilder.builder().addAll(fields.build()).build()); + } else { + final Schema.NamedPrimitiveType namedType = (Schema.NamedPrimitiveType) src; + final Type type = namedType.type(); + if (type instanceof Char) { + Char charType = (Char) type; + return new TypeDesc(CatalogUtil.newDataTypeWithLen(TajoDataTypes.Type.CHAR, charType.length())); + } else if (type instanceof Varchar) { + Varchar varcharType = (Varchar) type; + return new TypeDesc(CatalogUtil.newDataTypeWithLen(TajoDataTypes.Type.VARCHAR, varcharType.length())); + } else if (type instanceof Numeric) { + Numeric numericType = (Numeric) type; + return new TypeDesc(CatalogUtil.newDataTypeWithLen(TajoDataTypes.Type.NUMERIC, numericType.precision())); + } else if (type instanceof Protobuf) { + Protobuf protobuf = (Protobuf) type; + return new TypeDesc(CatalogUtil.newDataType(TajoDataTypes.Type.PROTOBUF, protobuf.getMessageName())); + } else { + return new TypeDesc(convert(namedType.type())); + } + } + } } From dbcf07aeeff19ab5822cefd809dc0b8eee073e64 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Thu, 21 Apr 2016 13:54:48 -0700 Subject: [PATCH 03/23] Add SchemaConverter. --- .../apache/tajo/catalog/SchemaConverter.java | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaConverter.java diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaConverter.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaConverter.java new file mode 100644 index 0000000000..677dde9af1 --- /dev/null +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaConverter.java @@ -0,0 +1,40 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.catalog; + +import com.google.common.base.Function; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import org.apache.tajo.schema.Schema; + +import javax.annotation.Nullable; + +public class SchemaConverter { + public static org.apache.tajo.catalog.Schema convert(Schema schema) { + ImmutableList columns = ImmutableList.copyOf( + Iterables.transform(schema, new Function() { + @Override + public Column apply(@Nullable Schema.NamedType namedType) { + return FieldConverter.convert(namedType); + } + }) + ); + return new SchemaLegacy(columns); + } +} From 63b17110f26c7cf83f4e91c5a228baa422f5232e Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Thu, 21 Apr 2016 13:55:59 -0700 Subject: [PATCH 04/23] Remove unused imports. --- .../src/main/java/org/apache/tajo/catalog/FieldConverter.java | 1 - 1 file changed, 1 deletion(-) diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FieldConverter.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FieldConverter.java index fc73ca4c42..96721b22d8 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FieldConverter.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FieldConverter.java @@ -30,7 +30,6 @@ import org.apache.tajo.schema.Schema; import org.apache.tajo.schema.Schema.NamedPrimitiveType; import org.apache.tajo.schema.Schema.NamedStructType; -import org.apache.tajo.type.*; import javax.annotation.Nullable; import java.util.Collection; From 5f6b8f3aa544ab7822ad2857e461baef32b2dd76 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Thu, 21 Apr 2016 13:56:33 -0700 Subject: [PATCH 05/23] Refactor Struct Type. --- .../src/main/java/org/apache/tajo/type/Struct.java | 10 ++++++---- .../src/main/java/org/apache/tajo/type/Type.java | 7 +++++-- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Struct.java b/tajo-common/src/main/java/org/apache/tajo/type/Struct.java index 551eb81dc5..4a17ce81e8 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Struct.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Struct.java @@ -20,6 +20,8 @@ import com.google.common.collect.ImmutableList; import org.apache.tajo.common.TajoDataTypes; +import org.apache.tajo.schema.Schema; +import org.apache.tajo.schema.Schema.NamedType; import org.apache.tajo.util.StringUtils; import java.util.Collection; @@ -27,9 +29,9 @@ import java.util.Objects; public class Struct extends Type { - private final ImmutableList memberTypes; + private final ImmutableList memberTypes; - public Struct(Collection memberTypes) { + public Struct(Collection memberTypes) { this.memberTypes = ImmutableList.copyOf(memberTypes); } @@ -37,11 +39,11 @@ public int size() { return memberTypes.size(); } - public Type memberType(int idx) { + public NamedType memberType(int idx) { return memberTypes.get(idx); } - public List memberTypes() { + public List memberTypes() { return this.memberTypes; } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Type.java b/tajo-common/src/main/java/org/apache/tajo/type/Type.java index e1a7180a33..dc8ea8c474 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Type.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Type.java @@ -19,10 +19,13 @@ package org.apache.tajo.type; import org.apache.tajo.common.TajoDataTypes; +import org.apache.tajo.schema.Schema; import java.util.Arrays; import java.util.Collection; +import static org.apache.tajo.common.TajoDataTypes.Type.*; + public abstract class Type { public abstract TajoDataTypes.Type baseType(); @@ -131,11 +134,11 @@ public static Inet4 Inet4() { return new Inet4(); } - public static Struct Struct(Collection types) { + public static Struct Struct(Collection types) { return new Struct(types); } - public static Struct Struct(Type ... types) { + public static Struct Struct(Schema.NamedType... types) { return new Struct(Arrays.asList(types)); } From 06598812afa7b5d9eee3f37e7f2361a5c91a796a Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Thu, 21 Apr 2016 13:57:20 -0700 Subject: [PATCH 06/23] Refactor convertDataType to return Type instead of DataType. --- .../org/apache/tajo/plan/LogicalPlanner.java | 50 ++++++++----------- .../org/apache/tajo/plan/TypeDeterminant.java | 7 +-- 2 files changed, 24 insertions(+), 33 deletions(-) diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java index 9176c2f515..a40e91c666 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java @@ -21,6 +21,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Function; import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import com.google.common.collect.Sets; import org.apache.commons.logging.Log; @@ -49,6 +50,8 @@ import org.apache.tajo.plan.rewrite.rules.ProjectionPushDownRule; import org.apache.tajo.plan.util.ExprFinder; import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.schema.Schema.NamedType; +import org.apache.tajo.type.Type; import org.apache.tajo.util.KeyValueSet; import org.apache.tajo.util.Pair; import org.apache.tajo.util.StringUtils; @@ -58,9 +61,11 @@ import java.util.*; import static org.apache.tajo.algebra.CreateTable.PartitionType; +import static org.apache.tajo.catalog.TypeConverter.convert; import static org.apache.tajo.plan.ExprNormalizer.ExprNormalizedResult; import static org.apache.tajo.plan.LogicalPlan.BlockType; import static org.apache.tajo.plan.verifier.SyntaxErrorUtil.makeSyntaxError; +import static org.apache.tajo.type.Type.*; /** * This class creates a logical plan from a nested tajo algebra expression ({@link org.apache.tajo.algebra}) @@ -2047,13 +2052,12 @@ public Column apply(@Nullable ColumnDefinition input) { * @param elements to be transformed * @return schema transformed from table definition elements */ - private static Schema convertTableElementsSchema(ColumnDefinition[] elements) { - return SchemaBuilder.builder().addAll(elements, new Function() { - @Override - public Column apply(@Nullable ColumnDefinition input) { - return convertColumn(input); - } - }).build(); + private static Collection convertTableElementsSchema(ColumnDefinition[] elements) { + ImmutableList.Builder list = ImmutableList.builder(); + for (ColumnDefinition colDef : elements) { + list.add(FieldConverter.convert(convertColumn(colDef))); + } + return list.build(); } /** @@ -2093,35 +2097,25 @@ private static String[] convertExprsToStrings(Expr[] exprs) { } private static Column convertColumn(ColumnDefinition columnDefinition) { - return new Column(columnDefinition.getColumnName(), convertDataType(columnDefinition)); + return new Column(columnDefinition.getColumnName(), convert(convertDataType(columnDefinition))); } - public static TypeDesc convertDataType(DataTypeExpr dataType) { + public static Type convertDataType(DataTypeExpr dataType) { TajoDataTypes.Type type = TajoDataTypes.Type.valueOf(dataType.getTypeName()); - TajoDataTypes.DataType.Builder builder = TajoDataTypes.DataType.newBuilder(); - builder.setType(type); - - if (dataType.hasLengthOrPrecision()) { - builder.setLength(dataType.getLengthOrPrecision()); + if (type == TajoDataTypes.Type.CHAR) { + return Char(dataType.getLengthOrPrecision()); + } else if (type == TajoDataTypes.Type.VARCHAR) { + return Varchar(dataType.getLengthOrPrecision()); + } else if (type == TajoDataTypes.Type.ARRAY) { + return Array(convertDataType(dataType.getElementType())); + } else if (type == TajoDataTypes.Type.RECORD) { + return Struct(convertTableElementsSchema(dataType.getNestedRecordTypes())); } else { - if (type == TajoDataTypes.Type.CHAR) { - builder.setLength(1); - } + return convert(type); } - - TypeDesc typeDesc; - if (type == TajoDataTypes.Type.RECORD) { - Schema nestedRecordSchema = convertTableElementsSchema(dataType.getNestedRecordTypes()); - typeDesc = new TypeDesc(nestedRecordSchema); - } else { - typeDesc = new TypeDesc(builder.build()); - } - - return typeDesc; } - @Override public LogicalNode visitDropTable(PlanContext context, Stack stack, DropTable dropTable) { DropTableNode dropTableNode = context.queryBlock.getNodeFromExpr(dropTable); diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/TypeDeterminant.java b/tajo-plan/src/main/java/org/apache/tajo/plan/TypeDeterminant.java index 1ff297f77e..af876369e0 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/TypeDeterminant.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/TypeDeterminant.java @@ -21,10 +21,7 @@ import com.google.common.base.Preconditions; import org.apache.tajo.DataTypeUtil; import org.apache.tajo.algebra.*; -import org.apache.tajo.catalog.CatalogService; -import org.apache.tajo.catalog.CatalogUtil; -import org.apache.tajo.catalog.Column; -import org.apache.tajo.catalog.FunctionDesc; +import org.apache.tajo.catalog.*; import org.apache.tajo.exception.UndefinedFunctionException; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.common.TajoDataTypes; @@ -276,7 +273,7 @@ public DataType visitWindowFunction(LogicalPlanner.PlanContext ctx, Stack @Override public DataType visitDataType(LogicalPlanner.PlanContext ctx, Stack stack, DataTypeExpr expr) throws TajoException { - return LogicalPlanner.convertDataType(expr).getDataType(); + return TypeConverter.convert(LogicalPlanner.convertDataType(expr)); } @Override From 30b30c2bf453c18726362f6b5231f2deedc456d1 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Fri, 22 Apr 2016 13:24:20 -0700 Subject: [PATCH 07/23] Extracted Field from Schema to an upper class. --- .../apache/tajo/catalog/FieldConverter.java | 2 +- .../tajo/catalog/ListSchemaBuilder.java | 2 +- .../apache/tajo/catalog/SchemaBuilder.java | 2 +- .../apache/tajo/catalog/SchemaConverter.java | 5 +- .../apache/tajo/catalog/SetSchemaBuilder.java | 2 +- .../apache/tajo/catalog/TypeConverter.java | 8 +- .../java/org/apache/tajo/schema/Field.java | 82 +++++++++++++++++++ .../java/org/apache/tajo/schema/Schema.java | 58 +------------ .../java/org/apache/tajo/type/Struct.java | 2 +- .../main/java/org/apache/tajo/type/Type.java | 6 +- .../java/org/apache/tajo/type/TestType.java | 2 +- .../org/apache/tajo/plan/LogicalPlanner.java | 3 +- 12 files changed, 100 insertions(+), 74 deletions(-) create mode 100644 tajo-common/src/main/java/org/apache/tajo/schema/Field.java diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FieldConverter.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FieldConverter.java index d8f6dc0062..f78400079e 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FieldConverter.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FieldConverter.java @@ -27,7 +27,7 @@ import org.apache.tajo.schema.Identifier; import org.apache.tajo.schema.IdentifierPolicy; import org.apache.tajo.schema.QualifiedIdentifier; -import org.apache.tajo.schema.Schema.Field; +import org.apache.tajo.schema.Field; import org.apache.tajo.type.Struct; import javax.annotation.Nullable; diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/ListSchemaBuilder.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/ListSchemaBuilder.java index 4407dd9db6..8f77489145 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/ListSchemaBuilder.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/ListSchemaBuilder.java @@ -20,7 +20,7 @@ import com.google.common.collect.ImmutableCollection; import com.google.common.collect.ImmutableList; -import org.apache.tajo.schema.Schema.Field; +import org.apache.tajo.schema.Field; import java.util.Iterator; diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaBuilder.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaBuilder.java index fba735f7ed..f5ae499f29 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaBuilder.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaBuilder.java @@ -23,7 +23,7 @@ import com.google.common.collect.ImmutableList; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.schema.QualifiedIdentifier; -import org.apache.tajo.schema.Schema.Field; +import org.apache.tajo.schema.Field; import org.apache.tajo.type.Type; import javax.annotation.Nullable; diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaConverter.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaConverter.java index 3710b6dc73..9fd7c0ea08 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaConverter.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaConverter.java @@ -21,6 +21,7 @@ import com.google.common.base.Function; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; +import org.apache.tajo.schema.Field; import org.apache.tajo.schema.Schema; import javax.annotation.Nullable; @@ -28,9 +29,9 @@ public class SchemaConverter { public static org.apache.tajo.catalog.Schema convert(Schema schema) { ImmutableList columns = ImmutableList.copyOf( - Iterables.transform(schema, new Function() { + Iterables.transform(schema, new Function() { @Override - public Column apply(@Nullable Schema.Field namedType) { + public Column apply(@Nullable Field namedType) { return FieldConverter.convert(namedType); } }) diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SetSchemaBuilder.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SetSchemaBuilder.java index 0b4b92ba7e..4a4099ef56 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SetSchemaBuilder.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SetSchemaBuilder.java @@ -21,7 +21,7 @@ import com.google.common.collect.ImmutableCollection; import com.google.common.collect.ImmutableList; import org.apache.tajo.schema.QualifiedIdentifier; -import org.apache.tajo.schema.Schema.Field; +import org.apache.tajo.schema.Field; import java.util.HashSet; import java.util.Iterator; diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TypeConverter.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TypeConverter.java index 20d90aa7d5..380e49aadd 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TypeConverter.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TypeConverter.java @@ -23,8 +23,8 @@ import org.apache.tajo.exception.NotImplementedException; import org.apache.tajo.exception.TajoRuntimeException; import org.apache.tajo.exception.UnsupportedException; +import org.apache.tajo.schema.Field; import org.apache.tajo.schema.IdentifierPolicy; -import org.apache.tajo.schema.Schema; import org.apache.tajo.type.*; import static org.apache.tajo.catalog.CatalogUtil.newDataTypeWithLen; @@ -34,7 +34,7 @@ public class TypeConverter { public static Type convert(TypeDesc type) { if (type.getDataType().getType() == TajoDataTypes.Type.RECORD) { - ImmutableList.Builder fields = ImmutableList.builder(); + ImmutableList.Builder fields = ImmutableList.builder(); for (Column c : type.getNestedSchema().getRootColumns()) { fields.add(FieldConverter.convert(c)); } @@ -111,7 +111,7 @@ public static Type convert(TajoDataTypes.Type legacyBaseType) { } } - public static TypeDesc convert(Schema.Field src) { + public static TypeDesc convert(Field src) { return convert(src.type()); } @@ -132,7 +132,7 @@ public static TypeDesc convert(Type type) { case RECORD: Struct struct = (Struct) type; ImmutableList.Builder fields = ImmutableList.builder(); - for (Schema.Field t: struct.fields()) { + for (Field t: struct.fields()) { fields.add(new Column(t.name().raw(IdentifierPolicy.DefaultPolicy()), convert(t))); } return new TypeDesc(SchemaBuilder.builder().addAll(fields.build()).build()); diff --git a/tajo-common/src/main/java/org/apache/tajo/schema/Field.java b/tajo-common/src/main/java/org/apache/tajo/schema/Field.java new file mode 100644 index 0000000000..e9adacf845 --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/schema/Field.java @@ -0,0 +1,82 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.schema; + +import org.apache.tajo.common.TajoDataTypes; +import org.apache.tajo.type.Type; + +import java.util.Objects; + +/** + * Represent a field in a schema. + */ +public class Field { + protected final Type type; + protected final QualifiedIdentifier name; + + public Field(Type type, QualifiedIdentifier name) { + this.type = type; + this.name = name; + } + + public QualifiedIdentifier name() { + return this.name; + } + + public TajoDataTypes.Type baseType() { + return this.type.baseType(); + } + + public T type() { + return (T) type; + } + + public boolean isStruct() { + return type.isStruct(); + } + + public boolean isNull() { + return type.isNull(); + } + + @Override + public String toString() { + return name + " " + type; + } + + @Override + public int hashCode() { + return Objects.hash(type, name); + } + + @Override + public boolean equals(Object obj) { + + if (this == obj) { + return true; + } + + if (obj instanceof Field) { + Field other = (Field) obj; + return this.type.equals(other) && this.name.equals(other.name); + } + + return false; + } +} diff --git a/tajo-common/src/main/java/org/apache/tajo/schema/Schema.java b/tajo-common/src/main/java/org/apache/tajo/schema/Schema.java index b0fdcec6f5..54be2bc3be 100644 --- a/tajo-common/src/main/java/org/apache/tajo/schema/Schema.java +++ b/tajo-common/src/main/java/org/apache/tajo/schema/Schema.java @@ -20,19 +20,17 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; -import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.type.Type; import org.apache.tajo.util.StringUtils; import java.util.Arrays; import java.util.Collection; import java.util.Iterator; -import java.util.Objects; /** * A field is a pair of a name and a type. Schema is an ordered list of fields. */ -public class Schema implements Iterable { +public class Schema implements Iterable { private final ImmutableList fields; public Schema(Collection fields) { @@ -74,58 +72,4 @@ public Iterator iterator() { return fields.iterator(); } - public static class Field { - protected final Type type; - protected final QualifiedIdentifier name; - - public Field(Type type, QualifiedIdentifier name) { - this.type = type; - this.name = name; - } - - public QualifiedIdentifier name() { - return this.name; - } - - public TajoDataTypes.Type baseType() { - return this.type.baseType(); - } - - public T type() { - return (T) type; - } - - public boolean isStruct() { - return type.isStruct(); - } - - public boolean isNull() { - return type.isNull(); - } - - @Override - public String toString() { - return name + " " + type; - } - - @Override - public int hashCode() { - return Objects.hash(type, name); - } - - @Override - public boolean equals(Object obj) { - - if (this == obj) { - return true; - } - - if (obj instanceof Field) { - Field other = (Field) obj; - return this.type.equals(other) && this.name.equals(other.name); - } - - return false; - } - } } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Struct.java b/tajo-common/src/main/java/org/apache/tajo/type/Struct.java index 457799ac1e..16544fe1e0 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Struct.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Struct.java @@ -20,7 +20,7 @@ import com.google.common.collect.ImmutableList; import org.apache.tajo.common.TajoDataTypes; -import org.apache.tajo.schema.Schema.Field; +import org.apache.tajo.schema.Field; import org.apache.tajo.util.StringUtils; import java.util.Collection; diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Type.java b/tajo-common/src/main/java/org/apache/tajo/type/Type.java index 9d012936c7..47ee55f175 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Type.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Type.java @@ -19,7 +19,7 @@ package org.apache.tajo.type; import org.apache.tajo.common.TajoDataTypes; -import org.apache.tajo.schema.Schema; +import org.apache.tajo.schema.Field; import java.util.Arrays; import java.util.Collection; @@ -98,7 +98,7 @@ public static Varchar Varchar(int len) { return new Varchar(len); } - public static Struct Struct(Collection types) { + public static Struct Struct(Collection types) { return new Struct(types); } @@ -106,7 +106,7 @@ public static Array Array(Type type) { return new Array(type); } - public static Struct Struct(Schema.Field... types) { + public static Struct Struct(Field... types) { return new Struct(Arrays.asList(types)); } diff --git a/tajo-common/src/test/java/org/apache/tajo/type/TestType.java b/tajo-common/src/test/java/org/apache/tajo/type/TestType.java index 733ee3ef01..8961a958d0 100644 --- a/tajo-common/src/test/java/org/apache/tajo/type/TestType.java +++ b/tajo-common/src/test/java/org/apache/tajo/type/TestType.java @@ -22,7 +22,7 @@ import static org.apache.tajo.common.TajoDataTypes.Type.*; import static org.apache.tajo.schema.QualifiedIdentifier.$; -import static org.apache.tajo.schema.Schema.Field; + import static org.apache.tajo.type.Type.Array; import static org.apache.tajo.type.Type.Blob; import static org.apache.tajo.type.Type.Bool; diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java index 9a3765f7f7..dcabbf5c47 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java @@ -50,8 +50,7 @@ import org.apache.tajo.plan.rewrite.rules.ProjectionPushDownRule; import org.apache.tajo.plan.util.ExprFinder; import org.apache.tajo.plan.util.PlannerUtil; -import org.apache.tajo.schema.Schema.Field; -import org.apache.tajo.type.Numeric; +import org.apache.tajo.schema.Field; import org.apache.tajo.type.Type; import org.apache.tajo.util.KeyValueSet; import org.apache.tajo.util.Pair; From 75e679e842cd7187075647157106e715c32338fd Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Fri, 22 Apr 2016 13:26:55 -0700 Subject: [PATCH 08/23] Add QualifiedIdentifier::raw(). --- .../main/java/org/apache/tajo/catalog/FieldConverter.java | 7 +++---- .../main/java/org/apache/tajo/catalog/SchemaBuilder.java | 2 +- .../main/java/org/apache/tajo/catalog/TypeConverter.java | 2 +- .../java/org/apache/tajo/schema/QualifiedIdentifier.java | 4 ++++ 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FieldConverter.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FieldConverter.java index f78400079e..1f3cc1b799 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FieldConverter.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FieldConverter.java @@ -24,10 +24,9 @@ import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.exception.NotImplementedException; import org.apache.tajo.exception.TajoRuntimeException; +import org.apache.tajo.schema.Field; import org.apache.tajo.schema.Identifier; -import org.apache.tajo.schema.IdentifierPolicy; import org.apache.tajo.schema.QualifiedIdentifier; -import org.apache.tajo.schema.Field; import org.apache.tajo.type.Struct; import javax.annotation.Nullable; @@ -68,9 +67,9 @@ public Column apply(@Nullable Field namedType) { return FieldConverter.convert(namedType); } }); - return new Column(field.name().raw(IdentifierPolicy.DefaultPolicy()), new TypeDesc(new SchemaLegacy(converted))); + return new Column(field.name().raw(), new TypeDesc(new SchemaLegacy(converted))); } else { - return new Column(field.name().displayString(IdentifierPolicy.DefaultPolicy()), TypeConverter.convert(field)); + return new Column(field.name().raw(), TypeConverter.convert(field)); } } } diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaBuilder.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaBuilder.java index f5ae499f29..1beb792fc2 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaBuilder.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaBuilder.java @@ -171,7 +171,7 @@ public SchemaBuilder addAll2(Iterator fields, Function fn) { public SchemaLegacy build() { ImmutableList.Builder columns = new ImmutableList.Builder(); for (Field field : fields.build()) { - columns.add(new Column(field.name().raw(DefaultPolicy()), TypeConverter.convert(field))); + columns.add(new Column(field.name().raw(), TypeConverter.convert(field))); } return new SchemaLegacy(columns.build()); diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TypeConverter.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TypeConverter.java index 380e49aadd..8dd1590b6e 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TypeConverter.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TypeConverter.java @@ -133,7 +133,7 @@ public static TypeDesc convert(Type type) { Struct struct = (Struct) type; ImmutableList.Builder fields = ImmutableList.builder(); for (Field t: struct.fields()) { - fields.add(new Column(t.name().raw(IdentifierPolicy.DefaultPolicy()), convert(t))); + fields.add(new Column(t.name().raw(), convert(t))); } return new TypeDesc(SchemaBuilder.builder().addAll(fields.build()).build()); default: diff --git a/tajo-common/src/main/java/org/apache/tajo/schema/QualifiedIdentifier.java b/tajo-common/src/main/java/org/apache/tajo/schema/QualifiedIdentifier.java index 57317b5f13..75c9b728ef 100644 --- a/tajo-common/src/main/java/org/apache/tajo/schema/QualifiedIdentifier.java +++ b/tajo-common/src/main/java/org/apache/tajo/schema/QualifiedIdentifier.java @@ -45,6 +45,10 @@ public String apply(@Nullable Identifier identifier) { }); } + public String raw() { + return raw(DefaultPolicy()); + } + /** * Raw string of qualified identifier * @param policy Identifier Policy From 89a1e956816381ef606aba1c892fc04fb2c95a61 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Fri, 22 Apr 2016 21:50:02 -0700 Subject: [PATCH 09/23] Refactor Identifier codes * Move some util methods in CatalogUtil to IdentifierUtil. * Rename struct to record. * Fix some bugs related to QualifiedIdentifier and Identifier. --- .../tajo/catalog/AbstractCatalogClient.java | 9 +- .../apache/tajo/catalog/CatalogConstants.java | 4 - .../org/apache/tajo/catalog/CatalogUtil.java | 260 +--------------- .../java/org/apache/tajo/catalog/Column.java | 7 +- .../org/apache/tajo/catalog/DDLBuilder.java | 19 +- .../apache/tajo/catalog/FieldConverter.java | 10 +- .../apache/tajo/catalog/TypeConverter.java | 5 +- .../apache/tajo/catalog/TestCatalogUtil.java | 3 +- .../tajo/catalog/store/HiveCatalogStore.java | 19 +- .../catalog/store/TestHiveCatalogStore.java | 23 +- .../apache/tajo/catalog/CatalogServer.java | 9 +- .../dictionary/AbstractTableDescriptor.java | 3 +- .../tajo/catalog/store/AbstractDBStore.java | 39 +-- .../tajo/catalog/CatalogTestingUtil.java | 23 +- .../tajo/catalog/MiniCatalogServer.java | 3 +- .../org/apache/tajo/catalog/TestCatalog.java | 29 +- .../TestCatalogAgainstCaseSensitivity.java | 35 +-- .../tajo/catalog/TestCatalogExceptions.java | 47 +-- .../org/apache/tajo/cli/tools/TajoDump.java | 9 +- .../cli/tsql/commands/DescTableCommand.java | 4 +- .../tajo/jdbc/TajoResultSetMetaData.java | 6 +- .../apache/tajo/LocalTajoTestingUtility.java | 3 +- .../org/apache/tajo/QueryTestCaseBase.java | 10 +- .../org/apache/tajo/TajoTestingCluster.java | 5 +- .../java/org/apache/tajo/schema/Field.java | 31 +- .../org/apache/tajo/schema/Identifier.java | 2 +- .../apache/tajo/schema/IdentifierUtil.java | 289 ++++++++++++++++++ .../tajo/schema/QualifiedIdentifier.java | 11 +- .../java/org/apache/tajo/schema/Schema.java | 21 +- .../tajo/type/{Struct.java => Record.java} | 10 +- .../main/java/org/apache/tajo/type/Type.java | 10 +- .../org/apache/tajo/schema/TestSchema.java | 17 +- .../java/org/apache/tajo/type/TestType.java | 18 +- .../apache/tajo/cli/tools/TestDDLBuilder.java | 5 +- .../org/apache/tajo/cli/tsql/TestTajoCli.java | 6 +- .../apache/tajo/client/TestTajoClient.java | 28 +- .../apache/tajo/engine/eval/ExprTestBase.java | 11 +- .../tajo/engine/eval/TestEvalTreeUtil.java | 3 +- .../planner/TestEvalNodeToExprConverter.java | 4 +- .../planner/TestJoinOrderAlgorithm.java | 9 +- .../engine/planner/TestLogicalOptimizer.java | 7 +- .../engine/planner/TestLogicalPlanner.java | 47 +-- .../tajo/engine/planner/TestPlannerUtil.java | 7 +- .../physical/TestFullOuterHashJoinExec.java | 9 +- .../physical/TestFullOuterMergeJoinExec.java | 11 +- .../physical/TestLeftOuterHashJoinExec.java | 9 +- .../planner/physical/TestPhysicalPlanner.java | 7 +- .../TestProgressExternalSortExec.java | 3 +- .../physical/TestRightOuterHashJoinExec.java | 7 +- .../physical/TestRightOuterMergeJoinExec.java | 11 +- .../engine/planner/physical/TestSortExec.java | 3 +- .../tajo/engine/query/TestAlterTable.java | 21 +- .../tajo/engine/query/TestCTASQuery.java | 16 +- .../tajo/engine/query/TestCreateDatabase.java | 8 +- .../tajo/engine/query/TestCreateTable.java | 13 +- .../tajo/engine/query/TestInsertQuery.java | 16 +- .../query/TestJoinOnPartitionedTables.java | 10 +- .../tajo/engine/query/TestSimpleQuery.java | 6 +- .../engine/query/TestTablePartitions.java | 47 +-- .../tajo/master/TestExecutionBlockCursor.java | 3 +- .../querymaster/TestTaskStatusUpdate.java | 4 +- .../ws/rs/resources/TestTablesResource.java | 8 +- .../tajo/master/TajoMasterClientService.java | 41 +-- .../tajo/master/exec/CreateTableExecutor.java | 7 +- .../apache/tajo/master/exec/DDLExecutor.java | 35 +-- .../NonForwardQueryResultSystemScanner.java | 3 +- .../tajo/master/exec/QueryExecutor.java | 7 +- .../master/exec/prehook/CreateTableHook.java | 4 +- .../org/apache/tajo/querymaster/Query.java | 13 +- .../tajo/ws/rs/resources/TablesResource.java | 12 +- .../tajo/jdbc/TajoDatabaseMetaData.java | 19 +- .../org/apache/tajo/jdbc/TestResultSet.java | 3 +- .../tajo/jdbc/TestTajoDatabaseMetaData.java | 10 +- .../org/apache/tajo/jdbc/TestTajoJdbc.java | 8 +- .../org/apache/tajo/plan/ExprNormalizer.java | 4 +- .../org/apache/tajo/plan/LogicalPlanner.java | 31 +- .../apache/tajo/plan/expr/EvalTreeUtil.java | 13 +- .../org/apache/tajo/plan/expr/FieldEval.java | 56 ++-- .../tajo/plan/logical/CreateDatabaseNode.java | 4 +- .../apache/tajo/plan/logical/ScanNode.java | 13 +- .../tajo/plan/nameresolver/NameResolver.java | 18 +- .../plan/nameresolver/ResolverByLegacy.java | 4 +- .../plan/rewrite/BaseSchemaBuildPhase.java | 15 +- .../rewrite/SelfDescSchemaBuildPhase.java | 7 +- .../rewrite/rules/FilterPushDownRule.java | 11 +- .../rules/PartitionedTableRewriter.java | 3 +- .../plan/verifier/PreLogicalPlanVerifier.java | 14 +- .../apache/tajo/storage/FileTablespace.java | 3 +- .../jdbc/JdbcMetadataProviderBase.java | 5 +- .../storage/jdbc/SQLExpressionGenerator.java | 9 +- .../tajo/storage/pgsql/PgSQLTablespace.java | 3 +- 91 files changed, 903 insertions(+), 804 deletions(-) create mode 100644 tajo-common/src/main/java/org/apache/tajo/schema/IdentifierUtil.java rename tajo-common/src/main/java/org/apache/tajo/type/{Struct.java => Record.java} (88%) diff --git a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java index b0acbb12a4..2c4d41e72b 100644 --- a/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java +++ b/tajo-catalog/tajo-catalog-client/src/main/java/org/apache/tajo/catalog/AbstractCatalogClient.java @@ -32,6 +32,7 @@ import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.NullProto; import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.ReturnState; import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.StringListResponse; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.util.ProtoUtil; import java.io.Closeable; @@ -281,7 +282,7 @@ public final TableDesc getTableDesc(final String databaseName, final String tabl @Override public TableDesc getTableDesc(String qualifiedName) throws UndefinedTableException { - String[] splitted = CatalogUtil.splitFQTableName(qualifiedName); + String[] splitted = IdentifierUtil.splitFQTableName(qualifiedName); return getTableDesc(splitted[0], splitted[1]); } @@ -596,7 +597,7 @@ public final void createTable(final TableDesc desc) public void dropTable(String tableName) throws UndefinedDatabaseException, UndefinedTableException, InsufficientPrivilegeException { - String[] splitted = CatalogUtil.splitFQTableName(tableName); + String[] splitted = IdentifierUtil.splitFQTableName(tableName); final String databaseName = splitted[0]; final String simpleName = splitted[1]; @@ -617,7 +618,7 @@ public void dropTable(String tableName) @Override public final boolean existsTable(final String databaseName, final String tableName) { - if (CatalogUtil.isFQTableName(tableName)) { + if (IdentifierUtil.isFQTableName(tableName)) { throw new IllegalArgumentException( "tableName cannot be composed of multiple parts, but it is \"" + tableName + "\""); } @@ -640,7 +641,7 @@ public final boolean existsTable(final String databaseName, final String tableNa @Override public final boolean existsTable(final String tableName) { - String[] splitted = CatalogUtil.splitFQTableName(tableName); + String[] splitted = IdentifierUtil.splitFQTableName(tableName); return existsTable(splitted[0], splitted[1]); } diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogConstants.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogConstants.java index f2acf983db..6acbc34e7f 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogConstants.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogConstants.java @@ -19,10 +19,6 @@ package org.apache.tajo.catalog; public class CatalogConstants { - public final static String IDENTIFIER_DELIMITER_REGEXP = "\\."; - public final static String IDENTIFIER_DELIMITER = "."; - public final static String IDENTIFIER_QUOTE_STRING = "\""; - public final static int MAX_IDENTIFIER_LENGTH = 128; // Linux and BSD's max username length is 32. For compatibility with other systems, we should follow it. public final static int MAX_USERNAME_LENGTH = 32; public final static int MAX_STATEMENT_LENGTH = 128 * 1024; diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java index 76990f2792..1034dc2438 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java @@ -18,12 +18,10 @@ package org.apache.tajo.catalog; -import com.google.common.base.Preconditions; import com.google.common.collect.Maps; import org.apache.hadoop.fs.Path; import org.apache.tajo.BuiltinStorages; import org.apache.tajo.DataTypeUtil; -import org.apache.tajo.TajoConstants; import org.apache.tajo.annotation.Nullable; import org.apache.tajo.catalog.partition.PartitionDesc; import org.apache.tajo.catalog.partition.PartitionMethodDesc; @@ -37,6 +35,7 @@ import org.apache.tajo.common.TajoDataTypes.DataType; import org.apache.tajo.exception.InvalidOperationException; import org.apache.tajo.exception.UndefinedOperatorException; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.storage.StorageConstants; import org.apache.tajo.util.KeyValueSet; import org.apache.tajo.util.Pair; @@ -53,233 +52,6 @@ public class CatalogUtil { - /** - * Normalize an identifier. Normalization means a translation from a identifier to be a refined identifier name. - * - * Identifier can be composed of multiple parts as follows: - * 
-   *   database_name.table_name.column_name
-   *
- * - * Each regular identifier part can be composed alphabet ([a-z][A-Z]), number([0-9]), and underscore([_]). - * Also, the first letter must be an alphabet character. - * - * normalizeIdentifier normalizes each part of an identifier. - * - * In detail, for each part, it performs as follows: - *
    - *
  • changing a part without double quotation to be lower case letters
    • - *
  • eliminating double quotation marks from identifier
    • - *
- * - * @param identifier The identifier to be normalized - * @return The normalized identifier - */ - public static String normalizeIdentifier(String identifier) { - if (identifier == null || identifier.equals("")) { - return identifier; - } - String [] splitted = identifier.split(CatalogConstants.IDENTIFIER_DELIMITER_REGEXP); - - StringBuilder sb = new StringBuilder(); - boolean first = true; - for (String part : splitted) { - if (first) { - first = false; - } else { - sb.append(CatalogConstants.IDENTIFIER_DELIMITER); - } - sb.append(normalizeIdentifierPart(part)); - } - return sb.toString(); - } - - public static String normalizeIdentifierPart(String part) { - return isDelimited(part) ? stripQuote(part) : part.toLowerCase(); - } - - /** - * Denormalize an identifier. Denormalize means a translation from a stored identifier - * to be a printable identifier name. - * - * In detail, for each part, it performs as follows: - *
    - *
  • changing a part including upper case character or non-ascii character to be lower case letters
    • - *
  • eliminating double quotation marks from identifier
    • - *
- * - * @param identifier The identifier to be normalized - * @return The denormalized identifier - */ - public static String denormalizeIdentifier(String identifier) { - String [] splitted = identifier.split(CatalogConstants.IDENTIFIER_DELIMITER_REGEXP); - - StringBuilder sb = new StringBuilder(); - boolean first = true; - for (String part : splitted) { - if (first) { - first = false; - } else { - sb.append(CatalogConstants.IDENTIFIER_DELIMITER); - } - sb.append(denormalizePart(part)); - } - return sb.toString(); - } - - public static String denormalizePart(String identifier) { - if (isShouldBeQuoted(identifier)) { - return StringUtils.doubleQuote(identifier); - } else { - return identifier; - } - } - - public static boolean isShouldBeQuoted(String columnName) { - for (char character : columnName.toCharArray()) { - if (Character.isUpperCase(character)) { - return true; - } - - if (!StringUtils.isPartOfAnsiSQLIdentifier(character)) { - return true; - } - - if (RESERVED_KEYWORDS_SET.contains(columnName.toUpperCase())) { - return true; - } - } - - return false; - } - - public static String stripQuote(String str) { - return str.substring(1, str.length() - 1); - } - - public static boolean isDelimited(String identifier) { - boolean openQuote = identifier.charAt(0) == '"'; - boolean closeQuote = identifier.charAt(identifier.length() - 1) == '"'; - - // if at least one quote mark exists, the identifier must be grater than equal to 2 characters, - if (openQuote ^ closeQuote && identifier.length() < 2) { - throw new IllegalArgumentException("Invalid Identifier: " + identifier); - } - - // does not allow the empty identifier (''), - if (openQuote && closeQuote && identifier.length() == 2) { - throw new IllegalArgumentException("zero-length delimited identifier: " + identifier); - } - - // Ensure the quote open and close - return openQuote && closeQuote; - } - - /** - * True if a given name is a simple identifier, meaning is not a dot-chained name. - * - * @param columnOrTableName Column or Table name to be checked - * @return True if a given name is a simple identifier. Otherwise, it will return False. - */ - public static boolean isSimpleIdentifier(String columnOrTableName) { - return columnOrTableName.split(CatalogConstants.IDENTIFIER_DELIMITER_REGEXP).length == 1; - } - - public static boolean isFQColumnName(String tableName) { - return tableName.split(CatalogConstants.IDENTIFIER_DELIMITER_REGEXP).length == 3; - } - - public static boolean isFQTableName(String tableName) { - int lastDelimiterIdx = tableName.lastIndexOf(CatalogConstants.IDENTIFIER_DELIMITER); - return lastDelimiterIdx > -1; - } - - public static String [] splitFQTableName(String qualifiedName) { - String [] splitted = CatalogUtil.splitTableName(qualifiedName); - if (splitted.length == 1) { - throw new IllegalArgumentException("Table name is expected to be qualified, but was \"" - + qualifiedName + "\"."); - } - return splitted; - } - - public static String [] splitTableName(String tableName) { - int lastDelimiterIdx = tableName.lastIndexOf(CatalogConstants.IDENTIFIER_DELIMITER); - if (lastDelimiterIdx > -1) { - return new String [] { - tableName.substring(0, lastDelimiterIdx), - tableName.substring(lastDelimiterIdx + 1, tableName.length()) - }; - } else { - return new String [] {tableName}; - } - } - - public static String buildFQName(String... identifiers) { - boolean first = true; - StringBuilder sb = new StringBuilder(); - for(String id : identifiers) { - if (first) { - first = false; - } else { - sb.append(CatalogConstants.IDENTIFIER_DELIMITER); - } - - sb.append(id); - } - - return sb.toString(); - } - - public static Pair separateQualifierAndName(String name) { - Preconditions.checkArgument(isFQTableName(name), "Must be a qualified name."); - return new Pair<>(extractQualifier(name), extractSimpleName(name)); - } - - /** - * Extract a qualification name from an identifier. - * - * For example, consider a table identifier like 'database1.table1'. - * In this case, this method extracts 'database1'. - * - * @param name The identifier to be extracted - * @return The extracted qualifier - */ - public static String extractQualifier(String name) { - int lastDelimiterIdx = name.lastIndexOf(CatalogConstants.IDENTIFIER_DELIMITER); - if (lastDelimiterIdx > -1) { - return name.substring(0, lastDelimiterIdx); - } else { - return TajoConstants.EMPTY_STRING; - } - } - - /** - * Extract a simple name from an identifier. - * - * For example, consider a table identifier like 'database1.table1'. - * In this case, this method extracts 'table1'. - * - * @param name The identifier to be extracted - * @return The extracted simple name - */ - public static String extractSimpleName(String name) { - int lastDelimiterIdx = name.lastIndexOf(CatalogConstants.IDENTIFIER_DELIMITER); - if (lastDelimiterIdx > -1) { - // plus one means skipping a delimiter. - return name.substring(lastDelimiterIdx + 1, name.length()); - } else { - return name; - } - } - - public static String getCanonicalTableName(String databaseName, String tableName) { - StringBuilder sb = new StringBuilder(databaseName); - sb.append(CatalogConstants.IDENTIFIER_DELIMITER); - sb.append(tableName); - return sb.toString(); - } - public static String getBackwardCompitableDataFormat(String dataFormat) { return getDataFormatAsString(asDataFormat(dataFormat)); @@ -402,7 +174,7 @@ public static DataType newDataTypeWithLen(Type type, int length) { } public static String columnToDDLString(Column column) { - StringBuilder sb = new StringBuilder(denormalizeIdentifier(column.getSimpleName())); + StringBuilder sb = new StringBuilder(IdentifierUtil.denormalizeIdentifier(column.getSimpleName())); sb.append(" ").append(column.getDataType().getType()); if (column.getDataType().hasLength()) { sb.append(" (").append(column.getDataType().getLength()).append(")"); @@ -740,32 +512,8 @@ public static void closeQuietly(Statement stmt, ResultSet res) { } } - public static final Set RESERVED_KEYWORDS_SET = new HashSet<>(); - - static final String [] RESERVED_KEYWORDS = { - "AS", "ALL", "AND", "ANY", "ASYMMETRIC", "ASC", - "BOTH", - "CASE", "CAST", "CREATE", "CROSS", "CURRENT_DATE", "CURRENT_TIME", "CURRENT_TIMESTAMP", - "DESC", "DISTINCT", - "END", "ELSE", "EXCEPT", - "FALSE", "FULL", "FROM", - "GROUP", - "HAVING", - "ILIKE", "IN", "INNER", "INTERSECT", "INTO", "IS", - "JOIN", - "LEADING", "LEFT", "LIKE", "LIMIT", - "NATURAL", "NOT", "NULL", - "ON", "OUTER", "OR", "ORDER", - "RIGHT", - "SELECT", "SOME", "SYMMETRIC", - "TABLE", "THEN", "TRAILING", "TRUE", - "OVER", - "UNION", "UNIQUE", "USING", - "WHEN", "WHERE", "WINDOW", "WITH" - }; - static { - Collections.addAll(RESERVED_KEYWORDS_SET, RESERVED_KEYWORDS); + Collections.addAll(IdentifierUtil.RESERVED_KEYWORDS_SET, IdentifierUtil.RESERVED_KEYWORDS); } public static AlterTableDesc renameColumn(String tableName, String oldColumName, String newColumName, @@ -1013,7 +761,7 @@ public static KeyValueSet newDefaultProperty(String dataFormat) { public static String getUnifiedSimpleColumnName(Schema originalSchema, String[] columnNames) { String[] simpleNames = new String[columnNames.length]; for (int i = 0; i < simpleNames.length; i++) { - String[] identifiers = columnNames[i].split(CatalogConstants.IDENTIFIER_DELIMITER_REGEXP); + String[] identifiers = columnNames[i].split(IdentifierUtil.IDENTIFIER_DELIMITER_REGEXP); simpleNames[i] = identifiers[identifiers.length-1]; } Arrays.sort(simpleNames, new ColumnPosComparator(originalSchema)); diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java index 641b674cc8..a8ab86f02c 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java @@ -26,6 +26,7 @@ import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.common.TajoDataTypes.DataType; import org.apache.tajo.json.GsonObject; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.type.Type; /** @@ -93,7 +94,7 @@ public Column(ColumnProto proto) { * @return True if a column includes a table name. Otherwise, it returns False. */ public boolean hasQualifier() { - return name.split(CatalogConstants.IDENTIFIER_DELIMITER_REGEXP).length > 1; + return name.split(IdentifierUtil.IDENTIFIER_DELIMITER_REGEXP).length > 1; } /** @@ -109,7 +110,7 @@ public String getQualifiedName() { * @return The qualifier */ public String getQualifier() { - return CatalogUtil.extractQualifier(name); + return IdentifierUtil.extractQualifier(name); } /** @@ -117,7 +118,7 @@ public String getQualifier() { * @return The simple name without qualifications */ public String getSimpleName() { - return CatalogUtil.extractSimpleName(name); + return IdentifierUtil.extractSimpleName(name); } /** diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java index e2250eb9ef..387c669b5e 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/DDLBuilder.java @@ -21,6 +21,7 @@ import com.google.common.base.Function; import org.apache.tajo.catalog.partition.PartitionMethodDesc; import org.apache.tajo.catalog.proto.CatalogProtos.PartitionDescProto; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.util.KeyValueSet; import org.apache.tajo.util.StringUtils; @@ -36,11 +37,11 @@ public static String buildDDLForExternalTable(TableDesc desc) { StringBuilder sb = new StringBuilder(); sb.append("--\n") - .append("-- Name: ").append(CatalogUtil.denormalizeIdentifier(desc.getName())).append("; Type: TABLE;") + .append("-- Name: ").append(IdentifierUtil.denormalizeIdentifier(desc.getName())).append("; Type: TABLE;") .append(" Storage: ").append(desc.getMeta().getDataFormat()); sb.append("\n-- Path: ").append(desc.getUri()); sb.append("\n--\n"); - sb.append("CREATE EXTERNAL TABLE ").append(CatalogUtil.denormalizeIdentifier(desc.getName())); + sb.append("CREATE EXTERNAL TABLE ").append(IdentifierUtil.denormalizeIdentifier(desc.getName())); buildSchema(sb, desc.getSchema()); buildUsingClause(sb, desc.getMeta()); buildWithClause(sb, desc.getMeta()); @@ -59,10 +60,10 @@ public static String buildDDLForBaseTable(TableDesc desc) { StringBuilder sb = new StringBuilder(); sb.append("--\n") - .append("-- Name: ").append(CatalogUtil.denormalizeIdentifier(desc.getName())).append("; Type: TABLE;") + .append("-- Name: ").append(IdentifierUtil.denormalizeIdentifier(desc.getName())).append("; Type: TABLE;") .append(" Storage: ").append(desc.getMeta().getDataFormat()); sb.append("\n--\n"); - sb.append("CREATE TABLE ").append(CatalogUtil.denormalizeIdentifier(desc.getName())); + sb.append("CREATE TABLE ").append(IdentifierUtil.denormalizeIdentifier(desc.getName())); buildSchema(sb, desc.getSchema()); buildUsingClause(sb, desc.getMeta()); buildWithClause(sb, desc.getMeta()); @@ -79,11 +80,11 @@ public static String buildDDLForIndex(IndexDesc desc) { StringBuilder sb = new StringBuilder(); sb.append("--\n") - .append("-- Name: ").append(CatalogUtil.denormalizeIdentifier(desc.getName())).append("; Type: INDEX;") + .append("-- Name: ").append(IdentifierUtil.denormalizeIdentifier(desc.getName())).append("; Type: INDEX;") .append(" Index Method: ").append(desc.getIndexMethod()); sb.append("\n--\n"); - sb.append("CREATE INDEX ").append(CatalogUtil.denormalizeIdentifier(desc.getName())); - sb.append(" on ").append(CatalogUtil.denormalizeIdentifier(desc.getTableName())).append(" ( "); + sb.append("CREATE INDEX ").append(IdentifierUtil.denormalizeIdentifier(desc.getName())); + sb.append(" on ").append(IdentifierUtil.denormalizeIdentifier(desc.getTableName())).append(" ( "); for (SortSpec sortSpec : desc.getKeySortSpecs()) { sb.append(sortSpec.getSortKey().getQualifiedName()).append(" "); @@ -108,7 +109,7 @@ public static void buildSchema(StringBuilder sb, Schema schema) { sb.append(", "); } - sb.append(CatalogUtil.denormalizeIdentifier(column.getSimpleName())).append(" "); + sb.append(IdentifierUtil.denormalizeIdentifier(column.getSimpleName())).append(" "); TypeDesc typeDesc = column.getTypeDesc(); sb.append(typeDesc); } @@ -177,7 +178,7 @@ private static void buildPartitionClause(StringBuilder sb, TableDesc desc) { */ public static String buildDDLForAddPartition(TableDesc table, PartitionDescProto partition) { StringBuilder sb = new StringBuilder(); - sb.append("ALTER TABLE ").append(CatalogUtil.denormalizeIdentifier(table.getName())) + sb.append("ALTER TABLE ").append(IdentifierUtil.denormalizeIdentifier(table.getName())) .append(" ADD IF NOT EXISTS PARTITION ("); diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FieldConverter.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FieldConverter.java index 1f3cc1b799..fbff4063c9 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FieldConverter.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FieldConverter.java @@ -26,8 +26,9 @@ import org.apache.tajo.exception.TajoRuntimeException; import org.apache.tajo.schema.Field; import org.apache.tajo.schema.Identifier; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.schema.QualifiedIdentifier; -import org.apache.tajo.type.Struct; +import org.apache.tajo.type.Record; import javax.annotation.Nullable; import java.util.Collection; @@ -39,8 +40,7 @@ public static QualifiedIdentifier toQualifiedIdentifier(String name) { final Collection identifiers = Collections2.transform(elems, new Function() { @Override public Identifier apply(@Nullable String input) { - boolean needQuote = CatalogUtil.isShouldBeQuoted(input); - return Identifier._(input, needQuote); + return Identifier._(input, IdentifierUtil.isShouldBeQuoted(input)); } }); return QualifiedIdentifier.$(identifiers); @@ -59,9 +59,9 @@ public static Field convert(Column column) { public static Column convert(Field field) { if (field.isStruct()) { - Struct struct = field.type(); + Record record = field.type(); Collection converted = Collections2 - .transform(struct.fields(), new Function() { + .transform(record.fields(), new Function() { @Override public Column apply(@Nullable Field namedType) { return FieldConverter.convert(namedType); diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TypeConverter.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TypeConverter.java index 8dd1590b6e..64489251c5 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TypeConverter.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TypeConverter.java @@ -24,7 +24,6 @@ import org.apache.tajo.exception.TajoRuntimeException; import org.apache.tajo.exception.UnsupportedException; import org.apache.tajo.schema.Field; -import org.apache.tajo.schema.IdentifierPolicy; import org.apache.tajo.type.*; import static org.apache.tajo.catalog.CatalogUtil.newDataTypeWithLen; @@ -130,9 +129,9 @@ public static TypeDesc convert(Type type) { Protobuf protobuf = (Protobuf) type; return new TypeDesc(CatalogUtil.newDataType(TajoDataTypes.Type.PROTOBUF, protobuf.getMessageName())); case RECORD: - Struct struct = (Struct) type; + Record record = (Record) type; ImmutableList.Builder fields = ImmutableList.builder(); - for (Field t: struct.fields()) { + for (Field t: record.fields()) { fields.add(new Column(t.name().raw(), convert(t))); } return new TypeDesc(SchemaBuilder.builder().addAll(fields.build()).build()); diff --git a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestCatalogUtil.java b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestCatalogUtil.java index 0a7fd0addd..c09909a233 100644 --- a/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestCatalogUtil.java +++ b/tajo-catalog/tajo-catalog-common/src/test/java/org/apache/tajo/catalog/TestCatalogUtil.java @@ -20,6 +20,7 @@ import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.function.FunctionUtil; +import org.apache.tajo.schema.IdentifierUtil; import org.junit.Test; import java.util.Arrays; @@ -51,7 +52,7 @@ public final void testGetCanonicalName() { @Test public final void testNormalizeIdentifier() { for (int i = 0; i < sources.length; i++) { - assertEquals(normalized[i], CatalogUtil.normalizeIdentifier(sources[i])); + assertEquals(normalized[i], IdentifierUtil.normalizeIdentifier(sources[i])); } } diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java index 6aa058561e..698ebe9cf3 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java @@ -57,6 +57,7 @@ import org.apache.tajo.exception.*; import org.apache.tajo.plan.expr.AlgebraicUtil; import org.apache.tajo.plan.util.PartitionFilterAlgebraVisitor; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.storage.StorageConstants; import org.apache.tajo.util.KeyValueSet; import org.apache.thrift.TException; @@ -165,8 +166,8 @@ public final CatalogProtos.TableDescProto getTable(String databaseName, final St } if (!isPartitionKey) { - String fieldName = databaseName + CatalogConstants.IDENTIFIER_DELIMITER + tableName + - CatalogConstants.IDENTIFIER_DELIMITER + eachField.getName(); + String fieldName = databaseName + IdentifierUtil.IDENTIFIER_DELIMITER + tableName + + IdentifierUtil.IDENTIFIER_DELIMITER + eachField.getName(); TajoDataTypes.Type dataType = HiveCatalogUtil.getTajoFieldType(eachField.getType()); schemaBuilder.add(fieldName, dataType); } @@ -246,8 +247,8 @@ public final CatalogProtos.TableDescProto getTable(String databaseName, final St for (int i = 0; i < partitionKeys.size(); i++) { FieldSchema fieldSchema = partitionKeys.get(i); TajoDataTypes.Type dataType = HiveCatalogUtil.getTajoFieldType(fieldSchema.getType()); - String fieldName = databaseName + CatalogConstants.IDENTIFIER_DELIMITER + tableName + - CatalogConstants.IDENTIFIER_DELIMITER + fieldSchema.getName(); + String fieldName = databaseName + IdentifierUtil.IDENTIFIER_DELIMITER + tableName + + IdentifierUtil.IDENTIFIER_DELIMITER + fieldSchema.getName(); expressionSchema.add(new Column(fieldName, dataType)); if (i > 0) { sb.append(","); @@ -421,7 +422,7 @@ public final void createTable(final CatalogProtos.TableDescProto tableDescProto) HiveCatalogStoreClientPool.HiveCatalogStoreClient client = null; TableDesc tableDesc = new TableDesc(tableDescProto); - String[] splitted = CatalogUtil.splitFQTableName(tableDesc.getName()); + String[] splitted = IdentifierUtil.splitFQTableName(tableDesc.getName()); String databaseName = splitted[0]; String tableName = splitted[1]; @@ -605,7 +606,7 @@ public void alterTable(final CatalogProtos.AlterTableDescProto alterTableDescPro throws DuplicateTableException, DuplicateColumnException, DuplicatePartitionException, UndefinedPartitionException { - final String[] split = CatalogUtil.splitFQTableName(alterTableDescProto.getTableName()); + final String[] split = IdentifierUtil.splitFQTableName(alterTableDescProto.getTableName()); if (split.length == 1) { throw new IllegalArgumentException("alterTable() requires a qualified table name, but it is \"" @@ -812,8 +813,8 @@ public CatalogProtos.PartitionMethodProto getPartitionMethod(String databaseName for (int i = 0; i < partitionKeys.size(); i++) { FieldSchema fieldSchema = partitionKeys.get(i); TajoDataTypes.Type dataType = HiveCatalogUtil.getTajoFieldType(fieldSchema.getType()); - String fieldName = databaseName + CatalogConstants.IDENTIFIER_DELIMITER + tableName + - CatalogConstants.IDENTIFIER_DELIMITER + fieldSchema.getName(); + String fieldName = databaseName + IdentifierUtil.IDENTIFIER_DELIMITER + tableName + + IdentifierUtil.IDENTIFIER_DELIMITER + fieldSchema.getName(); expressionSchema.add(new Column(fieldName, dataType)); if (i > 0) { sb.append(","); @@ -1041,7 +1042,7 @@ private List getPartitionsFromHiveMetaStore(String databaseN if (i > 0) { partitionName.append(File.separator); } - partitionName.append(CatalogUtil.extractSimpleName(parititonColumns.get(i).getName())); + partitionName.append(IdentifierUtil.extractSimpleName(parititonColumns.get(i).getName())); partitionName.append("="); partitionName.append(hivePartition.getValues().get(i)); } diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java index fdb18530dc..1260371c77 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java @@ -36,6 +36,7 @@ import org.apache.tajo.catalog.proto.CatalogProtos.PartitionKeyProto; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.storage.StorageConstants; import org.apache.tajo.util.CommonTestingUtil; import org.apache.tajo.util.KeyValueSet; @@ -106,7 +107,7 @@ public void testTableUsingTextFile() throws Exception { .add("c_comment", TajoDataTypes.Type.TEXT) .build(); - TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, CUSTOMER), schema, meta, + TableDesc table = new TableDesc(IdentifierUtil.buildFQName(DB_NAME, CUSTOMER), schema, meta, new Path(warehousePath, new Path(DB_NAME, CUSTOMER)).toUri()); store.createTable(table.getProto()); assertTrue(store.existTable(DB_NAME, CUSTOMER)); @@ -142,7 +143,7 @@ public void testTableUsingRCFileWithBinarySerde() throws Exception { .add("r_comment", TajoDataTypes.Type.TEXT) .build(); - TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, REGION), schema, meta, + TableDesc table = new TableDesc(IdentifierUtil.buildFQName(DB_NAME, REGION), schema, meta, new Path(warehousePath, new Path(DB_NAME, REGION)).toUri()); store.createTable(table.getProto()); assertTrue(store.existTable(DB_NAME, REGION)); @@ -177,7 +178,7 @@ public void testTableUsingRCFileWithTextSerde() throws Exception { .add("r_comment", TajoDataTypes.Type.TEXT) .build(); - TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, REGION), schema, meta, + TableDesc table = new TableDesc(IdentifierUtil.buildFQName(DB_NAME, REGION), schema, meta, new Path(warehousePath, new Path(DB_NAME, REGION)).toUri()); store.createTable(table.getProto()); assertTrue(store.existTable(DB_NAME, REGION)); @@ -216,7 +217,7 @@ public void testTableWithNullValue() throws Exception { .add("s_comment", TajoDataTypes.Type.TEXT) .build(); - TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, SUPPLIER), schema, meta, + TableDesc table = new TableDesc(IdentifierUtil.buildFQName(DB_NAME, SUPPLIER), schema, meta, new Path(warehousePath, new Path(DB_NAME, SUPPLIER)).toUri()); store.createTable(table.getProto()); @@ -263,7 +264,7 @@ public void testAddTableByPartition() throws Exception { .build(); - TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, NATION), schema, meta, + TableDesc table = new TableDesc(IdentifierUtil.buildFQName(DB_NAME, NATION), schema, meta, new Path(warehousePath, new Path(DB_NAME, NATION)).toUri()); org.apache.tajo.catalog.Schema expressionSchema = SchemaBuilder.builder() @@ -515,7 +516,7 @@ public void testGetAllTableNames() throws Exception{ String[] tableNames = new String[]{"table1", "table2", "table3"}; for(String tableName : tableNames){ - TableDesc table = new TableDesc(CatalogUtil.buildFQName("default", tableName), schema, meta, + TableDesc table = new TableDesc(IdentifierUtil.buildFQName("default", tableName), schema, meta, new Path(warehousePath, new Path(DB_NAME, tableName)).toUri()); store.createTable(table.getProto()); } @@ -567,7 +568,7 @@ public void testTableUsingSequenceFileWithBinarySerde() throws Exception { .add("r_comment", TajoDataTypes.Type.TEXT) .build(); - TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, REGION), schema, meta, + TableDesc table = new TableDesc(IdentifierUtil.buildFQName(DB_NAME, REGION), schema, meta, new Path(warehousePath, new Path(DB_NAME, REGION)).toUri()); store.createTable(table.getProto()); assertTrue(store.existTable(DB_NAME, REGION)); @@ -602,7 +603,7 @@ public void testTableUsingSequenceFileWithTextSerde() throws Exception { .add("r_comment", TajoDataTypes.Type.TEXT) .build(); - TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, REGION), schema, meta, + TableDesc table = new TableDesc(IdentifierUtil.buildFQName(DB_NAME, REGION), schema, meta, new Path(warehousePath, new Path(DB_NAME, REGION)).toUri()); store.createTable(table.getProto()); assertTrue(store.existTable(DB_NAME, REGION)); @@ -640,7 +641,7 @@ public void testTableUsingParquet() throws Exception { .add("c_comment", TajoDataTypes.Type.TEXT) .build(); - TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, CUSTOMER), schema, meta, + TableDesc table = new TableDesc(IdentifierUtil.buildFQName(DB_NAME, CUSTOMER), schema, meta, new Path(warehousePath, new Path(DB_NAME, CUSTOMER)).toUri()); store.createTable(table.getProto()); assertTrue(store.existTable(DB_NAME, CUSTOMER)); @@ -663,7 +664,7 @@ public void testTableUsingParquet() throws Exception { @Test public void testDataTypeCompatibility() throws Exception { - String tableName = CatalogUtil.normalizeIdentifier("testDataTypeCompatibility"); + String tableName = IdentifierUtil.normalizeIdentifier("testDataTypeCompatibility"); TableMeta meta = new TableMeta(BuiltinStorages.TEXT, new KeyValueSet()); @@ -681,7 +682,7 @@ public void testDataTypeCompatibility() throws Exception { .add("col11", TajoDataTypes.Type.DATE) .build(); - TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, tableName), schema, meta, + TableDesc table = new TableDesc(IdentifierUtil.buildFQName(DB_NAME, tableName), schema, meta, new Path(warehousePath, new Path(DB_NAME, tableName)).toUri()); store.createTable(table.getProto()); assertTrue(store.existTable(DB_NAME, tableName)); diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java index d99daeeb88..e2d203026c 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/CatalogServer.java @@ -49,6 +49,7 @@ import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.ReturnState; import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.StringListResponse; import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.StringProto; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.util.NetUtils; import org.apache.tajo.util.Pair; import org.apache.tajo.util.TUtil; @@ -444,7 +445,7 @@ public ReturnState updateTableStats(RpcController controller, UpdateTableStatsPr @Override public ReturnState alterTable(RpcController controller, AlterTableDescProto proto) { - String [] split = CatalogUtil.splitTableName(proto.getTableName()); + String [] split = IdentifierUtil.splitTableName(proto.getTableName()); if (linkedMetadataManager.existsDatabase(split[0])) { return errInsufficientPrivilege("alter a table in database '" + split[0] + "'"); @@ -669,7 +670,7 @@ public GetFunctionsResponse getFunctions(RpcController controller, @Override public ReturnState createTable(RpcController controller, TableDescProto request) { - String [] splitted = CatalogUtil.splitFQTableName(request.getTableName()); + String [] splitted = IdentifierUtil.splitFQTableName(request.getTableName()); String dbName = splitted[0]; String tbName = splitted[1]; @@ -686,7 +687,7 @@ public ReturnState createTable(RpcController controller, TableDescProto request) try { store.createTable(request); LOG.info(String.format("relation \"%s\" is added to the catalog (%s)", - CatalogUtil.getCanonicalTableName(dbName, tbName), bindAddressStr)); + IdentifierUtil.getCanonicalTableName(dbName, tbName), bindAddressStr)); return OK; } catch (Throwable t) { @@ -716,7 +717,7 @@ public ReturnState dropTable(RpcController controller, TableIdentifierProto requ try { store.dropTable(dbName, tbName); LOG.info(String.format("relation \"%s\" is deleted from the catalog (%s)", - CatalogUtil.getCanonicalTableName(dbName, tbName), bindAddressStr)); + IdentifierUtil.getCanonicalTableName(dbName, tbName), bindAddressStr)); return OK; diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/AbstractTableDescriptor.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/AbstractTableDescriptor.java index cb071aa783..473e3c69b2 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/AbstractTableDescriptor.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/AbstractTableDescriptor.java @@ -21,6 +21,7 @@ import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.proto.CatalogProtos.*; import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.KeyValueSetProto; +import org.apache.tajo.schema.IdentifierUtil; abstract class AbstractTableDescriptor implements TableDescriptor { @@ -71,7 +72,7 @@ protected TableStatsProto getTableStatsProto() { public TableDescProto getTableDescription() { TableDescProto.Builder tableBuilder = TableDescProto.newBuilder(); - tableBuilder.setTableName(CatalogUtil.buildFQName(dictionary.getSystemDatabaseName(), getTableNameString())); + tableBuilder.setTableName(IdentifierUtil.buildFQName(dictionary.getSystemDatabaseName(), getTableNameString())); tableBuilder.setPath(dictionary.getTablePath()); tableBuilder.setSchema(CatalogUtil.getQualfiedSchema( diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java index 88fabe2b8a..ed4dbb22dd 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java @@ -37,6 +37,7 @@ import org.apache.tajo.exception.*; import org.apache.tajo.plan.expr.AlgebraicUtil; import org.apache.tajo.plan.util.PartitionFilterAlgebraVisitor; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.util.JavaResourceUtil; import org.apache.tajo.util.Pair; @@ -756,7 +757,7 @@ public void createTable(final CatalogProtos.TableDescProto table) PreparedStatement pstmt = null; ResultSet res = null; - String[] splitted = CatalogUtil.splitTableName(table.getTableName()); + String[] splitted = IdentifierUtil.splitTableName(table.getTableName()); if (splitted.length == 1) { throw new TajoInternalError( "createTable() requires a qualified table name, but it is '" + table.getTableName() + "'"); @@ -825,7 +826,7 @@ public void createTable(final CatalogProtos.TableDescProto table) TajoDataTypes.DataType dataType = col.getDataType(); pstmt.setInt(1, tableId); - pstmt.setString(2, CatalogUtil.extractSimpleName(col.getName())); + pstmt.setString(2, IdentifierUtil.extractSimpleName(col.getName())); pstmt.setInt(3, i); // the default number of nested fields is 0. pstmt.setInt(4, dataType.hasNumNestedFields() ? dataType.getNumNestedFields() : 0); @@ -911,7 +912,7 @@ public void updateTableStats(final CatalogProtos.UpdateTableStatsProto statsProt PreparedStatement pstmt = null; ResultSet res = null; - String[] splitted = CatalogUtil.splitTableName(statsProto.getTableName()); + String[] splitted = IdentifierUtil.splitTableName(statsProto.getTableName()); if (splitted.length == 1) { throw new IllegalArgumentException("updateTableStats() requires a qualified table name, but it is \"" + statsProto.getTableName() + "\"."); @@ -978,7 +979,7 @@ public void alterTable(CatalogProtos.AlterTableDescProto alterTableDescProto) DuplicatePartitionException, UndefinedPartitionException, UndefinedColumnException, UndefinedTableException, UndefinedPartitionMethodException, AmbiguousTableException { - String[] splitted = CatalogUtil.splitTableName(alterTableDescProto.getTableName()); + String[] splitted = IdentifierUtil.splitTableName(alterTableDescProto.getTableName()); if (splitted.length == 1) { throw new IllegalArgumentException("alterTable() requires a qualified table name, but it is \"" + alterTableDescProto.getTableName() + "\"."); @@ -993,7 +994,7 @@ public void alterTable(CatalogProtos.AlterTableDescProto alterTableDescProto) switch (alterTableDescProto.getAlterTableType()) { case RENAME_TABLE: - String simpleNewTableName = CatalogUtil.extractSimpleName(alterTableDescProto.getNewTableName()); + String simpleNewTableName = IdentifierUtil.extractSimpleName(alterTableDescProto.getNewTableName()); if (existTable(databaseName, simpleNewTableName)) { throw new DuplicateTableException(alterTableDescProto.getNewTableName()); } @@ -1172,13 +1173,13 @@ private void renameColumn(final int tableId, final CatalogProtos.AlterColumnProt conn = getConnection(); conn.setAutoCommit(false); - String tableName = CatalogUtil.extractQualifier(alterColumnProto.getOldColumnName()); - String simpleOldColumnName = CatalogUtil.extractSimpleName(alterColumnProto.getOldColumnName()); - String simpleNewColumnName = CatalogUtil.extractSimpleName(alterColumnProto.getNewColumnName()); + String tableName = IdentifierUtil.extractQualifier(alterColumnProto.getOldColumnName()); + String simpleOldColumnName = IdentifierUtil.extractSimpleName(alterColumnProto.getOldColumnName()); + String simpleNewColumnName = IdentifierUtil.extractSimpleName(alterColumnProto.getNewColumnName()); - if (!tableName.equals(CatalogUtil.extractQualifier(alterColumnProto.getNewColumnName()))) { + if (!tableName.equals(IdentifierUtil.extractQualifier(alterColumnProto.getNewColumnName()))) { throw new AmbiguousTableException( - tableName + ", " + CatalogUtil.extractQualifier(alterColumnProto.getNewColumnName())); + tableName + ", " + IdentifierUtil.extractQualifier(alterColumnProto.getNewColumnName())); } //SELECT COLUMN @@ -1251,7 +1252,7 @@ private void addNewColumn(int tableId, CatalogProtos.ColumnProto columnProto) th conn = getConnection(); pstmt = conn.prepareStatement(existColumnSql); pstmt.setInt(1, tableId); - pstmt.setString(2, CatalogUtil.extractSimpleName(columnProto.getName())); + pstmt.setString(2, IdentifierUtil.extractSimpleName(columnProto.getName())); resultSet = pstmt.executeQuery(); if (resultSet.next()) { @@ -1275,7 +1276,7 @@ private void addNewColumn(int tableId, CatalogProtos.ColumnProto columnProto) th pstmt = conn.prepareStatement(insertNewColumnSql); pstmt.setInt(1, tableId); - pstmt.setString(2, CatalogUtil.extractSimpleName(columnProto.getName())); + pstmt.setString(2, IdentifierUtil.extractSimpleName(columnProto.getName())); pstmt.setInt(3, position + 1); pstmt.setInt(4, dataType.hasNumNestedFields() ? dataType.getNumNestedFields() : 0); pstmt.setString(5, dataType.getType().name()); @@ -1600,7 +1601,7 @@ public CatalogProtos.TableDescProto getTable(String databaseName, String tableNa } int tableId = res.getInt(1); - tableBuilder.setTableName(CatalogUtil.buildFQName(databaseName, res.getString(2).trim())); + tableBuilder.setTableName(IdentifierUtil.buildFQName(databaseName, res.getString(2).trim())); TableType tableType = TableType.valueOf(res.getString(3)); if (tableType == TableType.EXTERNAL) { tableBuilder.setIsExternal(true); @@ -1860,7 +1861,7 @@ public List getAllColumns() { String databaseName = getDatabaseNameOfTable(conn, tid); String tableName = getTableName(conn, tid); builder.setTid(tid); - builder.setName(CatalogUtil.buildFQName(databaseName, tableName, resultSet.getString("COLUMN_NAME"))); + builder.setName(IdentifierUtil.buildFQName(databaseName, tableName, resultSet.getString("COLUMN_NAME"))); int nestedFieldNum = resultSet.getInt("NESTED_FIELD_NUM"); @@ -2484,7 +2485,7 @@ public void createIndex(final IndexDescProto proto) PreparedStatement pstmt = null; final String databaseName = proto.getTableIdentifier().getDatabaseName(); - final String tableName = CatalogUtil.extractSimpleName(proto.getTableIdentifier().getTableName()); + final String tableName = IdentifierUtil.extractSimpleName(proto.getTableIdentifier().getTableName()); try { @@ -2576,7 +2577,7 @@ public void dropIndex(String databaseName, final String indexName) pstmt.setString(2, indexName); ResultSet res = pstmt.executeQuery(); if (!res.next()) { - throw new UndefinedIndexException(CatalogUtil.buildFQName(databaseName, indexName)); + throw new UndefinedIndexException(IdentifierUtil.buildFQName(databaseName, indexName)); } pstmt.close(); res.close(); @@ -2676,7 +2677,7 @@ public IndexDescProto getIndexByName(String databaseName, final String indexName IndexDescProto.Builder builder = IndexDescProto.newBuilder(); String tableName = getTableName(conn, res.getInt(COL_TABLES_PK)); builder.setTableIdentifier(CatalogUtil.buildTableIdentifier(databaseName, tableName)); - resultToIndexDescProtoBuilder(CatalogUtil.buildFQName(databaseName, tableName), builder, res); + resultToIndexDescProtoBuilder(IdentifierUtil.buildFQName(databaseName, tableName), builder, res); try { builder.setTargetRelationSchema(getTable(databaseName, tableName).getSchema()); @@ -2727,7 +2728,7 @@ public IndexDescProto getIndexByColumns(String databaseName, String tableName, S } IndexDescProto.Builder builder = IndexDescProto.newBuilder(); - resultToIndexDescProtoBuilder(CatalogUtil.buildFQName(databaseName, tableName), builder, res); + resultToIndexDescProtoBuilder(IdentifierUtil.buildFQName(databaseName, tableName), builder, res); builder.setTableIdentifier(CatalogUtil.buildTableIdentifier(databaseName, tableName)); builder.setTargetRelationSchema(tableDescProto.getSchema()); proto = builder.build(); @@ -2903,7 +2904,7 @@ private void resultToIndexDescProtoBuilder(final String qualifier, int columnNum = columnNames.length; for (int i = 0; i < columnNum; i++) { SortSpecProto.Builder colSpecBuilder = SortSpecProto.newBuilder(); - colSpecBuilder.setColumn(ColumnProto.newBuilder().setName(CatalogUtil.buildFQName(qualifier, columnNames[i])) + colSpecBuilder.setColumn(ColumnProto.newBuilder().setName(IdentifierUtil.buildFQName(qualifier, columnNames[i])) .setDataType(CatalogUtil.newSimpleDataType(getDataType(dataTypes[i]))).build()); colSpecBuilder.setAscending(orders[i].equals("true")); colSpecBuilder.setNullFirst(nullOrders[i].equals("true")); diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/CatalogTestingUtil.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/CatalogTestingUtil.java index 330580187d..0b69de0c66 100644 --- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/CatalogTestingUtil.java +++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/CatalogTestingUtil.java @@ -33,6 +33,7 @@ import org.apache.tajo.conf.TajoConf; import org.apache.tajo.conf.TajoConf.ConfVars; import org.apache.tajo.exception.UnsupportedCatalogStore; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.util.KeyValueSet; import java.io.IOException; @@ -184,10 +185,10 @@ public static void prepareBaseData(CatalogService catalog, String testDir) throw } public static void cleanupBaseData(CatalogService catalog) throws Exception { - catalog.dropTable(CatalogUtil.buildFQName("TestDatabase1", "testPartition1")); - catalog.dropTable(CatalogUtil.buildFQName("TestDatabase1", "TestPartition1")); - catalog.dropTable(CatalogUtil.buildFQName("TestDatabase1", "TestTable1")); - catalog.dropTable(CatalogUtil.buildFQName("TestDatabase1", "testTable1")); + catalog.dropTable(IdentifierUtil.buildFQName("TestDatabase1", "testPartition1")); + catalog.dropTable(IdentifierUtil.buildFQName("TestDatabase1", "TestPartition1")); + catalog.dropTable(IdentifierUtil.buildFQName("TestDatabase1", "TestTable1")); + catalog.dropTable(IdentifierUtil.buildFQName("TestDatabase1", "testTable1")); catalog.dropDatabase("TestDatabase1"); catalog.dropDatabase("testDatabase1"); @@ -198,13 +199,13 @@ public static void cleanupBaseData(CatalogService catalog) throws Exception { public static TableDesc buildTableDesc(String databaseName, String tableName, String testDir) throws IOException { Schema schema = SchemaBuilder.builder() - .add(CatalogUtil.buildFQName(tableName, "Column"), Type.BLOB) - .add(CatalogUtil.buildFQName(tableName, "column"), Type.INT4) - .add(CatalogUtil.buildFQName(tableName, "cOlumn"), Type.INT8) + .add(IdentifierUtil.buildFQName(tableName, "Column"), Type.BLOB) + .add(IdentifierUtil.buildFQName(tableName, "column"), Type.INT4) + .add(IdentifierUtil.buildFQName(tableName, "cOlumn"), Type.INT8) .build(); Path path = new Path(testDir + "/" + UUID.randomUUID().toString(), tableName); TableDesc desc = new TableDesc( - CatalogUtil.buildFQName(databaseName, tableName), + IdentifierUtil.buildFQName(databaseName, tableName), schema, new TableMeta("TEXT", new KeyValueSet()), path.toUri(), true); @@ -215,8 +216,8 @@ public static TableDesc buildTableDesc(String databaseName, String tableName, St public static TableDesc buildPartitionTableDesc(String databaseName, String tableName, String testDir) throws Exception { Schema partSchema = SchemaBuilder.builder() - .add(CatalogUtil.buildFQName(tableName, "DaTe"), Type.TEXT) - .add(CatalogUtil.buildFQName(tableName, "dAtE"), Type.TEXT) + .add(IdentifierUtil.buildFQName(tableName, "DaTe"), Type.TEXT) + .add(IdentifierUtil.buildFQName(tableName, "dAtE"), Type.TEXT) .build(); PartitionMethodDesc partitionMethodDesc = new PartitionMethodDesc(DEFAULT_DATABASE_NAME, tableName, @@ -254,7 +255,7 @@ public static IndexDesc buildIndexDescs(String databaseName, String indexName, T for (int i = 0; i < cols.length; i++) { colSpecs[i] = new SortSpec(cols[i], true, false); } - return new IndexDesc(databaseName, CatalogUtil.extractSimpleName(table.getName()), + return new IndexDesc(databaseName, IdentifierUtil.extractSimpleName(table.getName()), indexName, new URI("idx_test"), colSpecs, IndexMethod.TWO_LEVEL_BIN_TREE, true, true, table.getSchema()); } diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/MiniCatalogServer.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/MiniCatalogServer.java index da375b2ab3..edba45cbb6 100644 --- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/MiniCatalogServer.java +++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/MiniCatalogServer.java @@ -21,6 +21,7 @@ import org.apache.tajo.TajoConstants; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.exception.*; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.util.CommonTestingUtil; import java.io.IOException; @@ -83,7 +84,7 @@ public void serviceStop() throws Exception { public void cleanup() throws UndefinedTableException, InsufficientPrivilegeException, UndefinedDatabaseException, UndefinedTablespaceException { for (String table : catalog.getAllTableNames(DEFAULT_DATABASE_NAME)) { - catalog.dropTable(CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, table)); + catalog.dropTable(IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, table)); } for (String database : catalog.getAllDatabaseNames()) { if (!database.equals(TajoConstants.DEFAULT_DATABASE_NAME) && diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java index 9c1e4305cf..fed36adc53 100644 --- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java +++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalog.java @@ -32,6 +32,7 @@ import org.apache.tajo.exception.TajoException; import org.apache.tajo.exception.UndefinedFunctionException; import org.apache.tajo.function.Function; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.util.CommonTestingUtil; import org.apache.tajo.util.KeyValueSet; import org.apache.tajo.util.TUtil; @@ -188,7 +189,7 @@ private TableDesc createMockupTable(String databaseName, String tableName) throw .build(); Path path = new Path(CommonTestingUtil.getTestDir(), tableName); TableDesc table = new TableDesc( - CatalogUtil.buildFQName(databaseName, tableName), + IdentifierUtil.buildFQName(databaseName, tableName), schema1, new TableMeta("TEXT", new KeyValueSet()), path.toUri(), true); @@ -289,7 +290,7 @@ public void testDropDatabaseWithAllTables() throws Exception { Collection tablesForThisDatabase = catalog.getAllTableNames(entry.getKey()); assertEquals(createdTablesMap.get(entry.getKey()).size(), tablesForThisDatabase.size()); for (String tableName : tablesForThisDatabase) { - assertTrue(createdTablesMap.get(entry.getKey()).contains(CatalogUtil.extractSimpleName(tableName))); + assertTrue(createdTablesMap.get(entry.getKey()).contains(IdentifierUtil.extractSimpleName(tableName))); } } } @@ -307,7 +308,7 @@ public void testGetTable() throws Exception { .build(); Path path = new Path(CommonTestingUtil.getTestDir(), "table1"); TableDesc meta = new TableDesc( - CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "getTable"), + IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, "getTable"), schema1, "TEXT", new KeyValueSet(), @@ -317,7 +318,7 @@ public void testGetTable() throws Exception { catalog.createTable(meta); assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, "getTable")); - catalog.dropTable(CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "getTable")); + catalog.dropTable(IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, "getTable")); assertFalse(catalog.existsTable(DEFAULT_DATABASE_NAME, "getTable")); } @@ -327,7 +328,7 @@ public void testGetTable() throws Exception { private static void assertSchemaEquality(String tableName, Schema schema) throws IOException, TajoException { Path path = new Path(CommonTestingUtil.getTestDir(), tableName); TableDesc tableDesc = new TableDesc( - CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, tableName), + IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, tableName), schema, "TEXT", new KeyValueSet(), @@ -339,12 +340,12 @@ private static void assertSchemaEquality(String tableName, Schema schema) throws assertTrue(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName)); // change it for the equals test. - schema.setQualifier(CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, tableName)); + schema.setQualifier(IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, tableName)); TableDesc restored = catalog.getTableDesc(DEFAULT_DATABASE_NAME, tableName); assertEquals(schema, restored.getSchema()); // drop test - catalog.dropTable(CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, tableName)); + catalog.dropTable(IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, tableName)); assertFalse(catalog.existsTable(DEFAULT_DATABASE_NAME, tableName)); } @@ -435,7 +436,7 @@ public static TableDesc prepareTable() throws IOException { TableMeta meta = CatalogUtil.newTableMeta("TEXT"); return new TableDesc( - CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, tableName), relationSchema, meta, + IdentifierUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, tableName), relationSchema, meta, new Path(CommonTestingUtil.getTestDir(), "indexed").toUri()); } @@ -631,7 +632,7 @@ public final void testAddAndDeleteTablePartitionByHash1() throws Exception { .add("score", Type.FLOAT8) .build(); - String tableName = CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "addedtable"); + String tableName = IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, "addedtable"); KeyValueSet opts = new KeyValueSet(); opts.set("file.delimiter", ","); TableMeta meta = CatalogUtil.newTableMeta("TEXT", opts); @@ -673,7 +674,7 @@ public final void testAddAndDeleteTablePartitionByHash2() throws Exception { .add("score", Type.FLOAT8) .build(); - String tableName = CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "addedtable"); + String tableName = IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, "addedtable"); KeyValueSet opts = new KeyValueSet(); opts.set("file.delimiter", ","); TableMeta meta = CatalogUtil.newTableMeta("TEXT", opts); @@ -713,7 +714,7 @@ public final void testAddAndDeleteTablePartitionByList() throws Exception { .add("score", Type.FLOAT8) .build(); - String tableName = CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "addedtable"); + String tableName = IdentifierUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "addedtable"); KeyValueSet opts = new KeyValueSet(); opts.set("file.delimiter", ","); TableMeta meta = CatalogUtil.newTableMeta("TEXT", opts); @@ -752,7 +753,7 @@ public final void testAddAndDeleteTablePartitionByRange() throws Exception { .add("score", Type.FLOAT8) .build(); - String tableName = CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "addedtable"); + String tableName = IdentifierUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "addedtable"); KeyValueSet opts = new KeyValueSet(); opts.set("file.delimiter", ","); TableMeta meta = CatalogUtil.newTableMeta("TEXT", opts); @@ -792,7 +793,7 @@ public final void testAddAndDeleteTablePartitionByColumn() throws Exception { .build(); String simpleTableName = "addedtable"; - String tableName = CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, simpleTableName); + String tableName = IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, simpleTableName); KeyValueSet opts = new KeyValueSet(); opts.set("file.delimiter", ","); TableMeta meta = CatalogUtil.newTableMeta("TEXT", opts); @@ -943,7 +944,7 @@ private void testAddPartition(String tableName, String partitionName) throws Exc catalog.alterTable(alterTableDesc); - String [] split = CatalogUtil.splitFQTableName(tableName); + String [] split = IdentifierUtil.splitFQTableName(tableName); CatalogProtos.PartitionDescProto resultDesc = catalog.getPartition(split[0], split[1], partitionName); diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java index 6af2798ac7..873c938c29 100644 --- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java +++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java @@ -26,6 +26,7 @@ import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.exception.UndefinedPartitionException; import org.apache.tajo.exception.UndefinedTableException; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.util.CommonTestingUtil; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -111,7 +112,7 @@ public void testTable() throws Exception { tableDescs.put(desc.getName(), desc); } for (TableDescriptorProto eachTableDescriptor : catalog.getAllTables()) { - String qualifiedTableName = CatalogUtil.buildFQName("TestDatabase1", eachTableDescriptor.getName()); + String qualifiedTableName = IdentifierUtil.buildFQName("TestDatabase1", eachTableDescriptor.getName()); assertTrue(tableDescs.containsKey(qualifiedTableName)); TableDesc desc = tableDescs.get(qualifiedTableName); assertEquals(desc.getUri().toString(), eachTableDescriptor.getPath()); @@ -128,13 +129,13 @@ public void testTable() throws Exception { AlterTableDesc alterTableDesc = new AlterTableDesc(); alterTableDesc.setAlterTableType(AlterTableType.RENAME_TABLE); - alterTableDesc.setNewTableName(CatalogUtil.buildFQName("TestDatabase1", "renamed_table")); - alterTableDesc.setTableName(CatalogUtil.buildFQName("TestDatabase1", "newTable")); + alterTableDesc.setNewTableName(IdentifierUtil.buildFQName("TestDatabase1", "renamed_table")); + alterTableDesc.setTableName(IdentifierUtil.buildFQName("TestDatabase1", "newTable")); catalog.alterTable(alterTableDesc); assertFalse(catalog.existsTable("TestDatabase1", "newTable")); assertTrue(catalog.existsTable("TestDatabase1", "renamed_table")); - catalog.dropTable(CatalogUtil.buildFQName("TestDatabase1", "renamed_table")); + catalog.dropTable(IdentifierUtil.buildFQName("TestDatabase1", "renamed_table")); ////////////////////////////////////////////////////////////////////////////// // table stats @@ -176,7 +177,7 @@ public void testTablePartition() throws Exception { PartitionDesc partitionDesc = CatalogTestingUtil.buildPartitionDesc(partitionName); AlterTableDesc alterTableDesc = new AlterTableDesc(); - alterTableDesc.setTableName(CatalogUtil.buildFQName("TestDatabase1", "TestPartition1")); + alterTableDesc.setTableName(IdentifierUtil.buildFQName("TestDatabase1", "TestPartition1")); alterTableDesc.setPartitionDesc(partitionDesc); alterTableDesc.setAlterTableType(AlterTableType.ADD_PARTITION); @@ -194,7 +195,7 @@ public void testTablePartition() throws Exception { partitionDesc = CatalogTestingUtil.buildPartitionDesc(partitionName); alterTableDesc = new AlterTableDesc(); - alterTableDesc.setTableName(CatalogUtil.buildFQName("TestDatabase1", "TestPartition1")); + alterTableDesc.setTableName(IdentifierUtil.buildFQName("TestDatabase1", "TestPartition1")); alterTableDesc.setPartitionDesc(partitionDesc); alterTableDesc.setAlterTableType(AlterTableType.ADD_PARTITION); @@ -242,7 +243,7 @@ public void testTablePartition() throws Exception { partitionDesc = CatalogTestingUtil.buildPartitionDesc(partitionName); alterTableDesc = new AlterTableDesc(); - alterTableDesc.setTableName(CatalogUtil.buildFQName("TestDatabase1", "TestPartition1")); + alterTableDesc.setTableName(IdentifierUtil.buildFQName("TestDatabase1", "TestPartition1")); alterTableDesc.setPartitionDesc(partitionDesc); alterTableDesc.setAlterTableType(AlterTableType.DROP_PARTITION); catalog.alterTable(alterTableDesc); @@ -258,7 +259,7 @@ public void testTablePartition() throws Exception { partitionDesc = CatalogTestingUtil.buildPartitionDesc(partitionName); alterTableDesc = new AlterTableDesc(); - alterTableDesc.setTableName(CatalogUtil.buildFQName("TestDatabase1", "TestPartition1")); + alterTableDesc.setTableName(IdentifierUtil.buildFQName("TestDatabase1", "TestPartition1")); alterTableDesc.setPartitionDesc(partitionDesc); alterTableDesc.setAlterTableType(AlterTableType.DROP_PARTITION); catalog.alterTable(alterTableDesc); @@ -281,32 +282,32 @@ public void testTableColumn() throws Exception { ////////////////////////////////////////////////////////////////////////////// AlterTableDesc alterTableDesc = new AlterTableDesc(); - alterTableDesc.setAddColumn(new Column(CatalogUtil.buildFQName(databaseName, tableName, "AddedCol1"), + alterTableDesc.setAddColumn(new Column(IdentifierUtil.buildFQName(databaseName, tableName, "AddedCol1"), CatalogUtil.newSimpleDataType(Type.BLOB))); - alterTableDesc.setTableName(CatalogUtil.buildFQName(databaseName, tableName)); + alterTableDesc.setTableName(IdentifierUtil.buildFQName(databaseName, tableName)); alterTableDesc.setAlterTableType(AlterTableType.ADD_COLUMN); catalog.alterTable(alterTableDesc); TableDesc tableDesc = catalog.getTableDesc(databaseName, tableName); assertTrue( - tableDesc.getSchema().containsByQualifiedName(CatalogUtil.buildFQName(databaseName, tableName, "AddedCol1"))); + tableDesc.getSchema().containsByQualifiedName(IdentifierUtil.buildFQName(databaseName, tableName, "AddedCol1"))); ////////////////////////////////////////////////////////////////////////////// // Test rename column ////////////////////////////////////////////////////////////////////////////// alterTableDesc = new AlterTableDesc(); - alterTableDesc.setColumnName(CatalogUtil.buildFQName(databaseName, tableName, "AddedCol1")); - alterTableDesc.setNewColumnName(CatalogUtil.buildFQName(databaseName, tableName, "addedcol1")); - alterTableDesc.setTableName(CatalogUtil.buildFQName(databaseName, tableName)); + alterTableDesc.setColumnName(IdentifierUtil.buildFQName(databaseName, tableName, "AddedCol1")); + alterTableDesc.setNewColumnName(IdentifierUtil.buildFQName(databaseName, tableName, "addedcol1")); + alterTableDesc.setTableName(IdentifierUtil.buildFQName(databaseName, tableName)); alterTableDesc.setAlterTableType(AlterTableType.RENAME_COLUMN); catalog.alterTable(alterTableDesc); tableDesc = catalog.getTableDesc(databaseName, tableName); assertFalse( - tableDesc.getSchema().containsByQualifiedName(CatalogUtil.buildFQName(databaseName, tableName, "AddedCol1"))); + tableDesc.getSchema().containsByQualifiedName(IdentifierUtil.buildFQName(databaseName, tableName, "AddedCol1"))); assertTrue( - tableDesc.getSchema().containsByQualifiedName(CatalogUtil.buildFQName(databaseName, tableName, "addedcol1"))); + tableDesc.getSchema().containsByQualifiedName(IdentifierUtil.buildFQName(databaseName, tableName, "addedcol1"))); ////////////////////////////////////////////////////////////////////////////// // Test get all columns @@ -350,7 +351,7 @@ public void testTableColumnWithNestedField() throws Exception { .add("RecoRd1", new TypeDesc(schema)).build())).build(); TableDesc tableDesc = new TableDesc( - CatalogUtil.buildFQName(databaseName, tableName), + IdentifierUtil.buildFQName(databaseName, tableName), tableSchema, CatalogUtil.newTableMeta("JSON"), URI.create("hdfs://xxx.com/json_Table") diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogExceptions.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogExceptions.java index 8ba3d59ec9..3b90e218a3 100644 --- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogExceptions.java +++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogExceptions.java @@ -29,6 +29,7 @@ import org.apache.tajo.catalog.proto.CatalogProtos.UpdateTableStatsProto; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.exception.*; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.util.CommonTestingUtil; import org.apache.tajo.util.KeyValueSet; import org.junit.AfterClass; @@ -116,14 +117,14 @@ public void testCreateTableWithWrongUri() throws Exception { // TODO: currently, wrong uri does not occur any exception. String tableName = "wrongUri"; Schema schema = SchemaBuilder.builder() - .add(CatalogUtil.buildFQName(tableName, "Column"), Type.BLOB) - .add(CatalogUtil.buildFQName(tableName, "column"), Type.INT4) - .add(CatalogUtil.buildFQName(tableName, "cOlumn"), Type.INT8) + .add(IdentifierUtil.buildFQName(tableName, "Column"), Type.BLOB) + .add(IdentifierUtil.buildFQName(tableName, "column"), Type.INT4) + .add(IdentifierUtil.buildFQName(tableName, "cOlumn"), Type.INT8) .build(); Path path = new Path(CommonTestingUtil.getTestDir(), tableName); catalog.createTable( new TableDesc( - CatalogUtil.buildFQName("TestDatabase1", tableName), + IdentifierUtil.buildFQName("TestDatabase1", tableName), schema, new TableMeta("TEXT", new KeyValueSet()), path.toUri(), true)); @@ -137,19 +138,19 @@ public void testCreateDuplicateTable() throws Exception { @Test(expected = UndefinedDatabaseException.class) public void dropTableOfUndefinedDatabase() throws Exception { - catalog.dropTable(CatalogUtil.buildFQName("undefined", "testPartition1")); + catalog.dropTable(IdentifierUtil.buildFQName("undefined", "testPartition1")); } @Test(expected = UndefinedTableException.class) public void dropUndefinedTable() throws Exception { - catalog.dropTable(CatalogUtil.buildFQName("TestDatabase1", "undefined")); + catalog.dropTable(IdentifierUtil.buildFQName("TestDatabase1", "undefined")); } @Test(expected = UndefinedTableException.class) public void testUpdateTableStatsOfUndefinedTable() throws Exception { catalog.updateTableStats( UpdateTableStatsProto.newBuilder(). - setTableName(CatalogUtil.buildFQName("TestDatabase1", "undefined")). + setTableName(IdentifierUtil.buildFQName("TestDatabase1", "undefined")). setStats( TableStatsProto.newBuilder(). setNumRows(0). @@ -165,7 +166,7 @@ public void testAddPartitionWithWrongUri() throws Exception { PartitionDesc partitionDesc = CatalogTestingUtil.buildPartitionDesc(partitionName); AlterTableDesc alterTableDesc = new AlterTableDesc(); - alterTableDesc.setTableName(CatalogUtil.buildFQName("TestDatabase1", "TestPartition1")); + alterTableDesc.setTableName(IdentifierUtil.buildFQName("TestDatabase1", "TestPartition1")); alterTableDesc.setPartitionDesc(partitionDesc); alterTableDesc.setAlterTableType(AlterTableType.ADD_PARTITION); @@ -178,7 +179,7 @@ public void testAddDuplicatePartition() throws Exception { PartitionDesc partitionDesc = CatalogTestingUtil.buildPartitionDesc(partitionName); AlterTableDesc alterTableDesc = new AlterTableDesc(); - alterTableDesc.setTableName(CatalogUtil.buildFQName("TestDatabase1", "TestPartition1")); + alterTableDesc.setTableName(IdentifierUtil.buildFQName("TestDatabase1", "TestPartition1")); alterTableDesc.setPartitionDesc(partitionDesc); alterTableDesc.setAlterTableType(AlterTableType.ADD_PARTITION); @@ -188,7 +189,7 @@ public void testAddDuplicatePartition() throws Exception { partitionDesc = CatalogTestingUtil.buildPartitionDesc(partitionName); alterTableDesc = new AlterTableDesc(); - alterTableDesc.setTableName(CatalogUtil.buildFQName("TestDatabase1", "TestPartition1")); + alterTableDesc.setTableName(IdentifierUtil.buildFQName("TestDatabase1", "TestPartition1")); alterTableDesc.setPartitionDesc(partitionDesc); alterTableDesc.setAlterTableType(AlterTableType.ADD_PARTITION); @@ -201,7 +202,7 @@ public void testAddPartitionToUndefinedTable() throws Exception { PartitionDesc partitionDesc = CatalogTestingUtil.buildPartitionDesc(partitionName); AlterTableDesc alterTableDesc = new AlterTableDesc(); - alterTableDesc.setTableName(CatalogUtil.buildFQName("TestDatabase1", "undefined")); + alterTableDesc.setTableName(IdentifierUtil.buildFQName("TestDatabase1", "undefined")); alterTableDesc.setPartitionDesc(partitionDesc); alterTableDesc.setAlterTableType(AlterTableType.ADD_PARTITION); @@ -215,7 +216,7 @@ public void testAddPartitionNotimplementedException() throws Exception { PartitionDesc partitionDesc = CatalogTestingUtil.buildPartitionDesc(partitionName); AlterTableDesc alterTableDesc = new AlterTableDesc(); - alterTableDesc.setTableName(CatalogUtil.buildFQName("TestDatabase1", "TestPartition1")); + alterTableDesc.setTableName(IdentifierUtil.buildFQName("TestDatabase1", "TestPartition1")); alterTableDesc.setPartitionDesc(partitionDesc); alterTableDesc.setAlterTableType(AlterTableType.ADD_PARTITION); @@ -228,7 +229,7 @@ public void testDropUndefinedPartition() throws Exception { PartitionDesc partitionDesc = CatalogTestingUtil.buildPartitionDesc(partitionName); AlterTableDesc alterTableDesc = new AlterTableDesc(); - alterTableDesc.setTableName(CatalogUtil.buildFQName("TestDatabase1", "TestPartition1")); + alterTableDesc.setTableName(IdentifierUtil.buildFQName("TestDatabase1", "TestPartition1")); alterTableDesc.setPartitionDesc(partitionDesc); alterTableDesc.setAlterTableType(AlterTableType.DROP_PARTITION); catalog.alterTable(alterTableDesc); @@ -238,17 +239,17 @@ public void testDropUndefinedPartition() throws Exception { public void testRenameUndefinedTable() throws Exception { AlterTableDesc alterTableDesc = new AlterTableDesc(); alterTableDesc.setAlterTableType(AlterTableType.RENAME_TABLE); - alterTableDesc.setNewTableName(CatalogUtil.buildFQName("TestDatabase1", "renamed_table")); - alterTableDesc.setTableName(CatalogUtil.buildFQName("TestDatabase1", "undefined")); + alterTableDesc.setNewTableName(IdentifierUtil.buildFQName("TestDatabase1", "renamed_table")); + alterTableDesc.setTableName(IdentifierUtil.buildFQName("TestDatabase1", "undefined")); catalog.alterTable(alterTableDesc); } @Test(expected = UndefinedTableException.class) public void testRenameColumnOfUndefinedTable() throws Exception { AlterTableDesc alterTableDesc = new AlterTableDesc(); - alterTableDesc.setColumnName(CatalogUtil.buildFQName("TestDatabase1", "undefined", "AddedCol1")); - alterTableDesc.setNewColumnName(CatalogUtil.buildFQName("TestDatabase1", "undefined", "addedcol1")); - alterTableDesc.setTableName(CatalogUtil.buildFQName("TestDatabase1", "undefined")); + alterTableDesc.setColumnName(IdentifierUtil.buildFQName("TestDatabase1", "undefined", "AddedCol1")); + alterTableDesc.setNewColumnName(IdentifierUtil.buildFQName("TestDatabase1", "undefined", "addedcol1")); + alterTableDesc.setTableName(IdentifierUtil.buildFQName("TestDatabase1", "undefined")); alterTableDesc.setAlterTableType(AlterTableType.RENAME_COLUMN); catalog.alterTable(alterTableDesc); } @@ -256,9 +257,9 @@ public void testRenameColumnOfUndefinedTable() throws Exception { @Test(expected = UndefinedColumnException.class) public void testRenameUndefinedColumn() throws Exception { AlterTableDesc alterTableDesc = new AlterTableDesc(); - alterTableDesc.setColumnName(CatalogUtil.buildFQName("TestDatabase1", "TestTable1", "undefined")); - alterTableDesc.setNewColumnName(CatalogUtil.buildFQName("TestDatabase1", "TestTable1", "undefined")); - alterTableDesc.setTableName(CatalogUtil.buildFQName("TestDatabase1", "TestTable1")); + alterTableDesc.setColumnName(IdentifierUtil.buildFQName("TestDatabase1", "TestTable1", "undefined")); + alterTableDesc.setNewColumnName(IdentifierUtil.buildFQName("TestDatabase1", "TestTable1", "undefined")); + alterTableDesc.setTableName(IdentifierUtil.buildFQName("TestDatabase1", "TestTable1")); alterTableDesc.setAlterTableType(AlterTableType.RENAME_COLUMN); catalog.alterTable(alterTableDesc); } @@ -266,9 +267,9 @@ public void testRenameUndefinedColumn() throws Exception { @Test(expected = DuplicateColumnException.class) public void testAddDuplicateColumn() throws Exception { AlterTableDesc alterTableDesc = new AlterTableDesc(); - alterTableDesc.setAddColumn(new Column(CatalogUtil.buildFQName("TestDatabase1", "TestTable1", "cOlumn"), + alterTableDesc.setAddColumn(new Column(IdentifierUtil.buildFQName("TestDatabase1", "TestTable1", "cOlumn"), CatalogUtil.newSimpleDataType(Type.BLOB))); - alterTableDesc.setTableName(CatalogUtil.buildFQName("TestDatabase1", "TestTable1")); + alterTableDesc.setTableName(IdentifierUtil.buildFQName("TestDatabase1", "TestTable1")); alterTableDesc.setAlterTableType(AlterTableType.ADD_COLUMN); catalog.alterTable(alterTableDesc); } diff --git a/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java b/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java index c9fa2b488c..c1ada74364 100644 --- a/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java +++ b/tajo-cli/src/main/java/org/apache/tajo/cli/tools/TajoDump.java @@ -26,6 +26,7 @@ import org.apache.tajo.client.TajoClient; import org.apache.tajo.client.TajoClientImpl; import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.service.ServiceTrackerFactory; import org.apache.tajo.util.Pair; @@ -161,10 +162,10 @@ private static void dumpDatabase(TajoClient client, String databaseName, PrintWr throws SQLException, ServiceException { writer.write("\n"); writer.write("--\n"); - writer.write(String.format("-- Database name: %s%n", CatalogUtil.denormalizeIdentifier(databaseName))); + writer.write(String.format("-- Database name: %s%n", IdentifierUtil.denormalizeIdentifier(databaseName))); writer.write("--\n"); writer.write("\n"); - writer.write(String.format("CREATE DATABASE IF NOT EXISTS %s;", CatalogUtil.denormalizeIdentifier(databaseName))); + writer.write(String.format("CREATE DATABASE IF NOT EXISTS %s;", IdentifierUtil.denormalizeIdentifier(databaseName))); writer.write("\n\n"); // returned list is immutable. @@ -172,7 +173,7 @@ private static void dumpDatabase(TajoClient client, String databaseName, PrintWr Collections.sort(tableNames); for (String tableName : tableNames) { try { - String fqName = CatalogUtil.buildFQName(databaseName, tableName); + String fqName = IdentifierUtil.buildFQName(databaseName, tableName); TableDesc table = client.getTableDesc(fqName); if (table.getMeta().getDataFormat().equalsIgnoreCase("SYSTEM")) { @@ -195,7 +196,7 @@ private static void dumpDatabase(TajoClient client, String databaseName, PrintWr // writer.write(DDLBuilder.buildDDLForAddPartition(table, eachPartitionProto)); // } writer.write(String.format("ALTER TABLE %s REPAIR PARTITION;", - CatalogUtil.denormalizeIdentifier(databaseName) + "." + CatalogUtil.denormalizeIdentifier(tableName))); + IdentifierUtil.denormalizeIdentifier(databaseName) + "." + IdentifierUtil.denormalizeIdentifier(tableName))); writer.write("\n\n"); } diff --git a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java index 6f0191113c..3220dfda56 100644 --- a/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java +++ b/tajo-cli/src/main/java/org/apache/tajo/cli/tsql/commands/DescTableCommand.java @@ -24,7 +24,6 @@ import org.apache.commons.lang.CharUtils; import org.apache.commons.lang.StringEscapeUtils; import org.apache.tajo.TajoConstants; -import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.catalog.partition.PartitionMethodDesc; @@ -32,6 +31,7 @@ import org.apache.tajo.catalog.proto.CatalogProtos.SortSpecProto; import org.apache.tajo.cli.tsql.TajoCli; import org.apache.tajo.exception.TajoException; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.util.FileUtil; import org.apache.tajo.util.StringUtils; @@ -72,7 +72,7 @@ public void invoke(String[] cmd) throws TajoException { sb.append("\"").append(index.getIndexName()).append("\" "); sb.append(index.getIndexMethod()).append(" ("); for (SortSpecProto key : index.getKeySortSpecsList()) { - sb.append(CatalogUtil.extractSimpleName(key.getColumn().getName())); + sb.append(IdentifierUtil.extractSimpleName(key.getColumn().getName())); sb.append(key.getAscending() ? " ASC" : " DESC"); sb.append(key.getNullFirst() ? " NULLS FIRST, " : " NULLS LAST, "); } diff --git a/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSetMetaData.java b/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSetMetaData.java index 8811c134eb..2f654cd8b9 100644 --- a/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSetMetaData.java +++ b/tajo-client/src/main/java/org/apache/tajo/jdbc/TajoResultSetMetaData.java @@ -21,12 +21,12 @@ */ package org.apache.tajo.jdbc; -import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.Schema; import org.apache.tajo.client.ResultSetUtil; import org.apache.tajo.common.TajoDataTypes.DataType; import org.apache.tajo.common.type.TajoTypeUtil; +import org.apache.tajo.schema.IdentifierUtil; import java.sql.ResultSetMetaData; import java.sql.SQLException; @@ -53,8 +53,8 @@ public T unwrap(Class clazz) throws SQLException { @Override public String getCatalogName(int column) throws SQLException { Column c = schema.getColumn(column - 1); - if (CatalogUtil.isFQColumnName(c.getQualifiedName())) { - return CatalogUtil.splitFQTableName(c.getQualifier())[0]; + if (IdentifierUtil.isFQColumnName(c.getQualifiedName())) { + return IdentifierUtil.splitFQTableName(c.getQualifier())[0]; } return ""; } diff --git a/tajo-cluster-tests/src/test/java/org/apache/tajo/LocalTajoTestingUtility.java b/tajo-cluster-tests/src/test/java/org/apache/tajo/LocalTajoTestingUtility.java index f084138305..18cd8c170f 100644 --- a/tajo-cluster-tests/src/test/java/org/apache/tajo/LocalTajoTestingUtility.java +++ b/tajo-cluster-tests/src/test/java/org/apache/tajo/LocalTajoTestingUtility.java @@ -36,6 +36,7 @@ import org.apache.tajo.engine.planner.global.MasterPlan; import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.exception.TajoException; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.session.Session; import org.apache.tajo.util.CommonTestingUtil; import org.apache.tajo.util.FileUtil; @@ -122,7 +123,7 @@ public void setup(String[] names, TableStats stats = new TableStats(); stats.setNumBytes(TPCH.tableVolumes.get(names[i])); TableDesc tableDesc = new TableDesc( - CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, names[i]), schemas[i], meta, + IdentifierUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, names[i]), schemas[i], meta, tablePath.toUri()); tableDesc.setStats(stats); util.getMaster().getCatalog().createTable(tableDesc); diff --git a/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java b/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java index 116990f6fe..68c54f75b2 100644 --- a/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java +++ b/tajo-cluster-tests/src/test/java/org/apache/tajo/QueryTestCaseBase.java @@ -31,7 +31,6 @@ import org.apache.tajo.algebra.*; import org.apache.tajo.annotation.Nullable; import org.apache.tajo.catalog.CatalogService; -import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.cli.tsql.InvalidStatementException; import org.apache.tajo.cli.tsql.ParsedResult; @@ -53,6 +52,7 @@ import org.apache.tajo.plan.verifier.LogicalPlanVerifier; import org.apache.tajo.plan.verifier.PreLogicalPlanVerifier; import org.apache.tajo.plan.verifier.VerificationState; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.storage.BufferPool; import org.apache.tajo.storage.StorageUtil; import org.apache.tajo.util.FileUtil; @@ -224,7 +224,7 @@ public static void setUpClass() throws Exception { @AfterClass public static void tearDownClass() throws Exception { for (String tableName : createdTableGlobalSet) { - client.updateQuery("DROP TABLE IF EXISTS " + CatalogUtil.denormalizeIdentifier(tableName)); + client.updateQuery("DROP TABLE IF EXISTS " + IdentifierUtil.denormalizeIdentifier(tableName)); } createdTableGlobalSet.clear(); @@ -301,7 +301,7 @@ private void init() { try { // if the current database is "default", we don't need create it because it is already prepated at startup time. if (!currentDatabase.equals(TajoConstants.DEFAULT_DATABASE_NAME)) { - client.updateQuery("CREATE DATABASE IF NOT EXISTS " + CatalogUtil.denormalizeIdentifier(currentDatabase)); + client.updateQuery("CREATE DATABASE IF NOT EXISTS " + IdentifierUtil.denormalizeIdentifier(currentDatabase)); } client.selectDatabase(currentDatabase); currentResultFS = currentResultPath.getFileSystem(testBase.getTestingCluster().getConfiguration()); @@ -1045,10 +1045,10 @@ private List executeDDL(String ddlFileName, @Nullable String dataFileNam DropTable dropTable = (DropTable) expr; String tableName = dropTable.getTableName(); assertTrue("table '" + tableName + "' existence check", - client.existTable(CatalogUtil.buildFQName(currentDatabase, tableName))); + client.existTable(IdentifierUtil.buildFQName(currentDatabase, tableName))); assertTrue("table drop is failed.", client.updateQuery(parsedResult.getHistoryStatement())); assertFalse("table '" + tableName + "' dropped check", - client.existTable(CatalogUtil.buildFQName(currentDatabase, tableName))); + client.existTable(IdentifierUtil.buildFQName(currentDatabase, tableName))); if (isLocalTable) { createdTableGlobalSet.remove(tableName); } diff --git a/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java b/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java index 4e7d236e87..cfe43c2ae1 100644 --- a/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java +++ b/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java @@ -45,6 +45,7 @@ import org.apache.tajo.querymaster.QueryMasterTask; import org.apache.tajo.querymaster.Stage; import org.apache.tajo.querymaster.StageState; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.service.ServiceTrackerFactory; import org.apache.tajo.storage.FileTablespace; import org.apache.tajo.storage.TablespaceManager; @@ -665,8 +666,8 @@ public static void createTable(String tableName, Schema schema, fs.mkdirs(rootDir); } Path tablePath; - if (CatalogUtil.isFQTableName(tableName)) { - Pair name = CatalogUtil.separateQualifierAndName(tableName); + if (IdentifierUtil.isFQTableName(tableName)) { + Pair name = IdentifierUtil.separateQualifierAndName(tableName); tablePath = new Path(rootDir, new Path(name.getFirst(), name.getSecond())); } else { tablePath = new Path(rootDir, tableName); diff --git a/tajo-common/src/main/java/org/apache/tajo/schema/Field.java b/tajo-common/src/main/java/org/apache/tajo/schema/Field.java index e9adacf845..803236dfed 100644 --- a/tajo-common/src/main/java/org/apache/tajo/schema/Field.java +++ b/tajo-common/src/main/java/org/apache/tajo/schema/Field.java @@ -18,15 +18,18 @@ package org.apache.tajo.schema; +import com.google.common.annotations.VisibleForTesting; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.type.Type; +import java.util.Arrays; +import java.util.Collection; import java.util.Objects; /** * Represent a field in a schema. */ -public class Field { +public class Field implements Cloneable { protected final Type type; protected final QualifiedIdentifier name; @@ -35,6 +38,23 @@ public Field(Type type, QualifiedIdentifier name) { this.name = name; } + public static Field Record(QualifiedIdentifier name, Field ... fields) { + return Record(name, Arrays.asList(fields)); + } + + public static Field Record(QualifiedIdentifier name, Collection fields) { + return new Field(Type.Struct(fields), name); + } + + @VisibleForTesting + public static Field Field(String name, Type type) { + return new Field(type, QualifiedIdentifier.$(name)); + } + + public static Field Field(QualifiedIdentifier name, Type type) { + return new Field(type, name); + } + public QualifiedIdentifier name() { return this.name; } @@ -74,9 +94,16 @@ public boolean equals(Object obj) { if (obj instanceof Field) { Field other = (Field) obj; - return this.type.equals(other) && this.name.equals(other.name); + boolean eq = type.equals(other.type); + eq &= name.equals(other.name); + return eq; } return false; } + + @Override + public Field clone() throws CloneNotSupportedException { + return this; + } } diff --git a/tajo-common/src/main/java/org/apache/tajo/schema/Identifier.java b/tajo-common/src/main/java/org/apache/tajo/schema/Identifier.java index d71dbdc16c..6fb3693bd8 100644 --- a/tajo-common/src/main/java/org/apache/tajo/schema/Identifier.java +++ b/tajo-common/src/main/java/org/apache/tajo/schema/Identifier.java @@ -101,7 +101,7 @@ public boolean equals(Object obj) { if (obj instanceof Identifier) { Identifier other = (Identifier) obj; - return other.name == other.name && quoted == other.quoted; + return name.equals(other.name) && quoted == other.quoted; } return false; diff --git a/tajo-common/src/main/java/org/apache/tajo/schema/IdentifierUtil.java b/tajo-common/src/main/java/org/apache/tajo/schema/IdentifierUtil.java new file mode 100644 index 0000000000..78b4a3a21e --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/schema/IdentifierUtil.java @@ -0,0 +1,289 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.schema; + +import com.google.common.base.Preconditions; +import org.apache.tajo.TajoConstants; +import org.apache.tajo.util.Pair; +import org.apache.tajo.util.StringUtils; + +import java.util.HashSet; +import java.util.Set; + +/** + * Util methods for Identifiers + */ +public class IdentifierUtil { + + public final static String IDENTIFIER_DELIMITER_REGEXP = "\\."; + public final static String IDENTIFIER_DELIMITER = "."; + public final static String IDENTIFIER_QUOTE_STRING = "\""; + public final static int MAX_IDENTIFIER_LENGTH = 128; + + public static final Set RESERVED_KEYWORDS_SET = new HashSet<>(); + public static final String [] RESERVED_KEYWORDS = { + "AS", "ALL", "AND", "ANY", "ASYMMETRIC", "ASC", + "BOTH", + "CASE", "CAST", "CREATE", "CROSS", "CURRENT_DATE", "CURRENT_TIME", "CURRENT_TIMESTAMP", + "DESC", "DISTINCT", + "END", "ELSE", "EXCEPT", + "FALSE", "FULL", "FROM", + "GROUP", + "HAVING", + "ILIKE", "IN", "INNER", "INTERSECT", "INTO", "IS", + "JOIN", + "LEADING", "LEFT", "LIKE", "LIMIT", + "NATURAL", "NOT", "NULL", + "ON", "OUTER", "OR", "ORDER", + "RIGHT", + "SELECT", "SOME", "SYMMETRIC", + "TABLE", "THEN", "TRAILING", "TRUE", + "OVER", + "UNION", "UNIQUE", "USING", + "WHEN", "WHERE", "WINDOW", "WITH" + }; + + + /** + * Normalize an identifier. Normalization means a translation from a identifier to be a refined identifier name. + * + * Identifier can be composed of multiple parts as follows: + * 
+   *   database_name.table_name.column_name
+   *
+ * + * Each regular identifier part can be composed alphabet ([a-z][A-Z]), number([0-9]), and underscore([_]). + * Also, the first letter must be an alphabet character. + * + * normalizeIdentifier normalizes each part of an identifier. + * + * In detail, for each part, it performs as follows: + *
    + *
  • changing a part without double quotation to be lower case letters
    • + *
  • eliminating double quotation marks from identifier
    • + *
+ * + * @param identifier The identifier to be normalized + * @return The normalized identifier + */ + public static String normalizeIdentifier(String identifier) { + if (identifier == null || identifier.equals("")) { + return identifier; + } + String [] splitted = identifier.split(IDENTIFIER_DELIMITER_REGEXP); + + StringBuilder sb = new StringBuilder(); + boolean first = true; + for (String part : splitted) { + if (first) { + first = false; + } else { + sb.append(IDENTIFIER_DELIMITER); + } + sb.append(normalizeIdentifierPart(part)); + } + return sb.toString(); + } + + public static String normalizeIdentifierPart(String part) { + return isDelimited(part) ? stripQuote(part) : part.toLowerCase(); + } + + /** + * Denormalize an identifier. Denormalize means a translation from a stored identifier + * to be a printable identifier name. + * + * In detail, for each part, it performs as follows: + *
    + *
  • changing a part including upper case character or non-ascii character to be lower case letters
    • + *
  • eliminating double quotation marks from identifier
    • + *
+ * + * @param identifier The identifier to be normalized + * @return The denormalized identifier + */ + public static String denormalizeIdentifier(String identifier) { + String [] splitted = identifier.split(IDENTIFIER_DELIMITER_REGEXP); + + StringBuilder sb = new StringBuilder(); + boolean first = true; + for (String part : splitted) { + if (first) { + first = false; + } else { + sb.append(IDENTIFIER_DELIMITER); + } + sb.append(denormalizePart(part)); + } + return sb.toString(); + } + + public static String denormalizePart(String identifier) { + if (isShouldBeQuoted(identifier)) { + return StringUtils.doubleQuote(identifier); + } else { + return identifier; + } + } + + public static boolean isShouldBeQuoted(String columnName) { + for (char character : columnName.toCharArray()) { + if (Character.isUpperCase(character)) { + return true; + } + + if (!StringUtils.isPartOfAnsiSQLIdentifier(character)) { + return true; + } + + if (RESERVED_KEYWORDS_SET.contains(columnName.toUpperCase())) { + return true; + } + } + + return false; + } + + public static String stripQuote(String str) { + return str.substring(1, str.length() - 1); + } + + public static boolean isDelimited(String identifier) { + boolean openQuote = identifier.charAt(0) == '"'; + boolean closeQuote = identifier.charAt(identifier.length() - 1) == '"'; + + // if at least one quote mark exists, the identifier must be grater than equal to 2 characters, + if (openQuote ^ closeQuote && identifier.length() < 2) { + throw new IllegalArgumentException("Invalid Identifier: " + identifier); + } + + // does not allow the empty identifier (''), + if (openQuote && closeQuote && identifier.length() == 2) { + throw new IllegalArgumentException("zero-length delimited identifier: " + identifier); + } + + // Ensure the quote open and close + return openQuote && closeQuote; + } + + /** + * True if a given name is a simple identifier, meaning is not a dot-chained name. + * + * @param columnOrTableName Column or Table name to be checked + * @return True if a given name is a simple identifier. Otherwise, it will return False. + */ + public static boolean isSimpleIdentifier(String columnOrTableName) { + return columnOrTableName.split(IDENTIFIER_DELIMITER_REGEXP).length == 1; + } + + public static boolean isFQColumnName(String tableName) { + return tableName.split(IDENTIFIER_DELIMITER_REGEXP).length == 3; + } + + public static boolean isFQTableName(String tableName) { + int lastDelimiterIdx = tableName.lastIndexOf(IDENTIFIER_DELIMITER); + return lastDelimiterIdx > -1; + } + + public static String [] splitFQTableName(String qualifiedName) { + String [] splitted = splitTableName(qualifiedName); + if (splitted.length == 1) { + throw new IllegalArgumentException("Table name is expected to be qualified, but was \"" + + qualifiedName + "\"."); + } + return splitted; + } + + public static String [] splitTableName(String tableName) { + int lastDelimiterIdx = tableName.lastIndexOf(IDENTIFIER_DELIMITER); + if (lastDelimiterIdx > -1) { + return new String [] { + tableName.substring(0, lastDelimiterIdx), + tableName.substring(lastDelimiterIdx + 1, tableName.length()) + }; + } else { + return new String [] {tableName}; + } + } + + public static String buildFQName(String... identifiers) { + boolean first = true; + StringBuilder sb = new StringBuilder(); + for(String id : identifiers) { + if (first) { + first = false; + } else { + sb.append(IDENTIFIER_DELIMITER); + } + + sb.append(id); + } + + return sb.toString(); + } + + public static Pair separateQualifierAndName(String name) { + Preconditions.checkArgument(isFQTableName(name), "Must be a qualified name."); + return new Pair<>(extractQualifier(name), extractSimpleName(name)); + } + + /** + * Extract a qualification name from an identifier. + * + * For example, consider a table identifier like 'database1.table1'. + * In this case, this method extracts 'database1'. + * + * @param name The identifier to be extracted + * @return The extracted qualifier + */ + public static String extractQualifier(String name) { + int lastDelimiterIdx = name.lastIndexOf(IDENTIFIER_DELIMITER); + if (lastDelimiterIdx > -1) { + return name.substring(0, lastDelimiterIdx); + } else { + return TajoConstants.EMPTY_STRING; + } + } + + /** + * Extract a simple name from an identifier. + * + * For example, consider a table identifier like 'database1.table1'. + * In this case, this method extracts 'table1'. + * + * @param name The identifier to be extracted + * @return The extracted simple name + */ + public static String extractSimpleName(String name) { + int lastDelimiterIdx = name.lastIndexOf(IDENTIFIER_DELIMITER); + if (lastDelimiterIdx > -1) { + // plus one means skipping a delimiter. + return name.substring(lastDelimiterIdx + 1, name.length()); + } else { + return name; + } + } + + public static String getCanonicalTableName(String databaseName, String tableName) { + StringBuilder sb = new StringBuilder(databaseName); + sb.append(IDENTIFIER_DELIMITER); + sb.append(tableName); + return sb.toString(); + } +} diff --git a/tajo-common/src/main/java/org/apache/tajo/schema/QualifiedIdentifier.java b/tajo-common/src/main/java/org/apache/tajo/schema/QualifiedIdentifier.java index 75c9b728ef..b82057f3eb 100644 --- a/tajo-common/src/main/java/org/apache/tajo/schema/QualifiedIdentifier.java +++ b/tajo-common/src/main/java/org/apache/tajo/schema/QualifiedIdentifier.java @@ -26,6 +26,7 @@ import javax.annotation.Nullable; import java.util.Collection; +import java.util.Objects; import static org.apache.tajo.schema.IdentifierPolicy.DefaultPolicy; @@ -65,12 +66,12 @@ public String apply(@Nullable Identifier identifier) { @Override public String toString() { - return displayString(DefaultPolicy()); + return raw(DefaultPolicy()); } @Override public int hashCode() { - return names.hashCode(); + return Objects.hash(names); } public boolean equals(Object obj) { @@ -96,8 +97,10 @@ public boolean equals(Object obj) { @VisibleForTesting public static QualifiedIdentifier $(String...names) { ImmutableList.Builder builder = new ImmutableList.Builder(); - for (String n :names) { - builder.add(Identifier._(n)); + for (String n : names) { + for (String split : n.split(StringUtils.escapeRegexp(DefaultPolicy().getIdentifierSeperator()))) { + builder.add(Identifier._(split, IdentifierUtil.isShouldBeQuoted(split))); + } } return new QualifiedIdentifier(builder.build()); } diff --git a/tajo-common/src/main/java/org/apache/tajo/schema/Schema.java b/tajo-common/src/main/java/org/apache/tajo/schema/Schema.java index 54be2bc3be..b3b28b7e89 100644 --- a/tajo-common/src/main/java/org/apache/tajo/schema/Schema.java +++ b/tajo-common/src/main/java/org/apache/tajo/schema/Schema.java @@ -18,9 +18,7 @@ package org.apache.tajo.schema; -import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; -import org.apache.tajo.type.Type; import org.apache.tajo.util.StringUtils; import java.util.Arrays; @@ -47,24 +45,7 @@ public static Schema Schema(Collection fields) { @Override public String toString() { - return StringUtils.join(fields, ","); - } - - public static Field StructField(QualifiedIdentifier name, Field ... fields) { - return StructField(name, Arrays.asList(fields)); - } - - public static Field StructField(QualifiedIdentifier name, Collection fields) { - return new Field(Type.Struct(fields), name); - } - - @VisibleForTesting - public static Field Field(String name, Type type) { - return new Field(type, QualifiedIdentifier.$(name)); - } - - public static Field Field(QualifiedIdentifier name, Type type) { - return new Field(type, name); + return StringUtils.join(fields, ", "); } @Override diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Struct.java b/tajo-common/src/main/java/org/apache/tajo/type/Record.java similarity index 88% rename from tajo-common/src/main/java/org/apache/tajo/type/Struct.java rename to tajo-common/src/main/java/org/apache/tajo/type/Record.java index 16544fe1e0..ee8e2e9cae 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Struct.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Record.java @@ -27,10 +27,10 @@ import java.util.List; import java.util.Objects; -public class Struct extends Type { +public class Record extends Type { private final ImmutableList memberTypes; - public Struct(Collection memberTypes) { + public Record(Collection memberTypes) { this.memberTypes = ImmutableList.copyOf(memberTypes); } @@ -53,7 +53,7 @@ public TajoDataTypes.Type baseType() { @Override public String toString() { - return "struct(" + StringUtils.join(memberTypes, ", ") + ")"; + return "record(" + StringUtils.join(memberTypes, ", ") + ")"; } @Override @@ -63,8 +63,8 @@ public int hashCode() { @Override public boolean equals(Object object) { - if (object instanceof Struct) { - Struct other = (Struct) object; + if (object instanceof Record) { + Record other = (Record) object; return memberTypes.equals(other.memberTypes); } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Type.java b/tajo-common/src/main/java/org/apache/tajo/type/Type.java index 47ee55f175..59be3e6ea7 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Type.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Type.java @@ -24,7 +24,7 @@ import java.util.Arrays; import java.util.Collection; -public abstract class Type { +public abstract class Type implements Cloneable { // No paramter types public static final Any Any = new Any(); @@ -98,16 +98,16 @@ public static Varchar Varchar(int len) { return new Varchar(len); } - public static Struct Struct(Collection types) { - return new Struct(types); + public static Record Struct(Collection types) { + return new Record(types); } public static Array Array(Type type) { return new Array(type); } - public static Struct Struct(Field... types) { - return new Struct(Arrays.asList(types)); + public static Record Struct(Field... types) { + return new Record(Arrays.asList(types)); } public static Map Map(Type keyType, Type valueType) { diff --git a/tajo-common/src/test/java/org/apache/tajo/schema/TestSchema.java b/tajo-common/src/test/java/org/apache/tajo/schema/TestSchema.java index 28f67abf9b..09b47cd630 100644 --- a/tajo-common/src/test/java/org/apache/tajo/schema/TestSchema.java +++ b/tajo-common/src/test/java/org/apache/tajo/schema/TestSchema.java @@ -20,9 +20,10 @@ import org.junit.Test; +import static org.apache.tajo.schema.Field.Field; +import static org.apache.tajo.schema.Field.Record; import static org.apache.tajo.schema.Identifier._; import static org.apache.tajo.schema.QualifiedIdentifier.$; -import static org.apache.tajo.schema.Schema.*; import static org.apache.tajo.schema.Schema.Schema; import static org.apache.tajo.type.Type.*; import static org.junit.Assert.assertEquals; @@ -31,30 +32,30 @@ public class TestSchema { @Test public final void testSchema1() { - Field struct1 = StructField($("f12"), Field($("f1"), Int8), Field($("f2"), Int4)); - Field struct2 = StructField($("f34"), Field($("f3"), Int8), Field($("f4"), Int4)); + Field struct1 = Record($("f12"), Field($("f1"), Int8), Field($("f2"), Int4)); + Field struct2 = Record($("f34"), Field($("f3"), Int8), Field($("f4"), Int4)); Schema schema = Schema(struct1, struct2); - assertEquals(schema.toString(), "f12 record (f1 int8,f2 int4),f34 record (f3 int8,f4 int4)"); + assertEquals("f12 record(f1 int8, f2 int4), f34 record(f3 int8, f4 int4)", schema.toString()); } @Test public final void testSchema2() { Field f1 = Field($("x"), Array(Int8)); Field f2 = Field($("y"), Int8); - Field f3 = StructField($("z"), Field($("z-1"), Time), Field($("z-2"), Array(Int8))); + Field f3 = Record($("z"), Field($("z-1"), Time), Field($("z-2"), Array(Int8))); Schema schema = Schema(f1, f2, f3); - assertEquals(schema.toString(), "x array,y int8,z record (z-1 time,z-2 array)"); + assertEquals("x array, y int8, z record(z-1 time, z-2 array)", schema.toString()); } @Test public final void testSchemaWithIdentifiers() { Field f1 = Field($("x", "y"), Array(Int8)); Field f2 = Field($(_("y"), _("B", true)), Int8); - Field f3 = StructField($("z"), Field($("z-1"), Time), Field($("z-2"), Array(Int8))); + Field f3 = Record($("z"), Field($("z-1"), Time), Field($("z-2"), Array(Int8))); Schema schema = Schema(f1, f2, f3); - assertEquals(schema.toString(), "x.y array,y.'B' int8,z record (z-1 time,z-2 array)"); + assertEquals("x.y array, y.B int8, z record(z-1 time, z-2 array)", schema.toString()); } } \ No newline at end of file diff --git a/tajo-common/src/test/java/org/apache/tajo/type/TestType.java b/tajo-common/src/test/java/org/apache/tajo/type/TestType.java index 8961a958d0..318f5447a8 100644 --- a/tajo-common/src/test/java/org/apache/tajo/type/TestType.java +++ b/tajo-common/src/test/java/org/apache/tajo/type/TestType.java @@ -21,8 +21,8 @@ import org.junit.Test; import static org.apache.tajo.common.TajoDataTypes.Type.*; +import static org.apache.tajo.schema.Field.Field; import static org.apache.tajo.schema.QualifiedIdentifier.$; - import static org.apache.tajo.type.Type.Array; import static org.apache.tajo.type.Type.Blob; import static org.apache.tajo.type.Type.Bool; @@ -71,12 +71,12 @@ public final void testPrimitiveTypes() { assertEquals(varchar.baseType(), VARCHAR); assertEquals(varchar.length(), 2); - Struct struct = Struct(Field("x", Int8), Field("y", Array(Float8))); - assertEquals(struct.baseType(), RECORD); - assertEquals(struct.field(0).baseType(), INT8); - assertEquals(struct.field(0).name(), $("x")); - assertEquals(struct.field(1).baseType(), ARRAY); - assertEquals(struct.field(1).name(), $("y")); + Record record = Struct(Field("x", Int8), Field("y", Array(Float8))); + assertEquals(record.baseType(), RECORD); + assertEquals(record.field(0).baseType(), INT8); + assertEquals(record.field(0).name(), $("x")); + assertEquals(record.field(1).baseType(), ARRAY); + assertEquals(record.field(1).name(), $("y")); Map map = Map(Int8, Array(Timestamp)); assertEquals(map.baseType(), MAP); @@ -112,8 +112,8 @@ public final void testToString() { Varchar varchar = Varchar(2); assertEquals("varchar(2)", varchar.toString()); - Struct struct = Struct(Field("x", Int8), Field("y", Array(Float8))); - assertEquals("struct(x int8, y array)", struct.toString()); + Record record = Struct(Field("x", Int8), Field("y", Array(Float8))); + assertEquals("record(x int8, y array)", record.toString()); Map map = Map(Int8, Array(Timestamp)); assertEquals("map>", map.toString()); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestDDLBuilder.java b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestDDLBuilder.java index 8790a381d2..61bcfed595 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestDDLBuilder.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tools/TestDDLBuilder.java @@ -24,6 +24,7 @@ import org.apache.tajo.catalog.partition.PartitionMethodDesc; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.common.TajoDataTypes; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.storage.StorageConstants; import org.apache.tajo.util.JavaResourceUtil; import org.junit.Test; @@ -117,7 +118,7 @@ public void testBuildColumn() throws Exception { }; for (String columnName : tobeUnquoted) { - assertFalse(CatalogUtil.isShouldBeQuoted(columnName)); + assertFalse(IdentifierUtil.isShouldBeQuoted(columnName)); } String [] quoted = { @@ -131,7 +132,7 @@ public void testBuildColumn() throws Exception { }; for (String columnName : quoted) { - assertTrue(CatalogUtil.isShouldBeQuoted(columnName)); + assertTrue(IdentifierUtil.isShouldBeQuoted(columnName)); } } } diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java index c210a24949..d92bc2f97e 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestTajoCli.java @@ -28,7 +28,6 @@ import org.apache.tajo.SessionVars; import org.apache.tajo.TajoTestingCluster; import org.apache.tajo.TpchTestBase; -import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.cli.tsql.commands.TajoShellCommand; import org.apache.tajo.client.ClientParameters; @@ -36,6 +35,7 @@ import org.apache.tajo.client.TajoClient; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.rpc.RpcConstants; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.storage.StorageUtil; import org.apache.tajo.storage.TablespaceManager; import org.apache.tajo.util.FileUtil; @@ -542,7 +542,7 @@ public void testResultRowNumWhenSelectingOnPartitionedTable() throws Exception { // TODO: This should be removed at TAJO-1891 @Test public void testAddPartitionNotimplementedException() throws Exception { - String tableName = CatalogUtil.normalizeIdentifier("testAddPartitionNotimplementedException"); + String tableName = IdentifierUtil.normalizeIdentifier("testAddPartitionNotimplementedException"); tajoCli.executeScript("create table " + tableName + " (col1 int4, col2 int4) partition by column(key float8)"); tajoCli.executeScript("alter table " + tableName + " add partition (key2 = 0.1)"); @@ -553,7 +553,7 @@ public void testAddPartitionNotimplementedException() throws Exception { // TODO: This should be added at TAJO-1891 public void testAlterTableAddDropPartition() throws Exception { - String tableName = CatalogUtil.normalizeIdentifier("testAlterTableAddPartition"); + String tableName = IdentifierUtil.normalizeIdentifier("testAlterTableAddPartition"); tajoCli.executeScript("create table " + tableName + " (col1 int4, col2 int4) partition by column(key float8)"); tajoCli.executeScript("alter table " + tableName + " add partition (key2 = 0.1)"); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/client/TestTajoClient.java b/tajo-core-tests/src/test/java/org/apache/tajo/client/TestTajoClient.java index 75a0ad0fe3..f75be76d93 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/client/TestTajoClient.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/client/TestTajoClient.java @@ -29,7 +29,6 @@ import org.apache.tajo.*; import org.apache.tajo.TajoProtos.QueryState; import org.apache.tajo.annotation.NotThreadSafe; -import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.FunctionDesc; import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.catalog.proto.CatalogProtos; @@ -43,6 +42,7 @@ import org.apache.tajo.ipc.ClientProtos.QueryInfoProto; import org.apache.tajo.ipc.ClientProtos.StageHistoryProto; import org.apache.tajo.rpc.NettyClientBase; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.storage.StorageConstants; import org.apache.tajo.storage.StorageUtil; import org.apache.tajo.util.CommonTestingUtil; @@ -91,7 +91,7 @@ private static Path writeTmpTable(String tableName) throws IOException { public final void testCreateAndDropDatabases() throws TajoException { int currentNum = client.getAllDatabaseNames().size(); - String prefix = CatalogUtil.normalizeIdentifier("testCreateDatabase_"); + String prefix = IdentifierUtil.normalizeIdentifier("testCreateDatabase_"); for (int i = 0; i < 10; i++) { // test allDatabaseNames assertEquals(currentNum + i, client.getAllDatabaseNames().size()); @@ -123,7 +123,7 @@ public final void testCurrentDatabase() throws IOException, TajoException, Inter int currentNum = client.getAllDatabaseNames().size(); assertEquals(TajoConstants.DEFAULT_DATABASE_NAME, client.getCurrentDatabase()); - String databaseName = CatalogUtil.normalizeIdentifier("testcurrentdatabase"); + String databaseName = IdentifierUtil.normalizeIdentifier("testcurrentdatabase"); client.createDatabase(databaseName); assertEquals(currentNum + 1, client.getAllDatabaseNames().size()); assertEquals(TajoConstants.DEFAULT_DATABASE_NAME, client.getCurrentDatabase()); @@ -153,7 +153,7 @@ public final void testSelectDatabaseToInvalidOne() throws IOException, TajoExcep @Test public final void testDropCurrentDatabase() throws IOException, TajoException, InterruptedException { int currentNum = client.getAllDatabaseNames().size(); - String databaseName = CatalogUtil.normalizeIdentifier("testdropcurrentdatabase"); + String databaseName = IdentifierUtil.normalizeIdentifier("testdropcurrentdatabase"); client.createDatabase(databaseName); client.selectDatabase(databaseName); assertEquals(databaseName, client.getCurrentDatabase()); @@ -227,7 +227,7 @@ public final void testKillQuery() throws IOException, TajoException, Interrupted @Test public final void testUpdateQuery() throws IOException, TajoException { - final String tableName = CatalogUtil.normalizeIdentifier("testUpdateQuery"); + final String tableName = IdentifierUtil.normalizeIdentifier("testUpdateQuery"); Path tablePath = writeTmpTable(tableName); assertFalse(client.existTable(tableName)); @@ -279,7 +279,7 @@ public final void testCreateAndPurgeExternalTable() throws IOException, TajoExce @Test public final void testCreateAndDropExternalTableByExecuteQuery() throws IOException, TajoException { TajoConf conf = cluster.getConfiguration(); - final String tableName = CatalogUtil.normalizeIdentifier("testCreateAndDropExternalTableByExecuteQuery"); + final String tableName = IdentifierUtil.normalizeIdentifier("testCreateAndDropExternalTableByExecuteQuery"); Path tablePath = writeTmpTable(tableName); assertFalse(client.existTable(tableName)); @@ -299,7 +299,7 @@ public final void testCreateAndDropExternalTableByExecuteQuery() throws IOExcept @Test public final void testCreateAndPurgeExternalTableByExecuteQuery() throws IOException, TajoException { TajoConf conf = cluster.getConfiguration(); - final String tableName = CatalogUtil.normalizeIdentifier("testCreateAndPurgeExternalTableByExecuteQuery"); + final String tableName = IdentifierUtil.normalizeIdentifier("testCreateAndPurgeExternalTableByExecuteQuery"); Path tablePath = writeTmpTable(tableName); assertFalse(client.existTable(tableName)); @@ -319,7 +319,7 @@ public final void testCreateAndPurgeExternalTableByExecuteQuery() throws IOExcep @Test public final void testCreateAndDropTableByExecuteQuery() throws IOException, TajoException { TajoConf conf = cluster.getConfiguration(); - final String tableName = CatalogUtil.normalizeIdentifier("testCreateAndDropTableByExecuteQuery"); + final String tableName = IdentifierUtil.normalizeIdentifier("testCreateAndDropTableByExecuteQuery"); assertFalse(client.existTable(tableName)); @@ -340,7 +340,7 @@ public final void testCreateAndDropTableByExecuteQuery() throws IOException, Taj @Test public final void testCreateAndPurgeTableByExecuteQuery() throws IOException, TajoException { TajoConf conf = cluster.getConfiguration(); - final String tableName = CatalogUtil.normalizeIdentifier("testCreateAndPurgeTableByExecuteQuery"); + final String tableName = IdentifierUtil.normalizeIdentifier("testCreateAndPurgeTableByExecuteQuery"); assertFalse(client.existTable(tableName)); @@ -360,7 +360,7 @@ public final void testCreateAndPurgeTableByExecuteQuery() throws IOException, Ta @Test public final void testDDLByExecuteQuery() throws IOException, TajoException { - final String tableName = CatalogUtil.normalizeIdentifier("testDDLByExecuteQuery"); + final String tableName = IdentifierUtil.normalizeIdentifier("testDDLByExecuteQuery"); Path tablePath = writeTmpTable(tableName); assertFalse(client.existTable(tableName)); @@ -393,7 +393,7 @@ public final void testGetTableList() throws IOException, TajoException { @Test public final void testGetTableDesc() throws IOException, TajoException { - final String tableName1 = CatalogUtil.normalizeIdentifier("table3"); + final String tableName1 = IdentifierUtil.normalizeIdentifier("table3"); Path tablePath = writeTmpTable(tableName1); LOG.error("Full path:" + tablePath.toUri().getRawPath()); FileSystem fs = tablePath.getFileSystem(conf); @@ -407,7 +407,7 @@ public final void testGetTableDesc() throws IOException, TajoException { TableDesc desc = client.getTableDesc(tableName1); assertNotNull(desc); - assertEquals(CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, tableName1), desc.getName()); + assertEquals(IdentifierUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, tableName1), desc.getName()); assertTrue(desc.getStats().getNumBytes() > 0); } @@ -572,7 +572,7 @@ public final void testFailCreateTablePartitionedOtherExceptColumn() throws IOExc @Test public final void testCreateAndDropTablePartitionedColumnByExecuteQuery() throws IOException, TajoException { TajoConf conf = cluster.getConfiguration(); - final String tableName = CatalogUtil.normalizeIdentifier("testCreateAndDropTablePartitionedColumnByExecuteQuery"); + final String tableName = IdentifierUtil.normalizeIdentifier("testCreateAndDropTablePartitionedColumnByExecuteQuery"); assertFalse(client.existTable(tableName)); @@ -614,7 +614,7 @@ public final void testGetFunctions() throws IOException, TajoException { @Test public final void testGetFinishedQueryList() throws SQLException, TajoException { - final String tableName = CatalogUtil.normalizeIdentifier("testGetFinishedQueryList"); + final String tableName = IdentifierUtil.normalizeIdentifier("testGetFinishedQueryList"); String sql = "create table " + tableName + " (deptname text, score int4)"; client.updateQuery(sql); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java index 00fa590055..950f368954 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/ExprTestBase.java @@ -52,6 +52,7 @@ import org.apache.tajo.plan.verifier.LogicalPlanVerifier; import org.apache.tajo.plan.verifier.PreLogicalPlanVerifier; import org.apache.tajo.plan.verifier.VerificationState; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.storage.LazyTuple; import org.apache.tajo.storage.TablespaceManager; import org.apache.tajo.storage.Tuple; @@ -194,20 +195,20 @@ public void testSimpleEval(OverridableConf context, String query, String [] expe public void testEval(Schema schema, String tableName, String csvTuple, String query, String [] expected) throws TajoException { - testEval(null, schema, tableName != null ? CatalogUtil.normalizeIdentifier(tableName) : null, csvTuple, query, + testEval(null, schema, tableName != null ? IdentifierUtil.normalizeIdentifier(tableName) : null, csvTuple, query, expected, ',', true); } public void testEval(OverridableConf context, Schema schema, String tableName, String csvTuple, String query, String [] expected) throws TajoException { - testEval(context, schema, tableName != null ? CatalogUtil.normalizeIdentifier(tableName) : null, csvTuple, + testEval(context, schema, tableName != null ? IdentifierUtil.normalizeIdentifier(tableName) : null, csvTuple, query, expected, ',', true); } public void testEval(Schema schema, String tableName, String csvTuple, String query, String [] expected, char delimiter, boolean condition) throws TajoException { - testEval(null, schema, tableName != null ? CatalogUtil.normalizeIdentifier(tableName) : null, csvTuple, + testEval(null, schema, tableName != null ? IdentifierUtil.normalizeIdentifier(tableName) : null, csvTuple, query, expected, delimiter, condition); } @@ -227,8 +228,8 @@ public void testEval(OverridableConf context, Schema schema, String tableName, S LazyTuple lazyTuple; VTuple vtuple = null; String qualifiedTableName = - CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, - tableName != null ? CatalogUtil.normalizeIdentifier(tableName) : null); + IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, + tableName != null ? IdentifierUtil.normalizeIdentifier(tableName) : null); Schema inputSchema = null; if (schema != null) { inputSchema = SchemaUtil.clone(schema); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java index 2846f22efc..807c7febb1 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/TestEvalTreeUtil.java @@ -43,6 +43,7 @@ import org.apache.tajo.plan.logical.GroupbyNode; import org.apache.tajo.plan.logical.NodeType; import org.apache.tajo.plan.nameresolver.NameResolvingMode; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.storage.TablespaceManager; import org.apache.tajo.storage.Tuple; import org.apache.tajo.util.CommonTestingUtil; @@ -105,7 +106,7 @@ public static void setUp() throws Exception { TableMeta meta = CatalogUtil.newTableMeta("TEXT"); TableDesc desc = new TableDesc( - CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "people"), schema, meta, + IdentifierUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "people"), schema, meta, CommonTestingUtil.getTestDir().toUri()); catalog.createTable(desc); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestEvalNodeToExprConverter.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestEvalNodeToExprConverter.java index 1cf4cf26ac..d968c6b221 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestEvalNodeToExprConverter.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestEvalNodeToExprConverter.java @@ -32,12 +32,12 @@ import org.apache.tajo.plan.LogicalPlan; import org.apache.tajo.plan.LogicalPlanner; import org.apache.tajo.plan.expr.AlgebraicUtil; -import org.apache.tajo.plan.expr.EvalNode; import org.apache.tajo.plan.logical.LogicalNode; import org.apache.tajo.plan.logical.NodeType; import org.apache.tajo.plan.logical.ScanNode; import org.apache.tajo.plan.util.EvalNodeToExprConverter; import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.session.Session; import org.apache.tajo.storage.TablespaceManager; import org.apache.tajo.util.CommonTestingUtil; @@ -81,7 +81,7 @@ public static void setUp() throws Exception { for (String table : tpchTables) { TableMeta m = CatalogUtil.newTableMeta("TEXT"); TableDesc d = CatalogUtil.newTableDesc( - CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, table), tpch.getSchema(table), m, + IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, table), tpch.getSchema(table), m, CommonTestingUtil.getTestDir()); catalog.createTable(d); } diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestJoinOrderAlgorithm.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestJoinOrderAlgorithm.java index bd9c5f9351..b2a9cd754b 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestJoinOrderAlgorithm.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestJoinOrderAlgorithm.java @@ -38,6 +38,7 @@ import org.apache.tajo.plan.logical.NodeType; import org.apache.tajo.plan.logical.ScanNode; import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.storage.TablespaceManager; import org.apache.tajo.unit.StorageUnit; import org.apache.tajo.util.CommonTestingUtil; @@ -90,19 +91,19 @@ public static void setUp() throws Exception { TableMeta meta = CatalogUtil.newTableMeta("TEXT"); TableDesc people = new TableDesc( - CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "employee"), schema, meta, + IdentifierUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "employee"), schema, meta, CommonTestingUtil.getTestDir().toUri()); catalog.createTable(people); TableDesc student = new TableDesc( - CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "dept"), schema2, "TEXT", new KeyValueSet(), + IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, "dept"), schema2, "TEXT", new KeyValueSet(), CommonTestingUtil.getTestDir().toUri()); catalog.createTable(student); TableDesc score = new TableDesc( - CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "score"), schema3, "TEXT", new KeyValueSet(), + IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, "score"), schema3, "TEXT", new KeyValueSet(), CommonTestingUtil.getTestDir().toUri()); catalog.createTable(score); @@ -122,7 +123,7 @@ public static void setUp() throws Exception { fs.create(tablePath); largeDept = new TableDesc( - CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "large_dept"+(i+1)), schema4, largeTableMeta, + IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, "large_dept"+(i+1)), schema4, largeTableMeta, tablePath.toUri()); largeTableStats = new TableStats(); largeTableStats.setNumBytes(StorageUnit.PB * (i+1)); //1 PB * i diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java index e57921d46a..87e36631f6 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalOptimizer.java @@ -34,6 +34,7 @@ import org.apache.tajo.plan.LogicalPlan; import org.apache.tajo.plan.LogicalPlanner; import org.apache.tajo.plan.logical.*; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.storage.TablespaceManager; import org.apache.tajo.util.CommonTestingUtil; import org.apache.tajo.util.KeyValueSet; @@ -84,19 +85,19 @@ public static void setUp() throws Exception { TableMeta meta = CatalogUtil.newTableMeta("TEXT"); TableDesc people = new TableDesc( - CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "employee"), schema, meta, + IdentifierUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "employee"), schema, meta, CommonTestingUtil.getTestDir().toUri()); catalog.createTable(people); TableDesc student = new TableDesc( - CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "dept"), schema2, "TEXT", new KeyValueSet(), + IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, "dept"), schema2, "TEXT", new KeyValueSet(), CommonTestingUtil.getTestDir().toUri()); catalog.createTable(student); TableDesc score = new TableDesc( - CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "score"), schema3, "TEXT", new KeyValueSet(), + IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, "score"), schema3, "TEXT", new KeyValueSet(), CommonTestingUtil.getTestDir().toUri()); catalog.createTable(score); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java index 6673725aaa..864f075ef9 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestLogicalPlanner.java @@ -48,6 +48,7 @@ import org.apache.tajo.plan.expr.*; import org.apache.tajo.plan.logical.*; import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.session.Session; import org.apache.tajo.storage.TablespaceManager; import org.apache.tajo.util.CommonTestingUtil; @@ -103,17 +104,17 @@ public static void setUp() throws Exception { TableMeta meta = CatalogUtil.newTableMeta("TEXT"); TableDesc people = new TableDesc( - CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "employee"), schema, meta, + IdentifierUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "employee"), schema, meta, CommonTestingUtil.getTestDir().toUri()); catalog.createTable(people); TableDesc student = new TableDesc( - CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "dept"), schema2, "TEXT", new KeyValueSet(), + IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, "dept"), schema2, "TEXT", new KeyValueSet(), CommonTestingUtil.getTestDir().toUri()); catalog.createTable(student); TableDesc score = new TableDesc( - CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "score"), schema3, "TEXT", new KeyValueSet(), + IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, "score"), schema3, "TEXT", new KeyValueSet(), CommonTestingUtil.getTestDir().toUri()); catalog.createTable(score); @@ -132,7 +133,7 @@ public static void setUp() throws Exception { for (String table : tpchTables) { TableMeta m = CatalogUtil.newTableMeta("TEXT"); TableDesc d = CatalogUtil.newTableDesc( - CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, table), tpch.getSchema(table), m, + IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, table), tpch.getSchema(table), m, CommonTestingUtil.getTestDir()); catalog.createTable(d); } @@ -195,7 +196,7 @@ public final void testSingleRelation() throws CloneNotSupportedException, TajoEx assertEquals(NodeType.SCAN, selNode.getChild().getType()); ScanNode scanNode = selNode.getChild(); - assertEquals(CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "employee"), scanNode.getTableName()); + assertEquals(IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, "employee"), scanNode.getTableName()); } public static void assertSchema(Schema expected, Schema schema) { @@ -241,10 +242,10 @@ public final void testImplicityJoinPlan() throws CloneNotSupportedException, Taj assertEquals(NodeType.SCAN, joinNode.getLeftChild().getType()); ScanNode leftNode = joinNode.getLeftChild(); - assertEquals(CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "employee"), leftNode.getTableName()); + assertEquals(IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, "employee"), leftNode.getTableName()); assertEquals(NodeType.SCAN, joinNode.getRightChild().getType()); ScanNode rightNode = joinNode.getRightChild(); - assertEquals(CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "dept"), rightNode.getTableName()); + assertEquals(IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, "dept"), rightNode.getTableName()); // three relations expr = sqlAnalyzer.parse(QUERIES[2]); @@ -269,18 +270,18 @@ public final void testImplicityJoinPlan() throws CloneNotSupportedException, Taj assertEquals(NodeType.SCAN, joinNode.getRightChild().getType()); ScanNode scan1 = joinNode.getRightChild(); - assertEquals(CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "score"), scan1.getTableName()); + assertEquals(IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, "score"), scan1.getTableName()); JoinNode leftNode2 = joinNode.getLeftChild(); assertEquals(NodeType.JOIN, leftNode2.getType()); assertEquals(NodeType.SCAN, leftNode2.getLeftChild().getType()); ScanNode leftScan = leftNode2.getLeftChild(); - assertEquals(CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "employee"), leftScan.getTableName()); + assertEquals(IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, "employee"), leftScan.getTableName()); assertEquals(NodeType.SCAN, leftNode2.getRightChild().getType()); ScanNode rightScan = leftNode2.getRightChild(); - assertEquals(CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "dept"), rightScan.getTableName()); + assertEquals(IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, "dept"), rightScan.getTableName()); } @@ -708,10 +709,10 @@ static void testQuery7(LogicalNode plan) { assertEquals(NodeType.SCAN, joinNode.getLeftChild().getType()); ScanNode leftNode = joinNode.getLeftChild(); - assertEquals(CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "dept"), leftNode.getTableName()); + assertEquals(IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, "dept"), leftNode.getTableName()); assertEquals(NodeType.SCAN, joinNode.getRightChild().getType()); ScanNode rightNode = joinNode.getRightChild(); - assertEquals(CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "score"), rightNode.getTableName()); + assertEquals(IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, "score"), rightNode.getTableName()); } @@ -755,10 +756,10 @@ public final void testOrderBy() throws CloneNotSupportedException, TajoException assertEquals(NodeType.SCAN, joinNode.getLeftChild().getType()); ScanNode leftNode = joinNode.getLeftChild(); - assertEquals(CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "dept"), leftNode.getTableName()); + assertEquals(IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, "dept"), leftNode.getTableName()); assertEquals(NodeType.SCAN, joinNode.getRightChild().getType()); ScanNode rightNode = joinNode.getRightChild(); - assertEquals(CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "score"), rightNode.getTableName()); + assertEquals(IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, "score"), rightNode.getTableName()); } @Test @@ -798,7 +799,7 @@ public final void testSPJPush() throws CloneNotSupportedException, TajoException SelectionNode selNode = projNode.getChild(); assertEquals(NodeType.SCAN, selNode.getChild().getType()); ScanNode scanNode = selNode.getChild(); - assertEquals(CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "employee"), scanNode.getTableName()); + assertEquals(IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, "employee"), scanNode.getTableName()); } @@ -879,7 +880,7 @@ public final void testCreateIndexNode() throws TajoException { assertEquals(NodeType.SCAN, selNode.getChild().getType()); ScanNode scanNode = selNode.getChild(); - assertEquals(CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "employee"), scanNode.getTableName()); + assertEquals(IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, "employee"), scanNode.getTableName()); } @Test @@ -902,7 +903,7 @@ public final void testAsterisk() throws CloneNotSupportedException, TajoExceptio assertEquals(NodeType.SCAN, selNode.getChild().getType()); ScanNode scanNode = selNode.getChild(); - assertEquals(CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "employee"), scanNode.getTableName()); + assertEquals(IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, "employee"), scanNode.getTableName()); } static final String ALIAS [] = { @@ -1117,7 +1118,7 @@ public final void testInsertInto0() throws TajoException { InsertNode insertNode = getInsertNode(plan); assertFalse(insertNode.isOverwrite()); assertTrue(insertNode.hasTargetTable()); - assertEquals(CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "score"), insertNode.getTableName()); + assertEquals(IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, "score"), insertNode.getTableName()); } @Test @@ -1129,7 +1130,7 @@ public final void testInsertInto1() throws TajoException { assertEquals(1, plan.getQueryBlocks().size()); InsertNode insertNode = getInsertNode(plan); assertFalse(insertNode.isOverwrite()); - assertEquals(CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "score"), insertNode.getTableName()); + assertEquals(IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, "score"), insertNode.getTableName()); } @Test @@ -1141,7 +1142,7 @@ public final void testInsertInto2() throws TajoException { assertEquals(1, plan.getQueryBlocks().size()); InsertNode insertNode = getInsertNode(plan); assertFalse(insertNode.isOverwrite()); - assertEquals(CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "employee"), insertNode.getTableName()); + assertEquals(IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, "employee"), insertNode.getTableName()); assertTrue(insertNode.hasTargetSchema()); assertEquals(insertNode.getTargetSchema().getColumn(0).getSimpleName(), "name"); assertEquals(insertNode.getTargetSchema().getColumn(1).getSimpleName(), "deptname"); @@ -1169,7 +1170,7 @@ public final void testInsertInto4() throws TajoException { InsertNode insertNode = getInsertNode(plan); assertTrue(insertNode.isOverwrite()); assertTrue(insertNode.hasTargetTable()); - assertEquals(CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "employee"), insertNode.getTableName()); + assertEquals(IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, "employee"), insertNode.getTableName()); assertTrue(insertNode.hasTargetSchema()); assertEquals(insertNode.getTargetSchema().getColumn(0).getSimpleName(), "name"); assertEquals(insertNode.getTargetSchema().getColumn(1).getSimpleName(), "deptname"); @@ -1237,8 +1238,8 @@ public final void testAlterTableRepairPartiton() throws TajoException { // TODO: This should be added at TAJO-1891 public final void testAddPartitionAndDropPartition() throws TajoException { - String tableName = CatalogUtil.normalizeIdentifier("partitioned_table"); - String qualifiedTableName = CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, tableName); + String tableName = IdentifierUtil.normalizeIdentifier("partitioned_table"); + String qualifiedTableName = IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, tableName); Schema schema = SchemaBuilder.builder() .add("id", Type.INT4) diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java index 92557f1140..2781991712 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/TestPlannerUtil.java @@ -39,6 +39,7 @@ import org.apache.tajo.plan.expr.*; import org.apache.tajo.plan.logical.*; import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.storage.TablespaceManager; import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.TupleComparator; @@ -87,19 +88,19 @@ public static void setUp() throws Exception { TableMeta meta = CatalogUtil.newTableMeta("TEXT"); TableDesc people = new TableDesc( - CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "employee"), schema, meta, + IdentifierUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "employee"), schema, meta, CommonTestingUtil.getTestDir().toUri()); catalog.createTable(people); TableDesc student = new TableDesc( - CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "dept"), schema2, "TEXT", + IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, "dept"), schema2, "TEXT", new KeyValueSet(), CommonTestingUtil.getTestDir().toUri()); catalog.createTable(student); TableDesc score = new TableDesc( - CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "score"), schema3, "TEXT", + IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, "score"), schema3, "TEXT", new KeyValueSet(), CommonTestingUtil.getTestDir().toUri()); catalog.createTable(score); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java index 1d2e6b5d8e..fd73a44434 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterHashJoinExec.java @@ -38,6 +38,7 @@ import org.apache.tajo.plan.logical.LogicalNode; import org.apache.tajo.plan.logical.NodeType; import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.storage.Appender; import org.apache.tajo.storage.FileTablespace; import org.apache.tajo.storage.TablespaceManager; @@ -72,10 +73,10 @@ public class TestFullOuterHashJoinExec { private TableDesc emp3; private TableDesc phone3; - private final String DEP3_NAME = CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "dep3"); - private final String JOB3_NAME = CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "job3"); - private final String EMP3_NAME = CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "emp3"); - private final String PHONE3_NAME = CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "phone3"); + private final String DEP3_NAME = IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, "dep3"); + private final String JOB3_NAME = IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, "job3"); + private final String EMP3_NAME = IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, "emp3"); + private final String PHONE3_NAME = IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, "phone3"); @Before public void setUp() throws Exception { diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java index 0fb7bbb6b8..af07decbc2 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestFullOuterMergeJoinExec.java @@ -39,6 +39,7 @@ import org.apache.tajo.plan.logical.LogicalNode; import org.apache.tajo.plan.logical.NodeType; import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.storage.Appender; import org.apache.tajo.storage.FileTablespace; import org.apache.tajo.storage.TablespaceManager; @@ -74,11 +75,11 @@ public class TestFullOuterMergeJoinExec { private TableDesc emp3; private TableDesc phone3; - private final String DEP3_NAME = CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "dep3"); - private final String DEP4_NAME = CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "dep4"); - private final String JOB3_NAME = CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "job3"); - private final String EMP3_NAME = CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "emp3"); - private final String PHONE3_NAME = CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "phone3"); + private final String DEP3_NAME = IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, "dep3"); + private final String DEP4_NAME = IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, "dep4"); + private final String JOB3_NAME = IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, "job3"); + private final String EMP3_NAME = IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, "emp3"); + private final String PHONE3_NAME = IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, "phone3"); @Before public void setUp() throws Exception { diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java index c30806589e..05258e7bef 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestLeftOuterHashJoinExec.java @@ -38,6 +38,7 @@ import org.apache.tajo.plan.logical.LogicalNode; import org.apache.tajo.plan.logical.NodeType; import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.storage.*; import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.util.CommonTestingUtil; @@ -70,10 +71,10 @@ public class TestLeftOuterHashJoinExec { private TableDesc emp3; private TableDesc phone3; - private final String DEP3_NAME = CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "dep3"); - private final String JOB3_NAME = CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "job3"); - private final String EMP3_NAME = CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "emp3"); - private final String PHONE3_NAME = CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "phone3"); + private final String DEP3_NAME = IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, "dep3"); + private final String JOB3_NAME = IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, "job3"); + private final String EMP3_NAME = IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, "emp3"); + private final String PHONE3_NAME = IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, "phone3"); @Before public void setUp() throws Exception { diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java index 694193d8d5..04926c06a6 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestPhysicalPlanner.java @@ -49,6 +49,7 @@ import org.apache.tajo.plan.logical.*; import org.apache.tajo.plan.serder.PlanProto.ShuffleType; import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.session.Session; import org.apache.tajo.storage.*; import org.apache.tajo.storage.fragment.FileFragment; @@ -133,7 +134,7 @@ public static void setUp() throws Exception { appender.close(); employee = new TableDesc( - CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "employee"), employeeSchema, employeeMeta, + IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, "employee"), employeeSchema, employeeMeta, employeePath.toUri()); catalog.createTable(employee); @@ -142,7 +143,7 @@ public static void setUp() throws Exception { appender = sm.getAppender(scoreMeta, scoreSchema, scorePath); appender.init(); score = new TableDesc( - CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "score"), scoreSchema, scoreMeta, + IdentifierUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "score"), scoreSchema, scoreMeta, scorePath.toUri()); tuple = new VTuple(scoreSchema.size()); int m = 0; @@ -186,7 +187,7 @@ public static void createLargeScoreTable() throws IOException, TajoException { appender.enableStats(); appender.init(); largeScore = new TableDesc( - CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "score_large"), scoreSchmea, scoreLargeMeta, + IdentifierUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "score_large"), scoreSchmea, scoreLargeMeta, scoreLargePath.toUri()); VTuple tuple = new VTuple(scoreSchmea.size()); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java index eeb179fab4..0644c7e0c8 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestProgressExternalSortExec.java @@ -37,6 +37,7 @@ import org.apache.tajo.plan.LogicalPlan; import org.apache.tajo.plan.LogicalPlanner; import org.apache.tajo.plan.logical.LogicalNode; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.storage.*; import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.unit.StorageUnit; @@ -106,7 +107,7 @@ public void setUp() throws Exception { testDataStats = appender.getStats(); employee = new TableDesc( - CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "employee"), schema, employeeMeta, + IdentifierUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "employee"), schema, employeeMeta, employeePath.toUri()); catalog.createTable(employee); analyzer = new SQLAnalyzer(); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java index 83d26c7052..df8b43855d 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterHashJoinExec.java @@ -35,6 +35,7 @@ import org.apache.tajo.parser.sql.SQLAnalyzer; import org.apache.tajo.plan.LogicalPlanner; import org.apache.tajo.plan.logical.LogicalNode; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.storage.*; import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.util.CommonTestingUtil; @@ -65,9 +66,9 @@ public class TestRightOuterHashJoinExec { private TableDesc job3; private TableDesc emp3; - private final String DEP3_NAME = CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "dep3"); - private final String JOB3_NAME = CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "job3"); - private final String EMP3_NAME = CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "emp3"); + private final String DEP3_NAME = IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, "dep3"); + private final String JOB3_NAME = IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, "job3"); + private final String EMP3_NAME = IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, "emp3"); @Before public void setUp() throws Exception { diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java index 4bfe0f7bbb..4d3c1d0e92 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestRightOuterMergeJoinExec.java @@ -38,6 +38,7 @@ import org.apache.tajo.plan.logical.LogicalNode; import org.apache.tajo.plan.logical.NodeType; import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.storage.Appender; import org.apache.tajo.storage.FileTablespace; import org.apache.tajo.storage.TablespaceManager; @@ -74,11 +75,11 @@ public class TestRightOuterMergeJoinExec { private TableDesc emp3; private TableDesc phone3; - private final String DEP3_NAME = CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "dep3"); - private final String DEP4_NAME = CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "dep4"); - private final String JOB3_NAME = CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "job3"); - private final String EMP3_NAME = CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "emp3"); - private final String PHONE3_NAME = CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, "phone3"); + private final String DEP3_NAME = IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, "dep3"); + private final String DEP4_NAME = IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, "dep4"); + private final String JOB3_NAME = IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, "job3"); + private final String EMP3_NAME = IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, "emp3"); + private final String PHONE3_NAME = IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, "phone3"); @Before public void setUp() throws Exception { diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java index 07b392a081..7dc05fc175 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/planner/physical/TestSortExec.java @@ -43,6 +43,7 @@ import org.apache.tajo.plan.LogicalPlanner; import org.apache.tajo.plan.logical.LogicalNode; import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.storage.*; import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.util.CommonTestingUtil; @@ -107,7 +108,7 @@ public static void setUp() throws Exception { appender.close(); TableDesc desc = new TableDesc( - CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "employee"), schema, employeeMeta, + IdentifierUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "employee"), schema, employeeMeta, tablePath.toUri()); catalog.createTable(desc); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java index 4c9f367223..43d951dfd3 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestAlterTable.java @@ -22,11 +22,11 @@ import org.apache.hadoop.fs.Path; import org.apache.tajo.IntegrationTest; import org.apache.tajo.QueryTestCaseBase; -import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.exception.*; +import org.apache.tajo.schema.IdentifierUtil; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -34,7 +34,6 @@ import java.io.File; import java.sql.ResultSet; -import java.sql.SQLException; import java.util.List; import static org.junit.Assert.*; @@ -92,7 +91,7 @@ public final void testAlterTableSetProperty() throws Exception { public final void testAlterTableAddPartition() throws Exception { executeDDL("create_partitioned_table.sql", null); - String tableName = CatalogUtil.buildFQName("TestAlterTable", "partitioned_table"); + String tableName = IdentifierUtil.buildFQName("TestAlterTable", "partitioned_table"); assertTrue(catalog.existsTable(tableName)); TableDesc retrieved = catalog.getTableDesc(tableName); @@ -136,7 +135,7 @@ public final void testAlterTableRepairPartition() throws Exception { executeDDL("create_partitioned_table2.sql", null); String simpleTableName = "partitioned_table2"; - String tableName = CatalogUtil.buildFQName(getCurrentDatabase(), simpleTableName); + String tableName = IdentifierUtil.buildFQName(getCurrentDatabase(), simpleTableName); assertTrue(catalog.existsTable(tableName)); TableDesc tableDesc = catalog.getTableDesc(tableName); @@ -216,7 +215,7 @@ public final void testAlterTableRepairPartition() throws Exception { public final void testRepairPartitionWithDatabaseNameIncludeTableName() throws Exception { String databaseName = "test_repair_partition"; String tableName = "part"; - String canonicalTableName = CatalogUtil.getCanonicalTableName(databaseName, tableName); + String canonicalTableName = IdentifierUtil.getCanonicalTableName(databaseName, tableName); executeString("create database " + databaseName).close(); executeString("create table " + canonicalTableName + "(col1 int4, col2 int4) partition by column(key float8) " @@ -261,7 +260,7 @@ public final void testRepairPartitionWithDatabaseNameIncludeTableName() throws E public void testRepairPartitionWithAbnormalDirectories() throws Exception { String databaseName = getCurrentDatabase().toLowerCase(); String tableName = "testRepairPartitionWithAbnormalDirectories".toLowerCase(); - String canonicalTableName = CatalogUtil.getCanonicalTableName(databaseName, tableName); + String canonicalTableName = IdentifierUtil.getCanonicalTableName(databaseName, tableName); executeString("create table " + canonicalTableName + "(col1 int4, col2 int4) partition by column(key float8) " + " as select l_orderkey, l_partkey, l_quantity from default.lineitem").close(); @@ -316,7 +315,7 @@ public void testRepairPartitionWithAbnormalDirectories() throws Exception { public void testRepairPartitionWithDatePartitionColumn() throws Exception { String databaseName = getCurrentDatabase().toLowerCase(); String tableName = "testRepairPartitionWithDatePartitionColumn".toLowerCase(); - String canonicalTableName = CatalogUtil.getCanonicalTableName(databaseName, tableName); + String canonicalTableName = IdentifierUtil.getCanonicalTableName(databaseName, tableName); executeString( "create table " + canonicalTableName + "(col1 int4, col2 int4) partition by column(key date) " @@ -360,7 +359,7 @@ public void testRepairPartitionWithDatePartitionColumn() throws Exception { public void testRepairPartitionWithTimestampPartitionColumn() throws Exception { String databaseName = getCurrentDatabase().toLowerCase(); String tableName = "testRepairPartitionWithTimestampPartitionColumn".toLowerCase(); - String canonicalTableName = CatalogUtil.getCanonicalTableName(databaseName, tableName); + String canonicalTableName = IdentifierUtil.getCanonicalTableName(databaseName, tableName); executeString( "create table " + canonicalTableName + "(col1 int4, col2 int4) partition by column(key timestamp) " @@ -404,7 +403,7 @@ public void testRepairPartitionWithTimestampPartitionColumn() throws Exception public void testRepairPartitionWithTimesPartitionColumn() throws Exception { String databaseName = getCurrentDatabase().toLowerCase(); String tableName = "testRepairPartitionWithTimesPartitionColumn".toLowerCase(); - String canonicalTableName = CatalogUtil.getCanonicalTableName(databaseName, tableName); + String canonicalTableName = IdentifierUtil.getCanonicalTableName(databaseName, tableName); executeString( "create table " + canonicalTableName + "(col1 int4, col2 int4) partition by column(key time) " @@ -454,7 +453,7 @@ public void testRepairPartitionWithTimesPartitionColumn() throws Exception { public void testRepairPartitionWithMutiplePartitionColumn() throws Exception { String databaseName = getCurrentDatabase().toLowerCase(); String tableName = "testRepairPartitionWithMutiplePartitionColumn".toLowerCase(); - String canonicalTableName = CatalogUtil.getCanonicalTableName(databaseName, tableName); + String canonicalTableName = IdentifierUtil.getCanonicalTableName(databaseName, tableName); executeString("create table " + canonicalTableName + " (col4 text) " + " partition by column(col1 int4, col2 int4, col3 float8) as select l_returnflag, l_orderkey, l_partkey, " + @@ -512,7 +511,7 @@ private void verifyPartitionCount(String databaseName, String tableName, int exp private void dropPartitions(String databaseName, String tableName, List colums) throws Exception { - String canonicalTableName = CatalogUtil.getCanonicalTableName(databaseName, tableName); + String canonicalTableName = IdentifierUtil.getCanonicalTableName(databaseName, tableName); List partitions = catalog.getPartitionsOfTable(databaseName, tableName); StringBuilder sb = new StringBuilder(); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java index 8e0700b2c8..d80d3b2aa6 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestCTASQuery.java @@ -27,10 +27,10 @@ import org.apache.tajo.TajoConstants; import org.apache.tajo.TajoTestingCluster; import org.apache.tajo.catalog.CatalogService; -import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.catalog.partition.PartitionMethodDesc; import org.apache.tajo.catalog.proto.CatalogProtos; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.storage.StorageConstants; import org.apache.tajo.util.KeyValueSet; import org.junit.Test; @@ -40,7 +40,7 @@ import java.util.Map; import static org.apache.tajo.TajoConstants.DEFAULT_DATABASE_NAME; -import static org.apache.tajo.catalog.CatalogUtil.buildFQName; +import static org.apache.tajo.schema.IdentifierUtil.buildFQName; import static org.junit.Assert.*; @@ -59,7 +59,7 @@ public final void testCtasWithoutTableDefinition() throws Exception { ResultSet res = executeQuery(); res.close(); - String tableName = CatalogUtil.normalizeIdentifier("testCtasWithoutTableDefinition"); + String tableName = IdentifierUtil.normalizeIdentifier("testCtasWithoutTableDefinition"); CatalogService catalog = testBase.getTestingCluster().getMaster().getCatalog(); String qualifiedTableName = buildFQName(DEFAULT_DATABASE_NAME, tableName); TableDesc desc = catalog.getTableDesc(qualifiedTableName); @@ -103,7 +103,7 @@ public final void testCtasWithColumnedPartition() throws Exception { ResultSet res = executeQuery(); res.close(); - String tableName = CatalogUtil.normalizeIdentifier("testCtasWithColumnedPartition"); + String tableName = IdentifierUtil.normalizeIdentifier("testCtasWithColumnedPartition"); TajoTestingCluster cluster = testBase.getTestingCluster(); CatalogService catalog = cluster.getMaster().getCatalog(); @@ -205,7 +205,7 @@ public final void testCtasWithMultipleUnions() throws Exception { assertEquals(expected, actual); - TableDesc desc = client.getTableDesc(CatalogUtil.normalizeIdentifier(res2.getMetaData().getTableName(1))); + TableDesc desc = client.getTableDesc(IdentifierUtil.normalizeIdentifier(res2.getMetaData().getTableName(1))); assertNotNull(desc); } @@ -218,7 +218,7 @@ public final void testCtasWithDataFormat() throws Exception { resultSetToString(res2); res2.close(); - TableDesc desc = client.getTableDesc(CatalogUtil.normalizeIdentifier(res2.getMetaData().getTableName(1))); + TableDesc desc = client.getTableDesc(IdentifierUtil.normalizeIdentifier(res2.getMetaData().getTableName(1))); assertNotNull(desc); assertTrue("RCFILE".equalsIgnoreCase(desc.getMeta().getDataFormat())); } @@ -232,7 +232,7 @@ public final void testCtasWithTextFile() throws Exception { resultSetToString(res2); res2.close(); - TableDesc desc = client.getTableDesc(CatalogUtil.normalizeIdentifier(res2.getMetaData().getTableName(1))); + TableDesc desc = client.getTableDesc(IdentifierUtil.normalizeIdentifier(res2.getMetaData().getTableName(1))); assertNotNull(desc); assertTrue("TEXT".equalsIgnoreCase(desc.getMeta().getDataFormat())); } @@ -246,7 +246,7 @@ public final void testCtasWithOptions() throws Exception { resultSetToString(res2); res2.close(); - TableDesc desc = client.getTableDesc(CatalogUtil.normalizeIdentifier(res2.getMetaData().getTableName(1))); + TableDesc desc = client.getTableDesc(IdentifierUtil.normalizeIdentifier(res2.getMetaData().getTableName(1))); assertNotNull(desc); assertTrue("TEXT".equalsIgnoreCase(desc.getMeta().getDataFormat())); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestCreateDatabase.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestCreateDatabase.java index 453c174ca3..d1713c2fff 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestCreateDatabase.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestCreateDatabase.java @@ -20,7 +20,7 @@ import org.apache.tajo.IntegrationTest; import org.apache.tajo.QueryTestCaseBase; -import org.apache.tajo.catalog.CatalogUtil; +import org.apache.tajo.schema.IdentifierUtil; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -31,7 +31,7 @@ public class TestCreateDatabase extends QueryTestCaseBase { @Test public final void testCreateAndDropDatabase() throws Exception { - String databaseName = CatalogUtil.normalizeIdentifier("testCreateAndDropDatabase"); + String databaseName = IdentifierUtil.normalizeIdentifier("testCreateAndDropDatabase"); ResultSet res = null; try { @@ -46,7 +46,7 @@ public final void testCreateAndDropDatabase() throws Exception { @Test public final void testCreateIfNotExists() throws Exception { - String databaseName = CatalogUtil.normalizeIdentifier("testCreateIfNotExists"); + String databaseName = IdentifierUtil.normalizeIdentifier("testCreateIfNotExists"); assertDatabaseNotExists(databaseName); executeString("CREATE DATABASE " + databaseName + ";").close(); @@ -61,7 +61,7 @@ public final void testCreateIfNotExists() throws Exception { @Test public final void testDropIfExists() throws Exception { - String databaseName = CatalogUtil.normalizeIdentifier("testDropIfExists"); + String databaseName = IdentifierUtil.normalizeIdentifier("testDropIfExists"); assertDatabaseNotExists(databaseName); executeString("CREATE DATABASE " + databaseName + ";").close(); assertDatabaseExists(databaseName); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java index b7196bdfc9..3978db019f 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java @@ -24,6 +24,7 @@ import org.apache.tajo.catalog.*; import org.apache.tajo.catalog.partition.PartitionMethodDesc; import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.storage.StorageUtil; import org.apache.tajo.util.KeyValueSet; import org.junit.Test; @@ -101,9 +102,9 @@ private final void assertPathOfCreatedTable(final String databaseName, final String newTableName, String createTableStmt) throws Exception { // create one table - executeString("CREATE DATABASE " + CatalogUtil.denormalizeIdentifier(databaseName)).close(); - getClient().existDatabase(CatalogUtil.denormalizeIdentifier(databaseName)); - final String oldFQTableName = CatalogUtil.buildFQName(databaseName, originalTableName); + executeString("CREATE DATABASE " + IdentifierUtil.denormalizeIdentifier(databaseName)).close(); + getClient().existDatabase(IdentifierUtil.denormalizeIdentifier(databaseName)); + final String oldFQTableName = IdentifierUtil.buildFQName(databaseName, originalTableName); ResultSet res = executeString(createTableStmt); res.close(); @@ -119,11 +120,11 @@ private final void assertPathOfCreatedTable(final String databaseName, new Path(oldTableDesc.getUri())); // Rename - client.executeQuery("ALTER TABLE " + CatalogUtil.denormalizeIdentifier(oldFQTableName) - + " RENAME to " + CatalogUtil.denormalizeIdentifier(newTableName)); + client.executeQuery("ALTER TABLE " + IdentifierUtil.denormalizeIdentifier(oldFQTableName) + + " RENAME to " + IdentifierUtil.denormalizeIdentifier(newTableName)); // checking the existence of the new table directory and validating the path - final String newFQTableName = CatalogUtil.buildFQName(databaseName, newTableName); + final String newFQTableName = IdentifierUtil.buildFQName(databaseName, newTableName); TableDesc newTableDesc = client.getTableDesc(newFQTableName); assertTrue(fs.exists(new Path(newTableDesc.getUri()))); assertEquals(StorageUtil.concatPath(warehouseDir, databaseName, newTableName), new Path(newTableDesc.getUri())); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestInsertQuery.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestInsertQuery.java index 05c30dfa72..5034e28e55 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestInsertQuery.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestInsertQuery.java @@ -27,8 +27,8 @@ import org.apache.tajo.IntegrationTest; import org.apache.tajo.QueryTestCaseBase; import org.apache.tajo.catalog.CatalogService; -import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.TableDesc; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.util.CommonTestingUtil; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -177,7 +177,7 @@ public final void assertTestInsertIntoLocation(Path path) throws Exception { @Test public final void testInsertIntoPartitionedTable() throws Exception { - String tableName = CatalogUtil.normalizeIdentifier("testInsertIntoPartitionedTable"); + String tableName = IdentifierUtil.normalizeIdentifier("testInsertIntoPartitionedTable"); executeString("create table " + tableName + " (n_name TEXT, n_regionkey INT4)" + "USING csv PARTITION by column(n_nationkey INT4)" ).close(); @@ -334,7 +334,7 @@ public final void testInsertOverwriteWithTargetColumns() throws Exception { assertEquals(5, desc.getStats().getNumRows().intValue()); } - res = executeString("select * from " + CatalogUtil.denormalizeIdentifier(getCurrentDatabase()) + ".table1"); + res = executeString("select * from " + IdentifierUtil.denormalizeIdentifier(getCurrentDatabase()) + ".table1"); assertTrue(res.next()); assertEquals(1, res.getLong(1)); @@ -414,7 +414,7 @@ public final void testInsertOverwriteWithAsteriskAndMore() throws Exception { @Test public final void testInsertOverwriteIntoSelect() throws Exception { - String tableName = CatalogUtil.normalizeIdentifier("insertoverwriteintoselect"); + String tableName = IdentifierUtil.normalizeIdentifier("insertoverwriteintoselect"); ResultSet res = executeString("create table " + tableName + " as select l_orderkey from default.lineitem"); assertFalse(res.next()); res.close(); @@ -442,7 +442,7 @@ public final void testInsertOverwriteIntoSelect() throws Exception { @Test public final void testInsertOverwriteCapitalTableName() throws Exception { - String tableName = CatalogUtil.normalizeIdentifier("testInsertOverwriteCapitalTableName"); + String tableName = IdentifierUtil.normalizeIdentifier("testInsertOverwriteCapitalTableName"); ResultSet res = executeString("create table " + tableName + " as select * from default.lineitem"); res.close(); @@ -469,7 +469,7 @@ public final void testInsertOverwriteLocation() throws Exception { @Test public final void testInsertOverwriteWithCompression() throws Exception { - String tableName = CatalogUtil.normalizeIdentifier("testInsertOverwriteWithCompression"); + String tableName = IdentifierUtil.normalizeIdentifier("testInsertOverwriteWithCompression"); ResultSet res = executeFile("testInsertOverwriteWithCompression_ddl.sql"); res.close(); @@ -628,7 +628,7 @@ public final void testInsertOverwriteWithDatabase() throws Exception { } public final void testInsertOverwriteAllValues(String rawTableName, String query) throws Exception { - String tableName = CatalogUtil.normalizeIdentifier(rawTableName); + String tableName = IdentifierUtil.normalizeIdentifier(rawTableName); ResultSet res = executeString("create table " + tableName +" (col1 int4, col2 float4, col3 text)"); res.close(); CatalogService catalog = testingCluster.getMaster().getCatalog(); @@ -668,7 +668,7 @@ public final void testInsertOverwriteAllValues2() throws Exception { } public final void testInsertOverwriteSomeValues(String rawTableName, String sql) throws Exception { - String tableName = CatalogUtil.normalizeIdentifier(rawTableName); + String tableName = IdentifierUtil.normalizeIdentifier(rawTableName); ResultSet res = executeString("create table " + tableName +" (col1 int4, col2 float4, col3 text)"); res.close(); CatalogService catalog = testingCluster.getMaster().getCatalog(); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestJoinOnPartitionedTables.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestJoinOnPartitionedTables.java index 0d110d1b9d..042c1f1acf 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestJoinOnPartitionedTables.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestJoinOnPartitionedTables.java @@ -23,7 +23,7 @@ import org.apache.tajo.TajoTestingCluster; import org.apache.tajo.annotation.NotThreadSafe; import org.apache.tajo.catalog.CatalogService; -import org.apache.tajo.catalog.CatalogUtil; +import org.apache.tajo.schema.IdentifierUtil; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -157,7 +157,7 @@ public void testMultiplePartitionedBroadcastDataFileWithZeroLength2() throws Exc @Test public final void testCasebyCase1() throws Exception { // Left outer join with a small table and a large partition table which not matched any partition path. - String tableName = CatalogUtil.normalizeIdentifier("largePartitionedTable"); + String tableName = IdentifierUtil.normalizeIdentifier("largePartitionedTable"); executeString( "create table " + tableName + " (l_partkey int4, l_suppkey int4, l_linenumber int4, \n" + "l_quantity float8, l_extendedprice float8, l_discount float8, l_tax float8, \n" + @@ -195,7 +195,7 @@ public final void testCasebyCase1() throws Exception { // TODO: This test should be reverted after resolving TAJO-1600 // @Test public final void testBroadcastMultiColumnPartitionTable() throws Exception { - String tableName = CatalogUtil.normalizeIdentifier("testBroadcastMultiColumnPartitionTable"); + String tableName = IdentifierUtil.normalizeIdentifier("testBroadcastMultiColumnPartitionTable"); ResultSet res = testBase.execute( "create table " + tableName + " (col1 int4, col2 float4) partition by column(col3 text, col4 text) "); res.close(); @@ -223,7 +223,7 @@ public final void testBroadcastMultiColumnPartitionTable() throws Exception { @Test public final void testSelfJoin() throws Exception { - String tableName = CatalogUtil.normalizeIdentifier("paritioned_nation"); + String tableName = IdentifierUtil.normalizeIdentifier("paritioned_nation"); ResultSet res = executeString( "create table " + tableName + " (n_name text," + " n_comment text, n_regionkey int8) USING text " @@ -281,7 +281,7 @@ and a.o_orderstatus in ('F') Because of the where condition[where a.o_orderdate='1995-02-21 and a.o_orderstatus in ('F')], orders_partition table aliased a is small and broadcast target. */ - String tableName = CatalogUtil.normalizeIdentifier("partitioned_orders"); + String tableName = IdentifierUtil.normalizeIdentifier("partitioned_orders"); ResultSet res = executeString( "create table " + tableName + " (o_orderkey INT8, o_custkey INT8, o_totalprice FLOAT8, o_orderpriority TEXT,\n" + "o_clerk TEXT, o_shippriority INT4, o_comment TEXT) USING TEXT WITH ('text.delimiter'='|')\n" + diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestSimpleQuery.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestSimpleQuery.java index f62bb2378b..fafc75e891 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestSimpleQuery.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestSimpleQuery.java @@ -19,7 +19,6 @@ package org.apache.tajo.engine.query; import org.apache.tajo.*; -import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.client.QueryStatus; import org.apache.tajo.client.TajoClientUtil; import org.apache.tajo.exception.QueryNotFoundException; @@ -30,6 +29,7 @@ import org.apache.tajo.plan.LogicalPlan; import org.apache.tajo.plan.logical.NodeType; import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.session.Session; import org.apache.tajo.util.StringUtils; import org.junit.AfterClass; @@ -84,12 +84,12 @@ public static void tearDownClass() throws Exception { } private static void createTestTable() throws Exception { - partitionedTable = CatalogUtil.normalizeIdentifier("TestSimpleQuery_Partitioned"); + partitionedTable = IdentifierUtil.normalizeIdentifier("TestSimpleQuery_Partitioned"); client.executeQueryAndGetResult("create table " + partitionedTable + " (col4 text) partition by column(col1 int4, col2 int4, col3 float8) " + "as select l_returnflag, l_orderkey, l_partkey, l_quantity from lineitem"); - table = CatalogUtil.normalizeIdentifier("TestSimpleQuery"); + table = IdentifierUtil.normalizeIdentifier("TestSimpleQuery"); client.executeQueryAndGetResult("create table " + table + " (col4 text, col1 int4, col2 int4, col3 float8) " + "as select l_returnflag, l_orderkey, l_partkey, l_quantity from lineitem"); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java index 196f332fbc..80555a4c80 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java @@ -39,6 +39,7 @@ import org.apache.tajo.ipc.ClientProtos; import org.apache.tajo.plan.logical.NodeType; import org.apache.tajo.querymaster.QueryMasterTask; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.storage.StorageConstants; import org.apache.tajo.util.CommonTestingUtil; import org.apache.tajo.util.KeyValueSet; @@ -77,7 +78,7 @@ public static Collection generateParameters() { @Test public final void testCreateColumnPartitionedTable() throws Exception { ResultSet res; - String tableName = CatalogUtil.normalizeIdentifier("testCreateColumnPartitionedTable"); + String tableName = IdentifierUtil.normalizeIdentifier("testCreateColumnPartitionedTable"); ClientProtos.SubmitQueryResponse response; if (nodeType == NodeType.INSERT) { res = executeString( @@ -134,7 +135,7 @@ public final void testCreateColumnPartitionedTable() throws Exception { public final void testCreateColumnPartitionedTableWithJoin() throws Exception { ResultSet res; ClientProtos.SubmitQueryResponse response; - String tableName = CatalogUtil.normalizeIdentifier("testCreateColumnPartitionedTableWithJoin"); + String tableName = IdentifierUtil.normalizeIdentifier("testCreateColumnPartitionedTableWithJoin"); if (nodeType == NodeType.INSERT) { res = executeString( @@ -188,7 +189,7 @@ public final void testCreateColumnPartitionedTableWithJoin() throws Exception { @Test public final void testCreateColumnPartitionedTableWithSelectedColumns() throws Exception { ResultSet res = null; - String tableName = CatalogUtil.normalizeIdentifier("testCreateColumnPartitionedTableWithSelectedColumns"); + String tableName = IdentifierUtil.normalizeIdentifier("testCreateColumnPartitionedTableWithSelectedColumns"); if (nodeType == NodeType.INSERT) { res = executeString( @@ -217,7 +218,7 @@ public final void testCreateColumnPartitionedTableWithSelectedColumns() throws E @Test public final void testColumnPartitionedTableByOneColumn() throws Exception { ResultSet res = null; - String tableName = CatalogUtil.normalizeIdentifier("testColumnPartitionedTableByOneColumn"); + String tableName = IdentifierUtil.normalizeIdentifier("testColumnPartitionedTableByOneColumn"); if (nodeType == NodeType.INSERT) { res = executeString( @@ -275,7 +276,7 @@ private void assertPartitionDirectories(TableDesc desc) throws IOException { @Test public final void testQueryCasesOnColumnPartitionedTable() throws Exception { ResultSet res = null; - String tableName = CatalogUtil.normalizeIdentifier("testQueryCasesOnColumnPartitionedTable"); + String tableName = IdentifierUtil.normalizeIdentifier("testQueryCasesOnColumnPartitionedTable"); if (nodeType == NodeType.INSERT) { res = executeString( @@ -368,7 +369,7 @@ public final void testQueryCasesOnColumnPartitionedTable() throws Exception { @Test public final void testColumnPartitionedTableByThreeColumns() throws Exception { ResultSet res = null; - String tableName = CatalogUtil.normalizeIdentifier("testColumnPartitionedTableByThreeColumns"); + String tableName = IdentifierUtil.normalizeIdentifier("testColumnPartitionedTableByThreeColumns"); if (nodeType == NodeType.INSERT) { res = testBase.execute( @@ -455,7 +456,7 @@ public final void testColumnPartitionedTableByThreeColumns() throws Exception { @Test public final void testInsertIntoColumnPartitionedTableByThreeColumns() throws Exception { ResultSet res = null; - String tableName = CatalogUtil.normalizeIdentifier("testInsertIntoColumnPartitionedTableByThreeColumns"); + String tableName = IdentifierUtil.normalizeIdentifier("testInsertIntoColumnPartitionedTableByThreeColumns"); if (nodeType == NodeType.INSERT) { res = testBase.execute( @@ -636,7 +637,7 @@ private final void verifyDirectoriesForThreeColumns(FileSystem fs, Path path, in @Test public final void testColumnPartitionedTableByOneColumnsWithCompression() throws Exception { ResultSet res = null; - String tableName = CatalogUtil.normalizeIdentifier("testColumnPartitionedTableByOneColumnsWithCompression"); + String tableName = IdentifierUtil.normalizeIdentifier("testColumnPartitionedTableByOneColumnsWithCompression"); if (nodeType == NodeType.INSERT) { res = executeString( @@ -687,7 +688,7 @@ public final void testColumnPartitionedTableByOneColumnsWithCompression() throws @Test public final void testColumnPartitionedTableByTwoColumnsWithCompression() throws Exception { ResultSet res = null; - String tableName = CatalogUtil.normalizeIdentifier("testColumnPartitionedTableByTwoColumnsWithCompression"); + String tableName = IdentifierUtil.normalizeIdentifier("testColumnPartitionedTableByTwoColumnsWithCompression"); if (nodeType == NodeType.INSERT) { res = executeString("create table " + tableName + " (col3 float8, col4 text) USING text " + @@ -746,7 +747,7 @@ public final void testColumnPartitionedTableByTwoColumnsWithCompression() throws @Test public final void testColumnPartitionedTableByThreeColumnsWithCompression() throws Exception { ResultSet res = null; - String tableName = CatalogUtil.normalizeIdentifier("testColumnPartitionedTableByThreeColumnsWithCompression"); + String tableName = IdentifierUtil.normalizeIdentifier("testColumnPartitionedTableByThreeColumnsWithCompression"); if (nodeType == NodeType.INSERT) { res = executeString( @@ -844,7 +845,7 @@ public final void testColumnPartitionedTableByThreeColumnsWithCompression() thro @Test public final void testColumnPartitionedTableNoMatchedPartition() throws Exception { ResultSet res = null; - String tableName = CatalogUtil.normalizeIdentifier("testColumnPartitionedTableNoMatchedPartition"); + String tableName = IdentifierUtil.normalizeIdentifier("testColumnPartitionedTableNoMatchedPartition"); if (nodeType == NodeType.INSERT) { res = executeString( @@ -915,7 +916,7 @@ public final void testColumnPartitionedTableNoMatchedPartition() throws Exceptio @Test public final void testColumnPartitionedTableWithSmallerExpressions1() throws Exception { ResultSet res = null; - String tableName = CatalogUtil.normalizeIdentifier("testColumnPartitionedTableWithSmallerExpressions1"); + String tableName = IdentifierUtil.normalizeIdentifier("testColumnPartitionedTableWithSmallerExpressions1"); res = executeString( "create table " + tableName + " (col1 int4, col2 int4, null_col int4) partition by column(key float8) "); res.close(); @@ -943,7 +944,7 @@ public final void testColumnPartitionedTableWithSmallerExpressions1() throws Exc public final void testColumnPartitionedTableWithSmallerExpressions2() throws Exception { ResultSet res = null; ClientProtos.SubmitQueryResponse response = null; - String tableName = CatalogUtil.normalizeIdentifier("testColumnPartitionedTableWithSmallerExpressions2"); + String tableName = IdentifierUtil.normalizeIdentifier("testColumnPartitionedTableWithSmallerExpressions2"); if (nodeType == NodeType.INSERT) { res = executeString( @@ -1028,7 +1029,7 @@ public final void testColumnPartitionedTableWithSmallerExpressions3() throws Exc @Test public final void testColumnPartitionedTableWithSmallerExpressions5() throws Exception { ResultSet res = null; - String tableName = CatalogUtil.normalizeIdentifier("testColumnPartitionedTableWithSmallerExpressions5"); + String tableName = IdentifierUtil.normalizeIdentifier("testColumnPartitionedTableWithSmallerExpressions5"); if (nodeType == NodeType.INSERT) { res = executeString( @@ -1058,7 +1059,7 @@ public final void testColumnPartitionedTableWithSmallerExpressions5() throws Exc @Test public final void testColumnPartitionedTableWithSmallerExpressions6() throws Exception { ResultSet res = null; - String tableName = CatalogUtil.normalizeIdentifier("testColumnPartitionedTableWithSmallerExpressions6"); + String tableName = IdentifierUtil.normalizeIdentifier("testColumnPartitionedTableWithSmallerExpressions6"); if (nodeType == NodeType.INSERT) { res = executeString( @@ -1302,7 +1303,7 @@ private void verifyPartitionDirectoryFromCatalog(String databaseName, String tab @Test public final void testDuplicatedPartitions() throws Exception { - String tableName = CatalogUtil.normalizeIdentifier("testDuplicatedPartitions"); + String tableName = IdentifierUtil.normalizeIdentifier("testDuplicatedPartitions"); try { executeString("CREATE TABLE lineitem2 as select * from lineitem").close(); @@ -1344,7 +1345,7 @@ public final void testDuplicatedPartitions() throws Exception { @Test public final void testPatternMatchingPredicatesAndStringFunctions() throws Exception { ResultSet res = null; - String tableName = CatalogUtil.normalizeIdentifier("testPatternMatchingPredicatesAndStringFunctions"); + String tableName = IdentifierUtil.normalizeIdentifier("testPatternMatchingPredicatesAndStringFunctions"); String expectedResult; if (nodeType == NodeType.INSERT) { @@ -1442,7 +1443,7 @@ public final void testPatternMatchingPredicatesAndStringFunctions() throws Excep @Test public final void testDatePartitionColumn() throws Exception { ResultSet res = null; - String tableName = CatalogUtil.normalizeIdentifier("testDatePartitionColumn"); + String tableName = IdentifierUtil.normalizeIdentifier("testDatePartitionColumn"); String expectedResult; if (nodeType == NodeType.INSERT) { @@ -1551,7 +1552,7 @@ public final void testDatePartitionColumn() throws Exception { @Test public final void testTimestampPartitionColumn() throws Exception { ResultSet res = null; - String tableName = CatalogUtil.normalizeIdentifier("testTimestampPartitionColumn"); + String tableName = IdentifierUtil.normalizeIdentifier("testTimestampPartitionColumn"); String expectedResult; if (nodeType == NodeType.INSERT) { @@ -1620,7 +1621,7 @@ public final void testTimestampPartitionColumn() throws Exception { @Test public final void testTimePartitionColumn() throws Exception { ResultSet res = null; - String tableName = CatalogUtil.normalizeIdentifier("testTimePartitionColumn"); + String tableName = IdentifierUtil.normalizeIdentifier("testTimePartitionColumn"); String expectedResult; if (nodeType == NodeType.INSERT) { @@ -1700,7 +1701,7 @@ public final void testDatabaseNameIncludeTableName() throws Exception { executeString("create database test_partition").close(); String databaseName = "test_partition"; - String tableName = CatalogUtil.normalizeIdentifier("part"); + String tableName = IdentifierUtil.normalizeIdentifier("part"); if (nodeType == NodeType.INSERT) { executeString( @@ -1746,7 +1747,7 @@ public void testAbnormalDirectories() throws Exception { FileSystem fs = FileSystem.get(conf); Path path = null; - String tableName = CatalogUtil.normalizeIdentifier("testAbnormalDirectories"); + String tableName = IdentifierUtil.normalizeIdentifier("testAbnormalDirectories"); if (nodeType == NodeType.INSERT) { executeString( "create table " + tableName + " (col1 int4, col2 int4) partition by column(key float8) ").close(); @@ -1843,7 +1844,7 @@ public void testAbnormalDirectories() throws Exception { @Test public final void testPartitionWithInOperator() throws Exception { ResultSet res = null; - String tableName = CatalogUtil.normalizeIdentifier("testPartitionWithInOperator"); + String tableName = IdentifierUtil.normalizeIdentifier("testPartitionWithInOperator"); String result, expectedResult; if (nodeType == NodeType.INSERT) { diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java b/tajo-core-tests/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java index 7c6d8922b5..77dc498895 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/master/TestExecutionBlockCursor.java @@ -34,6 +34,7 @@ import org.apache.tajo.plan.LogicalOptimizer; import org.apache.tajo.plan.LogicalPlan; import org.apache.tajo.plan.LogicalPlanner; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.storage.TablespaceManager; import org.apache.tajo.util.CommonTestingUtil; import org.junit.AfterClass; @@ -71,7 +72,7 @@ public static void setUp() throws Exception { for (String table : tpch.getTableNames()) { TableMeta m = CatalogUtil.newTableMeta("TEXT"); TableDesc d = CatalogUtil.newTableDesc( - CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, table), tpch.getSchema(table), m, CommonTestingUtil.getTestDir()); + IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, table), tpch.getSchema(table), m, CommonTestingUtil.getTestDir()); TableStats stats = new TableStats(); stats.setNumBytes(TPCH.tableVolumes.get(table)); d.setStats(stats); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/querymaster/TestTaskStatusUpdate.java b/tajo-core-tests/src/test/java/org/apache/tajo/querymaster/TestTaskStatusUpdate.java index 425a7d6fc7..47ec77f3c1 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/querymaster/TestTaskStatusUpdate.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/querymaster/TestTaskStatusUpdate.java @@ -22,8 +22,8 @@ import org.apache.tajo.QueryId; import org.apache.tajo.QueryTestCaseBase; import org.apache.tajo.TajoConstants; -import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.util.history.QueryHistory; import org.apache.tajo.util.history.StageHistory; import org.junit.BeforeClass; @@ -118,7 +118,7 @@ public final void case3() throws Exception { } private void createColumnPartitionedTable() throws Exception { - String tableName = CatalogUtil.normalizeIdentifier("ColumnPartitionedTable"); + String tableName = IdentifierUtil.normalizeIdentifier("ColumnPartitionedTable"); ResultSet res = executeString( "create table " + tableName + " (col1 int4, col2 int4) partition by column(key float8) "); res.close(); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/ws/rs/resources/TestTablesResource.java b/tajo-core-tests/src/test/java/org/apache/tajo/ws/rs/resources/TestTablesResource.java index 381c044ef4..b1b4822c0b 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/ws/rs/resources/TestTablesResource.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/ws/rs/resources/TestTablesResource.java @@ -19,11 +19,11 @@ import org.apache.commons.lang.StringUtils; import org.apache.tajo.QueryTestCaseBase; -import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.conf.TajoConf.ConfVars; import org.apache.tajo.error.Errors.ResultCode; import org.apache.tajo.plan.serder.PlanGsonHelper; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.ws.rs.netty.gson.GsonFeature; import org.apache.tajo.ws.rs.requests.NewSessionRequest; import org.apache.tajo.ws.rs.requests.SubmitQueryRequest; @@ -137,7 +137,7 @@ public void testGetAllTable() throws Exception { boolean tableFound = false; for (String table: tableNames) { - if (StringUtils.equalsIgnoreCase(tableName, CatalogUtil.extractSimpleName(table))) { + if (StringUtils.equalsIgnoreCase(tableName, IdentifierUtil.extractSimpleName(table))) { tableFound = true; break; } @@ -158,7 +158,7 @@ public void testGetTable() throws Exception { .request().get(new GenericType<>(TableDesc.class)); assertNotNull(selectedTable); - assertTrue(StringUtils.equalsIgnoreCase(tableName, CatalogUtil.extractSimpleName(selectedTable.getName()))); + assertTrue(StringUtils.equalsIgnoreCase(tableName, IdentifierUtil.extractSimpleName(selectedTable.getName()))); } @Test @@ -183,7 +183,7 @@ public void testDropTable() throws Exception { .request().get(new GenericType<>(TableDesc.class)); assertNotNull(selectedTable); - assertTrue(StringUtils.equalsIgnoreCase(tableName, CatalogUtil.extractSimpleName(selectedTable.getName()))); + assertTrue(StringUtils.equalsIgnoreCase(tableName, IdentifierUtil.extractSimpleName(selectedTable.getName()))); Response response = restClient.target(tablesURI) .path("/{tableName}").resolveTemplate("tableName", tableName) diff --git a/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java b/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java index 205329bba1..4059f62241 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java @@ -54,6 +54,7 @@ import org.apache.tajo.querymaster.QueryJobEvent; import org.apache.tajo.rpc.BlockingRpcServer; import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.*; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.session.Session; import org.apache.tajo.util.KeyValueSet; import org.apache.tajo.util.NetUtils; @@ -772,8 +773,8 @@ public ReturnState existTable(RpcController controller, SessionedStringProto req String databaseName; String tableName; - if (CatalogUtil.isFQTableName(request.getValue())) { - String [] splitted = CatalogUtil.splitFQTableName(request.getValue()); + if (IdentifierUtil.isFQTableName(request.getValue())) { + String [] splitted = IdentifierUtil.splitFQTableName(request.getValue()); databaseName = splitted[0]; tableName = splitted[1]; } else { @@ -833,8 +834,8 @@ public TableResponse getTableDesc(RpcController controller, SessionedStringProto String databaseName; String tableName; - if (CatalogUtil.isFQTableName(request.getValue())) { - String [] splitted = CatalogUtil.splitFQTableName(request.getValue()); + if (IdentifierUtil.isFQTableName(request.getValue())) { + String [] splitted = IdentifierUtil.splitFQTableName(request.getValue()); databaseName = splitted[0]; tableName = splitted[1]; } else { @@ -969,8 +970,8 @@ public PartitionListResponse getPartitionsByTableName(RpcController controller, String databaseName; String tableName; - if (CatalogUtil.isFQTableName(request.getValue())) { - String [] splitted = CatalogUtil.splitFQTableName(request.getValue()); + if (IdentifierUtil.isFQTableName(request.getValue())) { + String [] splitted = IdentifierUtil.splitFQTableName(request.getValue()); databaseName = splitted[0]; tableName = splitted[1]; } else { @@ -998,8 +999,8 @@ public IndexResponse getIndexWithName(RpcController controller, SessionedStringP Session session = context.getSessionManager().getSession(request.getSessionId().getId()); String indexName, databaseName; - if (CatalogUtil.isFQTableName(request.getValue())) { - String [] splitted = CatalogUtil.splitFQTableName(request.getValue()); + if (IdentifierUtil.isFQTableName(request.getValue())) { + String [] splitted = IdentifierUtil.splitFQTableName(request.getValue()); databaseName = splitted[0]; indexName = splitted[1]; } else { @@ -1027,8 +1028,8 @@ public ReturnState existIndexWithName(RpcController controller, SessionedStringP Session session = context.getSessionManager().getSession(request.getSessionId().getId()); String indexName, databaseName; - if (CatalogUtil.isFQTableName(request.getValue())) { - String [] splitted = CatalogUtil.splitFQTableName(request.getValue()); + if (IdentifierUtil.isFQTableName(request.getValue())) { + String [] splitted = IdentifierUtil.splitFQTableName(request.getValue()); databaseName = splitted[0]; indexName = splitted[1]; } else { @@ -1054,8 +1055,8 @@ public IndexListResponse getIndexesForTable(RpcController controller, SessionedS Session session = context.getSessionManager().getSession(request.getSessionId().getId()); String tableName, databaseName; - if (CatalogUtil.isFQTableName(request.getValue())) { - String [] splitted = CatalogUtil.splitFQTableName(request.getValue()); + if (IdentifierUtil.isFQTableName(request.getValue())) { + String [] splitted = IdentifierUtil.splitFQTableName(request.getValue()); databaseName = splitted[0]; tableName = splitted[1]; } else { @@ -1083,8 +1084,8 @@ public ReturnState existIndexesForTable(RpcController controller, SessionedStrin Session session = context.getSessionManager().getSession(request.getSessionId().getId()); String tableName, databaseName; - if (CatalogUtil.isFQTableName(request.getValue())) { - String [] splitted = CatalogUtil.splitFQTableName(request.getValue()); + if (IdentifierUtil.isFQTableName(request.getValue())) { + String [] splitted = IdentifierUtil.splitFQTableName(request.getValue()); databaseName = splitted[0]; tableName = splitted[1]; } else { @@ -1109,8 +1110,8 @@ public IndexResponse getIndexWithColumns(RpcController controller, GetIndexWithC Session session = context.getSessionManager().getSession(request.getSessionId().getId()); String tableName, databaseName; - if (CatalogUtil.isFQTableName(request.getTableName())) { - String [] splitted = CatalogUtil.splitFQTableName(request.getTableName()); + if (IdentifierUtil.isFQTableName(request.getTableName())) { + String [] splitted = IdentifierUtil.splitFQTableName(request.getTableName()); databaseName = splitted[0]; tableName = splitted[1]; } else { @@ -1140,8 +1141,8 @@ public ReturnState existIndexWithColumns(RpcController controller, GetIndexWithC Session session = context.getSessionManager().getSession(request.getSessionId().getId()); String tableName, databaseName; - if (CatalogUtil.isFQTableName(request.getTableName())) { - String [] splitted = CatalogUtil.splitFQTableName(request.getTableName()); + if (IdentifierUtil.isFQTableName(request.getTableName())) { + String [] splitted = IdentifierUtil.splitFQTableName(request.getTableName()); databaseName = splitted[0]; tableName = splitted[1]; } else { @@ -1169,8 +1170,8 @@ public ReturnState dropIndex(RpcController controller, SessionedStringProto requ QueryContext queryContext = new QueryContext(conf, session); String indexName, databaseName; - if (CatalogUtil.isFQTableName(request.getValue())) { - String [] splitted = CatalogUtil.splitFQTableName(request.getValue()); + if (IdentifierUtil.isFQTableName(request.getValue())) { + String [] splitted = IdentifierUtil.splitFQTableName(request.getValue()); databaseName = splitted[0]; indexName = splitted[1]; } else { diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/CreateTableExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/CreateTableExecutor.java index a2b5ab9fee..3b3353167e 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/CreateTableExecutor.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/CreateTableExecutor.java @@ -31,6 +31,7 @@ import org.apache.tajo.master.TajoMaster; import org.apache.tajo.plan.logical.CreateTableNode; import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.storage.Tablespace; import org.apache.tajo.storage.TablespaceManager; import org.apache.tajo.util.Pair; @@ -89,7 +90,7 @@ public TableDesc create(QueryContext queryContext, Pair separatedNames = getQualifiedName(queryContext.getCurrentDatabase(), tableName); String databaseName = separatedNames.getFirst(); String simpleTableName = separatedNames.getSecond(); - String qualifiedName = CatalogUtil.buildFQName(databaseName, simpleTableName); + String qualifiedName = IdentifierUtil.buildFQName(databaseName, simpleTableName); // Check if the table to be created already exists boolean exists = catalog.existsTable(databaseName, simpleTableName); @@ -125,8 +126,8 @@ private TableDesc handlExistence(boolean ifNotExists, String qualifiedName) } private Pair getQualifiedName(String currentDatabase, String tableName) { - if (CatalogUtil.isFQTableName(tableName)) { - String [] splitted = CatalogUtil.splitFQTableName(tableName); + if (IdentifierUtil.isFQTableName(tableName)) { + String [] splitted = IdentifierUtil.splitFQTableName(tableName); return new Pair<>(splitted[0], splitted[1]); } else { return new Pair<>(currentDatabase, tableName); diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java index 6a2214191d..c480d04d44 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/DDLExecutor.java @@ -42,6 +42,7 @@ import org.apache.tajo.plan.logical.*; import org.apache.tajo.plan.rewrite.rules.PartitionedTableRewriter; import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.storage.FileTablespace; import org.apache.tajo.storage.StorageUtil; import org.apache.tajo.storage.Tablespace; @@ -138,15 +139,15 @@ public void createIndex(final QueryContext queryContext, final CreateIndexNode c throws DuplicateIndexException, UndefinedTableException, UndefinedDatabaseException { String databaseName, simpleIndexName, qualifiedIndexName; - if (CatalogUtil.isFQTableName(createIndexNode.getIndexName())) { - String[] splits = CatalogUtil.splitFQTableName(createIndexNode.getIndexName()); + if (IdentifierUtil.isFQTableName(createIndexNode.getIndexName())) { + String[] splits = IdentifierUtil.splitFQTableName(createIndexNode.getIndexName()); databaseName = splits[0]; simpleIndexName = splits[1]; qualifiedIndexName = createIndexNode.getIndexName(); } else { databaseName = queryContext.getCurrentDatabase(); simpleIndexName = createIndexNode.getIndexName(); - qualifiedIndexName = CatalogUtil.buildFQName(databaseName, simpleIndexName); + qualifiedIndexName = IdentifierUtil.buildFQName(databaseName, simpleIndexName); } if (catalog.existIndexByName(databaseName, simpleIndexName)) { @@ -158,7 +159,7 @@ public void createIndex(final QueryContext queryContext, final CreateIndexNode c throw new InternalError("Cannot find the table of the relation"); } - IndexDesc indexDesc = new IndexDesc(databaseName, CatalogUtil.extractSimpleName(scanNode.getTableName()), + IndexDesc indexDesc = new IndexDesc(databaseName, IdentifierUtil.extractSimpleName(scanNode.getTableName()), simpleIndexName, createIndexNode.getIndexPath(), createIndexNode.getKeySortSpecs(), createIndexNode.getIndexMethod(), createIndexNode.isUnique(), false, scanNode.getLogicalSchema()); @@ -170,8 +171,8 @@ public void dropIndex(final QueryContext queryContext, final DropIndexNode dropI throws UndefinedIndexException, UndefinedDatabaseException { String databaseName, simpleIndexName; - if (CatalogUtil.isFQTableName(dropIndexNode.getIndexName())) { - String[] splits = CatalogUtil.splitFQTableName(dropIndexNode.getIndexName()); + if (IdentifierUtil.isFQTableName(dropIndexNode.getIndexName())) { + String[] splits = IdentifierUtil.splitFQTableName(dropIndexNode.getIndexName()); databaseName = splits[0]; simpleIndexName = splits[1]; } else { @@ -293,15 +294,15 @@ public void dropTable(QueryContext queryContext, String tableName, boolean ifExi String databaseName; String simpleTableName; - if (CatalogUtil.isFQTableName(tableName)) { - String[] splitted = CatalogUtil.splitFQTableName(tableName); + if (IdentifierUtil.isFQTableName(tableName)) { + String[] splitted = IdentifierUtil.splitFQTableName(tableName); databaseName = splitted[0]; simpleTableName = splitted[1]; } else { databaseName = queryContext.getCurrentDatabase(); simpleTableName = tableName; } - String qualifiedName = CatalogUtil.buildFQName(databaseName, simpleTableName); + String qualifiedName = IdentifierUtil.buildFQName(databaseName, simpleTableName); boolean exists = catalog.existsTable(qualifiedName); if (!exists) { @@ -340,15 +341,15 @@ public void truncateTable(final QueryContext queryContext, final TruncateTableNo List tableDescList = new ArrayList<>(); for (String eachTableName : tableNames) { - if (CatalogUtil.isFQTableName(eachTableName)) { - String[] split = CatalogUtil.splitFQTableName(eachTableName); + if (IdentifierUtil.isFQTableName(eachTableName)) { + String[] split = IdentifierUtil.splitFQTableName(eachTableName); databaseName = split[0]; simpleTableName = split[1]; } else { databaseName = queryContext.getCurrentDatabase(); simpleTableName = eachTableName; } - final String qualifiedName = CatalogUtil.buildFQName(databaseName, simpleTableName); + final String qualifiedName = IdentifierUtil.buildFQName(databaseName, simpleTableName); if (!catalog.existsTable(databaseName, simpleTableName)) { throw new UndefinedTableException(qualifiedName); @@ -404,15 +405,15 @@ public void alterTable(TajoMaster.MasterContext context, final QueryContext quer String databaseName; String simpleTableName; - if (CatalogUtil.isFQTableName(tableName)) { - String[] split = CatalogUtil.splitFQTableName(tableName); + if (IdentifierUtil.isFQTableName(tableName)) { + String[] split = IdentifierUtil.splitFQTableName(tableName); databaseName = split[0]; simpleTableName = split[1]; } else { databaseName = queryContext.getCurrentDatabase(); simpleTableName = tableName; } - final String qualifiedName = CatalogUtil.buildFQName(databaseName, simpleTableName); + final String qualifiedName = IdentifierUtil.buildFQName(databaseName, simpleTableName); if (!catalog.existsTable(databaseName, simpleTableName)) { throw new UndefinedTableException(qualifiedName); @@ -573,8 +574,8 @@ public void repairPartition(TajoMaster.MasterContext context, final QueryContext String databaseName; String simpleTableName; - if (CatalogUtil.isFQTableName(tableName)) { - String[] split = CatalogUtil.splitFQTableName(tableName); + if (IdentifierUtil.isFQTableName(tableName)) { + String[] split = IdentifierUtil.splitFQTableName(tableName); databaseName = split[0]; simpleTableName = split[1]; } else { diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java index c7b97540af..b064952df9 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java @@ -53,6 +53,7 @@ import org.apache.tajo.resource.NodeResource; import org.apache.tajo.resource.NodeResources; import org.apache.tajo.exception.InvalidSessionException; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.storage.RowStoreUtil; import org.apache.tajo.storage.RowStoreUtil.RowStoreEncoder; import org.apache.tajo.storage.Tuple; @@ -557,7 +558,7 @@ private List getSessionInfo(Schema outSchema) { private List fetchSystemTable(TableDesc tableDesc, Schema inSchema) { List tuples = null; - String tableName = CatalogUtil.extractSimpleName(tableDesc.getName()); + String tableName = IdentifierUtil.extractSimpleName(tableDesc.getName()); if ("tablespace".equalsIgnoreCase(tableName)) { tuples = getTablespaces(inSchema); diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java index 1a51d98cf5..3158bd9efa 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java @@ -60,6 +60,7 @@ import org.apache.tajo.plan.function.python.TajoScriptEngine; import org.apache.tajo.plan.logical.*; import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.session.Session; import org.apache.tajo.storage.*; import org.apache.tajo.tuple.memory.MemoryBlock; @@ -589,15 +590,15 @@ private void checkIndexExistence(final QueryContext queryContext, final CreateIn throws DuplicateIndexException { String databaseName, simpleIndexName, qualifiedIndexName; - if (CatalogUtil.isFQTableName(createIndexNode.getIndexName())) { - String[] splits = CatalogUtil.splitFQTableName(createIndexNode.getIndexName()); + if (IdentifierUtil.isFQTableName(createIndexNode.getIndexName())) { + String[] splits = IdentifierUtil.splitFQTableName(createIndexNode.getIndexName()); databaseName = splits[0]; simpleIndexName = splits[1]; qualifiedIndexName = createIndexNode.getIndexName(); } else { databaseName = queryContext.getCurrentDatabase(); simpleIndexName = createIndexNode.getIndexName(); - qualifiedIndexName = CatalogUtil.buildFQName(databaseName, simpleIndexName); + qualifiedIndexName = IdentifierUtil.buildFQName(databaseName, simpleIndexName); } if (catalog.existIndexByName(databaseName, simpleIndexName)) { diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/prehook/CreateTableHook.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/prehook/CreateTableHook.java index 0c02b6e17c..fa2b5dfd36 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/prehook/CreateTableHook.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/prehook/CreateTableHook.java @@ -18,12 +18,12 @@ package org.apache.tajo.master.exec.prehook; -import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.plan.LogicalPlan; import org.apache.tajo.plan.logical.CreateTableNode; import org.apache.tajo.plan.logical.LogicalRootNode; import org.apache.tajo.plan.logical.NodeType; +import org.apache.tajo.schema.IdentifierUtil; public class CreateTableHook implements DistributedQueryHook { @@ -37,7 +37,7 @@ public boolean isEligible(QueryContext queryContext, LogicalPlan plan) { public void hook(QueryContext queryContext, LogicalPlan plan) throws Exception { LogicalRootNode rootNode = plan.getRootBlock().getRoot(); CreateTableNode createTableNode = rootNode.getChild(); - String [] splitted = CatalogUtil.splitFQTableName(createTableNode.getTableName()); + String [] splitted = IdentifierUtil.splitFQTableName(createTableNode.getTableName()); String databaseName = splitted[0]; String tableName = splitted[1]; queryContext.setOutputTable(tableName); diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java index d57040e3ff..289d933bd1 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/Query.java @@ -50,6 +50,7 @@ import org.apache.tajo.master.event.*; import org.apache.tajo.plan.logical.*; import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.storage.StorageConstants; import org.apache.tajo.storage.Tablespace; import org.apache.tajo.storage.TablespaceManager; @@ -544,8 +545,8 @@ private QueryState finalizeQuery(Query query, QueryCompletedEvent event) { finalOutputDir, partitions); String databaseName, simpleTableName; - if (CatalogUtil.isFQTableName(tableDesc.getName())) { - String[] split = CatalogUtil.splitFQTableName(tableDesc.getName()); + if (IdentifierUtil.isFQTableName(tableDesc.getName())) { + String[] split = IdentifierUtil.splitFQTableName(tableDesc.getName()); databaseName = split[0]; simpleTableName = split[1]; } else { @@ -630,21 +631,21 @@ public void execute(QueryMaster.QueryMasterContext context, QueryContext queryCo CreateIndexNode createIndexNode = (CreateIndexNode) lastStage.getBlock().getPlan(); String databaseName, simpleIndexName, qualifiedIndexName; - if (CatalogUtil.isFQTableName(createIndexNode.getIndexName())) { - String [] splits = CatalogUtil.splitFQTableName(createIndexNode.getIndexName()); + if (IdentifierUtil.isFQTableName(createIndexNode.getIndexName())) { + String [] splits = IdentifierUtil.splitFQTableName(createIndexNode.getIndexName()); databaseName = splits[0]; simpleIndexName = splits[1]; qualifiedIndexName = createIndexNode.getIndexName(); } else { databaseName = queryContext.getCurrentDatabase(); simpleIndexName = createIndexNode.getIndexName(); - qualifiedIndexName = CatalogUtil.buildFQName(databaseName, simpleIndexName); + qualifiedIndexName = IdentifierUtil.buildFQName(databaseName, simpleIndexName); } ScanNode scanNode = PlannerUtil.findTopNode(createIndexNode, NodeType.SCAN); if (scanNode == null) { throw new IOException("Cannot find the table of the relation"); } - IndexDesc indexDesc = new IndexDesc(databaseName, CatalogUtil.extractSimpleName(scanNode.getTableName()), + IndexDesc indexDesc = new IndexDesc(databaseName, IdentifierUtil.extractSimpleName(scanNode.getTableName()), simpleIndexName, createIndexNode.getIndexPath(), createIndexNode.getKeySortSpecs(), createIndexNode.getIndexMethod(), createIndexNode.isUnique(), false, scanNode.getLogicalSchema()); diff --git a/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/TablesResource.java b/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/TablesResource.java index 3afe7c812b..e5bd0f67f8 100644 --- a/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/TablesResource.java +++ b/tajo-core/src/main/java/org/apache/tajo/ws/rs/resources/TablesResource.java @@ -21,10 +21,10 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.tajo.catalog.CatalogService; -import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.exception.TajoException; import org.apache.tajo.master.TajoMaster.MasterContext; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.ws.rs.*; import javax.ws.rs.*; @@ -174,8 +174,8 @@ public Response run(JerseyResourceDelegateContext context) { JerseyResourceDelegateContextKey.valueOf(JerseyResourceDelegateUtil.MasterContextKey, MasterContext.class); MasterContext masterContext = context.get(masterContextKey); - if (CatalogUtil.isFQTableName(tableName)) { - tableName = CatalogUtil.extractSimpleName(tableName); + if (IdentifierUtil.isFQTableName(tableName)) { + tableName = IdentifierUtil.extractSimpleName(tableName); } CatalogService catalogService = masterContext.getCatalog(); @@ -239,8 +239,8 @@ public Response run(JerseyResourceDelegateContext context) { JerseyResourceDelegateContextKey.valueOf(JerseyResourceDelegateUtil.MasterContextKey, MasterContext.class); MasterContext masterContext = context.get(masterContextKey); - if (CatalogUtil.isFQTableName(tableName)) { - tableName = CatalogUtil.extractSimpleName(tableName); + if (IdentifierUtil.isFQTableName(tableName)) { + tableName = IdentifierUtil.extractSimpleName(tableName); } CatalogService catalogService = masterContext.getCatalog(); @@ -249,7 +249,7 @@ public Response run(JerseyResourceDelegateContext context) { return Response.status(Status.NOT_FOUND).build(); } - String canonicalTableName = CatalogUtil.getCanonicalTableName(databaseName, tableName); + String canonicalTableName = IdentifierUtil.getCanonicalTableName(databaseName, tableName); try { catalogService.dropTable(canonicalTableName); diff --git a/tajo-jdbc/src/main/java/org/apache/tajo/jdbc/TajoDatabaseMetaData.java b/tajo-jdbc/src/main/java/org/apache/tajo/jdbc/TajoDatabaseMetaData.java index e3cf73a3cf..8c76abaed6 100644 --- a/tajo-jdbc/src/main/java/org/apache/tajo/jdbc/TajoDatabaseMetaData.java +++ b/tajo-jdbc/src/main/java/org/apache/tajo/jdbc/TajoDatabaseMetaData.java @@ -27,6 +27,7 @@ import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.NullDatum; import org.apache.tajo.datum.TextDatum; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.util.VersionInfo; import java.sql.*; @@ -110,7 +111,7 @@ public int getDriverMinorVersion() { @Override public String getIdentifierQuoteString() throws SQLException { - return CatalogConstants.IDENTIFIER_QUOTE_STRING; + return IdentifierUtil.IDENTIFIER_QUOTE_STRING; } @Override @@ -167,7 +168,7 @@ public String getCatalogTerm() throws SQLException { @Override public String getCatalogSeparator() throws SQLException { - return CatalogConstants.IDENTIFIER_DELIMITER; + return IdentifierUtil.IDENTIFIER_DELIMITER; } @Override @@ -182,7 +183,7 @@ public int getMaxCharLiteralLength() throws SQLException { @Override public int getMaxColumnNameLength() throws SQLException { - return CatalogConstants.MAX_IDENTIFIER_LENGTH; + return IdentifierUtil.MAX_IDENTIFIER_LENGTH; } @Override @@ -217,7 +218,7 @@ public int getMaxConnections() throws SQLException { @Override public int getMaxCursorNameLength() throws SQLException { - return CatalogConstants.MAX_IDENTIFIER_LENGTH; + return IdentifierUtil.MAX_IDENTIFIER_LENGTH; } @Override @@ -227,17 +228,17 @@ public int getMaxIndexLength() throws SQLException { @Override public int getMaxSchemaNameLength() throws SQLException { - return CatalogConstants.MAX_IDENTIFIER_LENGTH; + return IdentifierUtil.MAX_IDENTIFIER_LENGTH; } @Override public int getMaxProcedureNameLength() throws SQLException { - return CatalogConstants.MAX_IDENTIFIER_LENGTH; + return IdentifierUtil.MAX_IDENTIFIER_LENGTH; } @Override public int getMaxCatalogNameLength() throws SQLException { - return CatalogConstants.MAX_IDENTIFIER_LENGTH; + return IdentifierUtil.MAX_IDENTIFIER_LENGTH; } @Override @@ -262,7 +263,7 @@ public int getMaxStatements() throws SQLException { @Override public int getMaxTableNameLength() throws SQLException { - return CatalogConstants.MAX_IDENTIFIER_LENGTH; + return IdentifierUtil.MAX_IDENTIFIER_LENGTH; } @Override @@ -503,7 +504,7 @@ public ResultSet getColumns(@Nullable String catalog, @Nullable String schemaPat for (String table: tables) { if (table.matches(regtableNamePattern)) { TableDesc tableDesc = conn.getCatalogAdminClient().getTableDesc( - CatalogUtil.buildFQName(databaseName, table)); + IdentifierUtil.buildFQName(databaseName, table)); int pos = 0; for (Column column: tableDesc.getLogicalSchema().getRootColumns()) { diff --git a/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestResultSet.java b/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestResultSet.java index 0e9b2a794c..b360cb4c6a 100644 --- a/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestResultSet.java +++ b/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestResultSet.java @@ -35,6 +35,7 @@ import org.apache.tajo.conf.TajoConf; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.ipc.ClientProtos.SerializedResultSet; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.storage.*; import org.apache.tajo.tuple.memory.MemoryBlock; import org.apache.tajo.tuple.memory.MemoryRowBlock; @@ -101,7 +102,7 @@ public static void setup() throws Exception { stats.setAvgRows(tupleNum); stats.setNumBlocks(1000); stats.setNumShuffleOutputs(100); - desc = new TableDesc(CatalogUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "score"), + desc = new TableDesc(IdentifierUtil.buildFQName(TajoConstants.DEFAULT_DATABASE_NAME, "score"), scoreSchema, scoreMeta, p.toUri()); desc.setStats(stats); assertEquals(tupleNum, rowBlock.rows()); diff --git a/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestTajoDatabaseMetaData.java b/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestTajoDatabaseMetaData.java index 1b694c0bf1..8951590e8c 100644 --- a/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestTajoDatabaseMetaData.java +++ b/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestTajoDatabaseMetaData.java @@ -21,8 +21,8 @@ import com.google.common.collect.Sets; import org.apache.tajo.QueryTestCaseBase; import org.apache.tajo.TajoConstants; -import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.common.type.TajoTypeUtil; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.util.TUtil; import org.junit.BeforeClass; import org.junit.Test; @@ -260,9 +260,9 @@ public void testGetTablesWithPattern() throws Exception { private static String getTestColName(String dbName, String tableName, int i) { if (i % 2 == 1) { - return CatalogUtil.denormalizeIdentifier(dbName + "_" + tableName + "_col") + " int"; + return IdentifierUtil.denormalizeIdentifier(dbName + "_" + tableName + "_col") + " int"; } else { - return CatalogUtil.denormalizeIdentifier(dbName + "_" + tableName + "_COL") + " int"; + return IdentifierUtil.denormalizeIdentifier(dbName + "_" + tableName + "_COL") + " int"; } } @@ -293,7 +293,7 @@ public void testGetColumnsWithPattern() throws Exception { TUtil.putToNestedList(tables, dbName, tableName); - executeString("CREATE TABLE " + dbName + "." + CatalogUtil.denormalizeIdentifier(tableName) + + executeString("CREATE TABLE " + dbName + "." + IdentifierUtil.denormalizeIdentifier(tableName) + " (" + getTestColName(dbName, tableName, 1) + ") PARTITION BY COLUMN (" + getTestColName(dbName, tableName, 2) + ")").close(); assertTableExists(dbName + "." + tableName); @@ -308,7 +308,7 @@ public void testGetColumnsWithPattern() throws Exception { TUtil.putToNestedList(tables, dbName, tableName); - executeString("CREATE TABLE " + dbName + "." + CatalogUtil.denormalizeIdentifier(tableName) + + executeString("CREATE TABLE " + dbName + "." + IdentifierUtil.denormalizeIdentifier(tableName) + " (" + getTestColName(dbName, tableName, 1) + ") PARTITION BY COLUMN (" + getTestColName(dbName, tableName, 2) + ")").close(); assertTableExists(dbName + "." + tableName); diff --git a/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java b/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java index fb1dd61bfd..11851510c8 100644 --- a/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java +++ b/tajo-jdbc/src/test/java/org/apache/tajo/jdbc/TestTajoJdbc.java @@ -20,10 +20,10 @@ import com.google.common.collect.Maps; import org.apache.tajo.*; -import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.client.QueryStatus; +import org.apache.tajo.schema.IdentifierUtil; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -333,7 +333,7 @@ public void testDatabaseMetaDataGetColumns() throws Exception { assertEquals(22, numCols); int numColumns = 0; - TableDesc tableDesc = client.getTableDesc(CatalogUtil.buildFQName(DEFAULT_DATABASE_NAME, tableName)); + TableDesc tableDesc = client.getTableDesc(IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, tableName)); assertNotNull(tableDesc); List columns = tableDesc.getSchema().getRootColumns(); @@ -480,7 +480,7 @@ public void testMultipleConnectionsSequentialClose() throws Exception { @Test public void testCreateTableWithDateAndTimestamp() throws Exception { - String tableName = CatalogUtil.normalizeIdentifier("testCreateTableWithDateAndTimestamp"); + String tableName = IdentifierUtil.normalizeIdentifier("testCreateTableWithDateAndTimestamp"); int result; Statement stmt = null; @@ -597,7 +597,7 @@ public void testAlterTableAddPartition() throws Exception { conn = DriverManager.getConnection(connUri); assertTrue(conn.isValid(100)); - String tableName = CatalogUtil.normalizeIdentifier("testAlterTablePartition"); + String tableName = IdentifierUtil.normalizeIdentifier("testAlterTablePartition"); resultSet = executeString( "create table " + tableName + " (col1 int4, col2 int4) partition by column(key float8) "); resultSet.close(); diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/ExprNormalizer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/ExprNormalizer.java index d66263f4f0..967bf8b5ae 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/ExprNormalizer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/ExprNormalizer.java @@ -20,12 +20,12 @@ import com.google.common.collect.Sets; import org.apache.tajo.algebra.*; -import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.exception.UndefinedColumnException; import org.apache.tajo.exception.TajoException; import org.apache.tajo.plan.nameresolver.NameResolver; import org.apache.tajo.plan.nameresolver.NameResolvingMode; import org.apache.tajo.plan.visitor.SimpleAlgebraVisitor; +import org.apache.tajo.schema.IdentifierUtil; import java.util.ArrayList; import java.util.List; @@ -339,7 +339,7 @@ public Expr visitColumnReference(ExprNormalizedResult ctx, Stack stack, Co return expr; } // if a column reference is not qualified, it finds and sets the qualified column name. - if (!(expr.hasQualifier() && CatalogUtil.isFQTableName(expr.getQualifier()))) { + if (!(expr.hasQualifier() && IdentifierUtil.isFQTableName(expr.getQualifier()))) { if (!ctx.block.namedExprsMgr.contains(expr.getCanonicalName()) && expr.getType() == OpType.Column) { try { String normalized = diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java index dcabbf5c47..fce4e69d31 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java @@ -51,6 +51,7 @@ import org.apache.tajo.plan.util.ExprFinder; import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.schema.Field; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.type.Type; import org.apache.tajo.util.KeyValueSet; import org.apache.tajo.util.Pair; @@ -1478,7 +1479,7 @@ private ProjectionNode insertProjectionGroupbyBeforeSetOperation(PlanContext con // make table subquery node which has set operation as its subquery TableSubQueryNode setOpTableSubQueryNode = context.plan.createNode(TableSubQueryNode.class); - setOpTableSubQueryNode.init(CatalogUtil.buildFQName(context.queryContext.get(SessionVars.CURRENT_DATABASE), + setOpTableSubQueryNode.init(IdentifierUtil.buildFQName(context.queryContext.get(SessionVars.CURRENT_DATABASE), context.generateUniqueSubQueryName()), setOperationNode); setTargetOfTableSubQuery(context, currentBlock, setOpTableSubQueryNode); currentBlock.registerNode(setOpTableSubQueryNode); @@ -1629,9 +1630,9 @@ private InsertNode buildInsertIntoTablePlan(PlanContext context, InsertNode inse // Get and set a target table String databaseName; String tableName; - if (CatalogUtil.isFQTableName(expr.getTableName())) { - databaseName = CatalogUtil.extractQualifier(expr.getTableName()); - tableName = CatalogUtil.extractSimpleName(expr.getTableName()); + if (IdentifierUtil.isFQTableName(expr.getTableName())) { + databaseName = IdentifierUtil.extractQualifier(expr.getTableName()); + tableName = IdentifierUtil.extractSimpleName(expr.getTableName()); } else { databaseName = context.queryContext.get(SessionVars.CURRENT_DATABASE); tableName = expr.getTableName(); @@ -1849,8 +1850,8 @@ public LogicalNode handleCreateTableLike(PlanContext context, CreateTable expr, throws TajoException { String parentTableName = expr.getLikeParentTableName(); - if (CatalogUtil.isFQTableName(parentTableName) == false) { - parentTableName = CatalogUtil.buildFQName(context.queryContext.get(SessionVars.CURRENT_DATABASE), + if (IdentifierUtil.isFQTableName(parentTableName) == false) { + parentTableName = IdentifierUtil.buildFQName(context.queryContext.get(SessionVars.CURRENT_DATABASE), parentTableName); } TableDesc baseTable = catalog.getTableDesc(parentTableName); @@ -1880,11 +1881,11 @@ public LogicalNode visitCreateTable(PlanContext context, Stack stack, Crea createTableNode.setIfNotExists(expr.isIfNotExists()); // Set a table name to be created. - if (CatalogUtil.isFQTableName(expr.getTableName())) { + if (IdentifierUtil.isFQTableName(expr.getTableName())) { createTableNode.setTableName(expr.getTableName()); } else { createTableNode.setTableName( - CatalogUtil.buildFQName(context.queryContext.get(SessionVars.CURRENT_DATABASE), expr.getTableName())); + IdentifierUtil.buildFQName(context.queryContext.get(SessionVars.CURRENT_DATABASE), expr.getTableName())); } // This is CREATE TABLE LIKE if(expr.getLikeParentTableName() != null) { @@ -2011,11 +2012,11 @@ private URI getCreatedTableURI(PlanContext context, CreateTable createTable) { } else { String tableName = createTable.getTableName(); - String databaseName = CatalogUtil.isFQTableName(tableName) ? - CatalogUtil.extractQualifier(tableName) : context.queryContext.get(SessionVars.CURRENT_DATABASE); + String databaseName = IdentifierUtil.isFQTableName(tableName) ? + IdentifierUtil.extractQualifier(tableName) : context.queryContext.get(SessionVars.CURRENT_DATABASE); return storage.getTableURI( - createTable.getTableSpaceName(), databaseName, CatalogUtil.extractSimpleName(tableName)); + createTable.getTableSpaceName(), databaseName, IdentifierUtil.extractSimpleName(tableName)); } } @@ -2143,10 +2144,10 @@ public static Type convertDataType(DataTypeExpr dataType) { public LogicalNode visitDropTable(PlanContext context, Stack stack, DropTable dropTable) { DropTableNode dropTableNode = context.queryBlock.getNodeFromExpr(dropTable); String qualified; - if (CatalogUtil.isFQTableName(dropTable.getTableName())) { + if (IdentifierUtil.isFQTableName(dropTable.getTableName())) { qualified = dropTable.getTableName(); } else { - qualified = CatalogUtil.buildFQName( + qualified = IdentifierUtil.buildFQName( context.queryContext.get(SessionVars.CURRENT_DATABASE), dropTable.getTableName()); } dropTableNode.init(qualified, dropTable.isIfExists(), dropTable.isPurge()); @@ -2206,11 +2207,11 @@ public LogicalNode visitCreateIndex(PlanContext context, Stack stack, Crea QueryBlock block = context.queryBlock; CreateIndexNode createIndexNode = block.getNodeFromExpr(createIndex); - if (CatalogUtil.isFQTableName(createIndex.getIndexName())) { + if (IdentifierUtil.isFQTableName(createIndex.getIndexName())) { createIndexNode.setIndexName(createIndex.getIndexName()); } else { createIndexNode.setIndexName( - CatalogUtil.buildFQName(context.queryContext.get(SessionVars.CURRENT_DATABASE), createIndex.getIndexName())); + IdentifierUtil.buildFQName(context.queryContext.get(SessionVars.CURRENT_DATABASE), createIndex.getIndexName())); } createIndexNode.setUnique(createIndex.isUnique()); Sort.SortSpec[] sortSpecs = createIndex.getSortSpecs(); diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalTreeUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalTreeUtil.java index e9ccf14818..a4de1ed366 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalTreeUtil.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalTreeUtil.java @@ -32,6 +32,7 @@ import org.apache.tajo.plan.LogicalPlan; import org.apache.tajo.plan.Target; import org.apache.tajo.plan.util.ExprFinder; +import org.apache.tajo.schema.IdentifierUtil; import java.util.*; @@ -342,14 +343,14 @@ private static boolean isJoinQualwithSchemas(Schema leftSchema, Schema rightSche private static boolean isJoinQualWithOnlyColumns(@Nullable LogicalPlan.QueryBlock block, Column left, Column right) { - String leftQualifier = CatalogUtil.extractQualifier(left.getQualifiedName()); - String rightQualifier = CatalogUtil.extractQualifier(right.getQualifiedName()); + String leftQualifier = IdentifierUtil.extractQualifier(left.getQualifiedName()); + String rightQualifier = IdentifierUtil.extractQualifier(right.getQualifiedName()); // if block is given, it will track an original expression of each term in order to decide whether // this expression is a join condition, or not. if (block != null) { - boolean leftQualified = CatalogUtil.isFQColumnName(left.getQualifiedName()); - boolean rightQualified = CatalogUtil.isFQColumnName(right.getQualifiedName()); + boolean leftQualified = IdentifierUtil.isFQColumnName(left.getQualifiedName()); + boolean rightQualified = IdentifierUtil.isFQColumnName(right.getQualifiedName()); if (!leftQualified) { // if left one is aliased name @@ -359,7 +360,7 @@ private static boolean isJoinQualWithOnlyColumns(@Nullable LogicalPlan.QueryBloc // ensure there is only one column of an original expression if (foundColumns.size() == 1) { - leftQualifier = CatalogUtil.extractQualifier(foundColumns.iterator().next().getCanonicalName()); + leftQualifier = IdentifierUtil.extractQualifier(foundColumns.iterator().next().getCanonicalName()); } } if (!rightQualified) { // if right one is aliased name @@ -370,7 +371,7 @@ private static boolean isJoinQualWithOnlyColumns(@Nullable LogicalPlan.QueryBloc // ensure there is only one column of an original expression if (foundColumns.size() == 1) { - rightQualifier = CatalogUtil.extractQualifier(foundColumns.iterator().next().getCanonicalName()); + rightQualifier = IdentifierUtil.extractQualifier(foundColumns.iterator().next().getCanonicalName()); } } } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/FieldEval.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/FieldEval.java index 60144cea0e..d905e4b1ef 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/FieldEval.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/FieldEval.java @@ -18,45 +18,49 @@ package org.apache.tajo.plan.expr; -import com.google.gson.annotations.Expose; -import org.apache.tajo.catalog.Column; -import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.TypeConverter; +import org.apache.tajo.catalog.*; import org.apache.tajo.common.TajoDataTypes.DataType; import org.apache.tajo.datum.Datum; +import org.apache.tajo.schema.Field; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.storage.Tuple; import org.apache.tajo.type.Type; +import static org.apache.tajo.schema.Field.Field; +import static org.apache.tajo.schema.QualifiedIdentifier.$; + public class FieldEval extends EvalNode implements Cloneable { - @Expose private Column column; - @Expose private int fieldId = -1; + private Field field; + private int fieldId = -1; - public FieldEval(String columnName, DataType domain) { + public FieldEval(String columnName, DataType type) { super(EvalType.FIELD); - this.column = new Column(columnName, domain); + this.field = Field($(columnName), TypeConverter.convert(type)); } public FieldEval(String columnName, Type type) { super(EvalType.FIELD); - this.column = new Column(columnName, TypeConverter.convert(type)); + this.field = Field($(columnName), type); } - public FieldEval(Column column) { + public FieldEval(Column field) { super(EvalType.FIELD); - this.column = column; + this.field = FieldConverter.convert(field); } @Override public EvalNode bind(EvalContext evalContext, Schema schema) { super.bind(evalContext, schema); // TODO - column namespace should be improved to simplify name handling and resolving. - if (column.hasQualifier()) { - fieldId = schema.getColumnId(column.getQualifiedName()); - } else { - fieldId = schema.getColumnIdByName(column.getSimpleName()); + + fieldId = schema.getColumnId(field.name().raw()); + + if (fieldId == -1) { // fallback + fieldId = schema.getColumnIdByName(field.name().raw()); } + if (fieldId == -1) { - throw new IllegalStateException("No Such Column Reference: " + column + ", schema: " + schema); + throw new IllegalStateException("No Such Column Reference: " + field + ", schema: " + schema); } return this; } @@ -70,7 +74,7 @@ public Datum eval(Tuple tuple) { @Override public Type getValueType() { - return TypeConverter.convert(column.getDataType()); + return field.type(); } @Override @@ -84,48 +88,48 @@ public EvalNode getChild(int idx) { } public Column getColumnRef() { - return column; + return FieldConverter.convert(field); } public String getQualifier() { - return column.getQualifier(); + return IdentifierUtil.extractQualifier(field.name().raw()); } public String getColumnName() { - return column.getSimpleName(); + return IdentifierUtil.extractSimpleName(field.name().raw()); } public void replaceColumnRef(String columnName) { - this.column = new Column(columnName, this.column.getDataType()); + this.field = Field(columnName, this.field.type()); } @Override public String getName() { - return this.column.getQualifiedName(); + return this.field.name().raw(); } public String toString() { - return this.column.toString(); + return this.field.toString(); } public boolean equals(Object obj) { if (obj instanceof FieldEval) { FieldEval other = (FieldEval) obj; - return column.equals(other.column); + return field.equals(other.field); } return false; } @Override public int hashCode() { - return column.hashCode(); + return field.hashCode(); } @Override public Object clone() throws CloneNotSupportedException { FieldEval eval = (FieldEval) super.clone(); - eval.column = this.column; + eval.field = this.field; eval.fieldId = fieldId; return eval; diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/CreateDatabaseNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/CreateDatabaseNode.java index 28bd4cd82c..404dd33ddd 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/CreateDatabaseNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/CreateDatabaseNode.java @@ -19,8 +19,8 @@ package org.apache.tajo.plan.logical; import com.google.common.base.Objects; -import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.plan.PlanString; +import org.apache.tajo.schema.IdentifierUtil; public class CreateDatabaseNode extends LogicalNode implements Cloneable { private String databaseName; @@ -82,7 +82,7 @@ public Object clone() throws CloneNotSupportedException { @Override public String toString() { return "CREATE DATABASE " + (ifNotExists ? " IF NOT EXISTS " : "") - + CatalogUtil.denormalizeIdentifier(databaseName); + + IdentifierUtil.denormalizeIdentifier(databaseName); } @Override diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ScanNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ScanNode.java index 0729d6e952..f482e30421 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ScanNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/ScanNode.java @@ -27,6 +27,7 @@ import org.apache.tajo.plan.Target; import org.apache.tajo.plan.expr.EvalNode; import org.apache.tajo.plan.util.PlannerUtil; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.util.TUtil; import java.util.ArrayList; @@ -72,13 +73,13 @@ public void init(TableDesc desc, String alias) { this.tableDesc = desc; this.alias = alias; - if (!CatalogUtil.isFQTableName(this.tableDesc.getName())) { + if (!IdentifierUtil.isFQTableName(this.tableDesc.getName())) { throw new IllegalArgumentException("the name in TableDesc must be qualified, but it is \"" + desc.getName() + "\""); } - String databaseName = CatalogUtil.extractQualifier(this.tableDesc.getName()); - String qualifiedAlias = CatalogUtil.buildFQName(databaseName, alias); + String databaseName = IdentifierUtil.extractQualifier(this.tableDesc.getName()); + String qualifiedAlias = IdentifierUtil.buildFQName(databaseName, alias); this.setInSchema(tableDesc.getSchema()); this.getInSchema().setQualifier(qualifiedAlias); this.setOutSchema(SchemaBuilder.builder().addAll(getInSchema().getRootColumns()).build()); @@ -108,9 +109,9 @@ public boolean isBroadcastTable() { } public String getCanonicalName() { - if (CatalogUtil.isFQTableName(this.tableDesc.getName())) { - String databaseName = CatalogUtil.extractQualifier(this.tableDesc.getName()); - return hasAlias() ? CatalogUtil.buildFQName(databaseName, alias) : tableDesc.getName(); + if (IdentifierUtil.isFQTableName(this.tableDesc.getName())) { + String databaseName = IdentifierUtil.extractQualifier(this.tableDesc.getName()); + return hasAlias() ? IdentifierUtil.buildFQName(databaseName, alias) : tableDesc.getName(); } else { return hasAlias() ? alias : tableDesc.getName(); } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java b/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java index cbe93d2d61..95f8f98090 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/NameResolver.java @@ -22,7 +22,6 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import org.apache.tajo.algebra.ColumnReferenceExpr; -import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.NestedPathUtil; import org.apache.tajo.catalog.Schema; @@ -31,6 +30,7 @@ import org.apache.tajo.plan.LogicalPlan; import org.apache.tajo.plan.logical.RelationNode; import org.apache.tajo.plan.logical.ScanNode; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.util.Pair; import org.apache.tajo.util.StringUtils; @@ -100,8 +100,8 @@ public static RelationNode lookupTable(LogicalPlan.QueryBlock block, String tabl found.add(relation); // if a table name is not qualified - } else if (CatalogUtil.extractSimpleName(relation.getCanonicalName()).equals(tableName) || - CatalogUtil.extractSimpleName(relation.getTableName()).equals(tableName)) { + } else if (IdentifierUtil.extractSimpleName(relation.getCanonicalName()).equals(tableName) || + IdentifierUtil.extractSimpleName(relation.getTableName()).equals(tableName)) { found.add(relation); } } @@ -184,7 +184,7 @@ static Column resolveFromRelsWithinBlock(LogicalPlan plan, LogicalPlan.QueryBloc Column column; if (includeSeflDescTable && describeSchemaByItself(relationOp)) { - column = guessColumn(CatalogUtil.buildFQName(normalized.getFirst(), normalized.getSecond())); + column = guessColumn(IdentifierUtil.buildFQName(normalized.getFirst(), normalized.getSecond())); } else { // Please consider a query case: @@ -192,9 +192,9 @@ static Column resolveFromRelsWithinBlock(LogicalPlan plan, LogicalPlan.QueryBloc // // The relation lineitem is already renamed to "a", but lineitem.l_orderkey still should be available. // The below code makes it possible. Otherwise, it cannot find any match in the relation schema. - if (block.isAlreadyRenamedTableName(CatalogUtil.extractQualifier(canonicalName))) { + if (block.isAlreadyRenamedTableName(IdentifierUtil.extractQualifier(canonicalName))) { canonicalName = - CatalogUtil.buildFQName(relationOp.getCanonicalName(), CatalogUtil.extractSimpleName(canonicalName)); + IdentifierUtil.buildFQName(relationOp.getCanonicalName(), IdentifierUtil.extractSimpleName(canonicalName)); } Schema schema = relationOp.getLogicalSchema(); @@ -242,7 +242,7 @@ static Column resolveFromCurrentAndChildNode(LogicalPlan.QueryBlock block, Colum */ static Column lookupColumnFromAllRelsInBlock(LogicalPlan.QueryBlock block, String columnName, boolean includeSelfDescTable) throws AmbiguousColumnException { - Preconditions.checkArgument(CatalogUtil.isSimpleIdentifier(columnName)); + Preconditions.checkArgument(IdentifierUtil.isSimpleIdentifier(columnName)); List candidates = new ArrayList<>(); @@ -266,7 +266,7 @@ static Column lookupColumnFromAllRelsInBlock(LogicalPlan.QueryBlock block, } } if (candidateRels.size() == 1) { - return guessColumn(CatalogUtil.buildFQName(candidateRels.get(0).getCanonicalName(), columnName)); + return guessColumn(IdentifierUtil.buildFQName(candidateRels.get(0).getCanonicalName(), columnName)); } else if (candidateRels.size() > 1) { throw new AmbiguousColumnException(columnName); } @@ -377,7 +377,7 @@ static Pair lookupQualifierAndCanonicalName(LogicalPlan.QueryBlo // check for dbname.tbname.column_name.nested_field if (qualifierParts.length >= 2) { - RelationNode rel = lookupTable(block, CatalogUtil.buildFQName(qualifierParts[0], qualifierParts[1])); + RelationNode rel = lookupTable(block, IdentifierUtil.buildFQName(qualifierParts[0], qualifierParts[1])); if (rel != null) { guessedRelations.add(rel); columnNamePosition = 2; diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/ResolverByLegacy.java b/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/ResolverByLegacy.java index 343b9abfde..8cfb1eb6d3 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/ResolverByLegacy.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/nameresolver/ResolverByLegacy.java @@ -19,7 +19,6 @@ package org.apache.tajo.plan.nameresolver; import org.apache.tajo.algebra.ColumnReferenceExpr; -import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.Schema; import org.apache.tajo.exception.AmbiguousColumnException; @@ -29,6 +28,7 @@ import org.apache.tajo.plan.logical.LogicalNode; import org.apache.tajo.plan.logical.NodeType; import org.apache.tajo.plan.logical.RelationNode; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.util.Pair; import java.util.ArrayList; @@ -57,7 +57,7 @@ private static Column resolveColumnWithQualifier(LogicalPlan plan, LogicalPlan.Q Pair normalized = lookupQualifierAndCanonicalName(block, columnRef, includeSeflDescTable); qualifier = normalized.getFirst(); - qualifiedName = CatalogUtil.buildFQName(qualifier, columnRef.getName()); + qualifiedName = IdentifierUtil.buildFQName(qualifier, columnRef.getName()); Column found = resolveFromRelsWithinBlock(plan, block, columnRef, includeSeflDescTable); if (found == null) { diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseSchemaBuildPhase.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseSchemaBuildPhase.java index 99921c3817..a262e563e0 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseSchemaBuildPhase.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseSchemaBuildPhase.java @@ -36,6 +36,7 @@ import org.apache.tajo.plan.nameresolver.NameResolvingMode; import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.plan.visitor.SimpleAlgebraVisitor; +import org.apache.tajo.schema.IdentifierUtil; import java.util.*; @@ -117,10 +118,10 @@ public static Column[] getColumns(LogicalPlanner.PlanContext ctx, QualifiedAster if (asteriskExpr.hasQualifier()) { String qualifier; - if (CatalogUtil.isFQTableName(asteriskExpr.getQualifier())) { + if (IdentifierUtil.isFQTableName(asteriskExpr.getQualifier())) { qualifier = asteriskExpr.getQualifier(); } else { - qualifier = CatalogUtil.buildFQName( + qualifier = IdentifierUtil.buildFQName( ctx.getQueryContext().get(SessionVars.CURRENT_DATABASE), asteriskExpr.getQualifier()); } @@ -138,7 +139,7 @@ public static Column[] getColumns(LogicalPlanner.PlanContext ctx, QualifiedAster // If we cannot find any relation against a qualified column name if (relationOp == null) { - throw new UndefinedColumnException(CatalogUtil.buildFQName(qualifier, "*")); + throw new UndefinedColumnException(IdentifierUtil.buildFQName(qualifier, "*")); } Schema schema = relationOp.getLogicalSchema(); @@ -403,11 +404,11 @@ public LogicalNode visitRelation(LogicalPlanner.PlanContext ctx, Stack sta throws TajoException { String actualRelationName; - if (CatalogUtil.isFQTableName(relation.getName())) { + if (IdentifierUtil.isFQTableName(relation.getName())) { actualRelationName = relation.getName(); } else { actualRelationName = - CatalogUtil.buildFQName(ctx.getQueryContext().get(SessionVars.CURRENT_DATABASE), relation.getName()); + IdentifierUtil.buildFQName(ctx.getQueryContext().get(SessionVars.CURRENT_DATABASE), relation.getName()); } TableDesc desc = catalog.getTableDesc(actualRelationName); @@ -439,7 +440,7 @@ public LogicalNode visitTableSubQuery(LogicalPlanner.PlanContext ctx, Stack predicates = new HashSet<>(); for (EvalNode eval : PlannerUtil.getAllEqualEvals(qual)) { BinaryEval binaryEval = (BinaryEval) eval; diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java index cf92ea0a25..708f0d199f 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java @@ -39,6 +39,7 @@ import org.apache.tajo.plan.util.EvalNodeToExprConverter; import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.plan.visitor.BasicLogicalPlanVisitor; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.VTuple; import org.apache.tajo.util.StringUtils; @@ -134,7 +135,7 @@ public String toString() { Path [] filteredPaths = null; FileSystem fs = tablePath.getFileSystem(queryContext.getConf()); - String [] splits = CatalogUtil.splitFQTableName(tableName); + String [] splits = IdentifierUtil.splitFQTableName(tableName); List partitions = null; try { diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java index 7fab5ba5b1..2b197ff0ec 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java @@ -25,11 +25,11 @@ import org.apache.tajo.TajoConstants; import org.apache.tajo.algebra.*; import org.apache.tajo.catalog.CatalogService; -import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.TableDesc; import org.apache.tajo.exception.*; import org.apache.tajo.plan.algebra.BaseAlgebraVisitor; import org.apache.tajo.plan.util.ExprFinder; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.validation.ConstraintViolation; import java.util.Collection; @@ -148,10 +148,10 @@ public Expr visitRelation(Context context, Stack stack, Relation expr) thr private boolean assertRelationExistence(Context context, String tableName) { String qualifiedName; - if (CatalogUtil.isFQTableName(tableName)) { + if (IdentifierUtil.isFQTableName(tableName)) { qualifiedName = tableName; } else { - qualifiedName = CatalogUtil.buildFQName(context.queryContext.get(SessionVars.CURRENT_DATABASE), tableName); + qualifiedName = IdentifierUtil.buildFQName(context.queryContext.get(SessionVars.CURRENT_DATABASE), tableName); } if (!catalog.existsTable(qualifiedName)) { @@ -171,10 +171,10 @@ private void assertRelationSchema(Context context, CreateTable createTable) { private static String guessTableName(Context context, String givenName) { String qualifiedName; - if (CatalogUtil.isFQTableName(givenName)) { + if (IdentifierUtil.isFQTableName(givenName)) { qualifiedName = givenName; } else { - qualifiedName = CatalogUtil.buildFQName(context.queryContext.get(SessionVars.CURRENT_DATABASE), givenName); + qualifiedName = IdentifierUtil.buildFQName(context.queryContext.get(SessionVars.CURRENT_DATABASE), givenName); } return qualifiedName; @@ -327,8 +327,8 @@ public Expr visitInsert(Context context, Stack stack, Insert expr) throws } else { if (expr.hasTableName()) { String qualifiedName = expr.getTableName(); - if (TajoConstants.EMPTY_STRING.equals(CatalogUtil.extractQualifier(expr.getTableName()))) { - qualifiedName = CatalogUtil.buildFQName(context.queryContext.get(SessionVars.CURRENT_DATABASE), + if (TajoConstants.EMPTY_STRING.equals(IdentifierUtil.extractQualifier(expr.getTableName()))) { + qualifiedName = IdentifierUtil.buildFQName(context.queryContext.get(SessionVars.CURRENT_DATABASE), expr.getTableName()); } diff --git a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileTablespace.java b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileTablespace.java index 3d12a409e6..34240ca906 100644 --- a/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileTablespace.java +++ b/tajo-storage/tajo-storage-hdfs/src/main/java/org/apache/tajo/storage/FileTablespace.java @@ -40,6 +40,7 @@ import org.apache.tajo.plan.expr.EvalNode; import org.apache.tajo.plan.logical.LogicalNode; import org.apache.tajo.plan.logical.NodeType; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.storage.fragment.Fragment; import org.apache.tajo.util.Bytes; @@ -612,7 +613,7 @@ public List getSplits(String inputSourceId, @Override public void createTable(TableDesc tableDesc, boolean ifNotExists) throws IOException { if (!tableDesc.isExternal()) { - String [] splitted = CatalogUtil.splitFQTableName(tableDesc.getName()); + String [] splitted = IdentifierUtil.splitFQTableName(tableDesc.getName()); String databaseName = splitted[0]; String simpleTableName = splitted[1]; diff --git a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcMetadataProviderBase.java b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcMetadataProviderBase.java index 8f5e7a8665..fb095a40bf 100644 --- a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcMetadataProviderBase.java +++ b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/JdbcMetadataProviderBase.java @@ -30,6 +30,7 @@ import org.apache.tajo.exception.TajoInternalError; import org.apache.tajo.exception.UndefinedTablespaceException; import org.apache.tajo.exception.UnsupportedDataTypeException; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.util.KeyValueSet; import org.apache.tajo.util.Pair; @@ -197,7 +198,7 @@ public TableDesc getTableDesc(String schemaName, String tableName) throws Undefi final String qualifier = resultForColumns.getString("TABLE_NAME"); final String columnName = resultForColumns.getString("COLUMN_NAME"); final TypeDesc type = convertDataType(resultForColumns); - final Column c = new Column(CatalogUtil.buildFQName(databaseName, qualifier, columnName), type); + final Column c = new Column(IdentifierUtil.buildFQName(databaseName, qualifier, columnName), type); columns.add(new Pair<>(ordinalPos, c)); } @@ -224,7 +225,7 @@ public Column apply(@Nullable Pair columnPair) { stats.setNumRows(-1); // unknown final TableDesc table = new TableDesc( - CatalogUtil.buildFQName(databaseName, name), + IdentifierUtil.buildFQName(databaseName, name), schema, new TableMeta("rowstore", new KeyValueSet()), space.getTableUri(databaseName, name) diff --git a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/SQLExpressionGenerator.java b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/SQLExpressionGenerator.java index 0b1df31054..cd5f2926fc 100644 --- a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/SQLExpressionGenerator.java +++ b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/SQLExpressionGenerator.java @@ -19,13 +19,12 @@ package org.apache.tajo.storage.jdbc; import com.google.common.base.Function; -import org.apache.tajo.catalog.CatalogUtil; -import org.apache.tajo.common.TajoDataTypes.DataType; import org.apache.tajo.datum.Datum; import org.apache.tajo.exception.NotImplementedException; import org.apache.tajo.exception.TajoRuntimeException; import org.apache.tajo.exception.UnsupportedDataTypeException; import org.apache.tajo.plan.expr.*; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.type.Type; import org.apache.tajo.util.StringUtils; @@ -145,13 +144,13 @@ public String apply(Datum v) { protected EvalNode visitField(Context context, FieldEval field, Stack stack) { // strip the database name String tableName; - if (CatalogUtil.isSimpleIdentifier(field.getQualifier())) { + if (IdentifierUtil.isSimpleIdentifier(field.getQualifier())) { tableName = field.getQualifier(); } else { - tableName = CatalogUtil.extractSimpleName(field.getQualifier()); + tableName = IdentifierUtil.extractSimpleName(field.getQualifier()); } - context.append(CatalogUtil.buildFQName(tableName, field.getColumnName())); + context.append(IdentifierUtil.buildFQName(tableName, field.getColumnName())); return field; } diff --git a/tajo-storage/tajo-storage-pgsql/src/main/java/org/apache/tajo/storage/pgsql/PgSQLTablespace.java b/tajo-storage/tajo-storage-pgsql/src/main/java/org/apache/tajo/storage/pgsql/PgSQLTablespace.java index 8386f74995..90c52011c8 100644 --- a/tajo-storage/tajo-storage-pgsql/src/main/java/org/apache/tajo/storage/pgsql/PgSQLTablespace.java +++ b/tajo-storage/tajo-storage-pgsql/src/main/java/org/apache/tajo/storage/pgsql/PgSQLTablespace.java @@ -24,6 +24,7 @@ import org.apache.tajo.exception.TajoRuntimeException; import org.apache.tajo.exception.UndefinedTableException; import org.apache.tajo.plan.expr.EvalNode; +import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.storage.NullScanner; import org.apache.tajo.storage.Scanner; import org.apache.tajo.storage.fragment.Fragment; @@ -82,7 +83,7 @@ public int hashCode() { @Override public long getTableVolume(TableDesc table, Optional filter) { - String sql = "SELECT pg_table_size('" + CatalogUtil.extractSimpleName(table.getName()) + "')"; + String sql = "SELECT pg_table_size('" + IdentifierUtil.extractSimpleName(table.getName()) + "')"; try (Statement stmt = conn.createStatement(); ResultSet rs = stmt.executeQuery(sql)) { From 3c94e202b55c2042398cce3c4f02659a81183ba5 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Mon, 25 Apr 2016 14:50:39 -0700 Subject: [PATCH 10/23] Fix hash problem and the comparator of SortSpec. --- .../java/org/apache/tajo/schema/Field.java | 7 ++-- .../main/java/org/apache/tajo/type/Array.java | 2 +- .../main/java/org/apache/tajo/type/Map.java | 2 +- .../java/org/apache/tajo/type/Record.java | 2 +- .../main/java/org/apache/tajo/type/Type.java | 2 +- .../tajo/cli/tsql/TestTajoCliNegatives.java | 2 +- .../apache/tajo/engine/eval/TestEvalTree.java | 36 ++++++++++++++++--- .../TestCommonConditionReduce/test51.plan | 2 +- .../testJoinWithMultipleJoinQual2.Hash.plan | 4 +-- ...ithMultipleJoinQual2.Hash_NoBroadcast.plan | 4 +-- .../testJoinWithMultipleJoinQual2.Sort.plan | 4 +-- ...ithMultipleJoinQual2.Sort_NoBroadcast.plan | 4 +-- ...uterJoinPredicationCaseByCase3.1.Hash.plan | 4 +-- ...icationCaseByCase3.1.Hash_NoBroadcast.plan | 4 +-- ...uterJoinPredicationCaseByCase3.1.Sort.plan | 4 +-- ...icationCaseByCase3.1.Sort_NoBroadcast.plan | 4 +-- .../testOuterJoinAndCaseWhen1.Hash.plan | 4 +-- ...uterJoinAndCaseWhen1.Hash_NoBroadcast.plan | 4 +-- .../testOuterJoinAndCaseWhen1.Sort.plan | 4 +-- ...uterJoinAndCaseWhen1.Sort_NoBroadcast.plan | 4 +-- ...uterJoinPredicationCaseByCase3.1.Hash.plan | 4 +-- ...icationCaseByCase3.1.Hash_NoBroadcast.plan | 4 +-- ...uterJoinPredicationCaseByCase3.1.Sort.plan | 4 +-- ...icationCaseByCase3.1.Sort_NoBroadcast.plan | 4 +-- .../results/TestTPCH/testFirstJoinInQ7.plan | 16 ++++----- .../org/apache/tajo/plan/expr/EvalNode.java | 1 - .../org/apache/tajo/plan/expr/FieldEval.java | 14 ++++---- .../apache/tajo/plan/logical/JoinSpec.java | 6 ++-- 28 files changed, 91 insertions(+), 65 deletions(-) diff --git a/tajo-common/src/main/java/org/apache/tajo/schema/Field.java b/tajo-common/src/main/java/org/apache/tajo/schema/Field.java index 803236dfed..24909bb349 100644 --- a/tajo-common/src/main/java/org/apache/tajo/schema/Field.java +++ b/tajo-common/src/main/java/org/apache/tajo/schema/Field.java @@ -77,7 +77,7 @@ public boolean isNull() { @Override public String toString() { - return name + " " + type; + return name + " (" + type + ")"; } @Override @@ -87,16 +87,13 @@ public int hashCode() { @Override public boolean equals(Object obj) { - if (this == obj) { return true; } if (obj instanceof Field) { Field other = (Field) obj; - boolean eq = type.equals(other.type); - eq &= name.equals(other.name); - return eq; + return type.equals(other.type) && name.equals(other.name); } return false; diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Array.java b/tajo-common/src/main/java/org/apache/tajo/type/Array.java index 9a595fd0b8..b6cf069329 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Array.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Array.java @@ -45,7 +45,7 @@ public TajoDataTypes.Type baseType() { @Override public String toString() { - return "array<" + elementType + ">"; + return "ARRAY<" + elementType + ">"; } @Override diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Map.java b/tajo-common/src/main/java/org/apache/tajo/type/Map.java index afc1cc38f9..1978cb6434 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Map.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Map.java @@ -46,7 +46,7 @@ public TajoDataTypes.Type baseType() { @Override public String toString() { - return "map<" + keyType + "," + valueType + ">"; + return "MAP<" + keyType + "," + valueType + ">"; } @Override diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Record.java b/tajo-common/src/main/java/org/apache/tajo/type/Record.java index ee8e2e9cae..e8fb6a83ba 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Record.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Record.java @@ -53,7 +53,7 @@ public TajoDataTypes.Type baseType() { @Override public String toString() { - return "record(" + StringUtils.join(memberTypes, ", ") + ")"; + return "RECORD(" + StringUtils.join(memberTypes, ", ") + ")"; } @Override diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Type.java b/tajo-common/src/main/java/org/apache/tajo/type/Type.java index 59be3e6ea7..89f9b8dc58 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Type.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Type.java @@ -51,7 +51,7 @@ public boolean hasParam() { } protected static String typeName(TajoDataTypes.Type type) { - return type.name().toLowerCase(); + return type.name().toUpperCase(); } @Override diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestTajoCliNegatives.java b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestTajoCliNegatives.java index 0fd29d9307..b5d2e3795b 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestTajoCliNegatives.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestTajoCliNegatives.java @@ -124,7 +124,7 @@ public void testQueryNotImplementedFeature() throws Exception { client.updateQuery("CREATE TABLE TestTajoCliNegatives.table12u79 ( name RECORD(last TEXT, first TEXT) )"); assertScriptFailure("select name FROM TestTajoCliNegatives.table12u79", - "ERROR: org.apache.tajo.exception.NotImplementedException: not implemented feature: record projection\n"); + "ERROR: org.apache.tajo.exception.NotImplementedException: not implemented feature: record type projection\n"); } finally { client.updateQuery("DROP TABLE IF EXISTS TestTajoCliNegatives.table12u79"); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/TestEvalTree.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/TestEvalTree.java index 26e0615e52..22e8e56127 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/TestEvalTree.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/TestEvalTree.java @@ -18,10 +18,7 @@ package org.apache.tajo.engine.eval; -import org.apache.tajo.catalog.CatalogUtil; -import org.apache.tajo.catalog.Column; -import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.SchemaBuilder; +import org.apache.tajo.catalog.*; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.DatumFactory; @@ -32,6 +29,7 @@ import org.junit.Test; import static org.apache.tajo.common.TajoDataTypes.Type.INT4; +import static org.apache.tajo.common.TajoDataTypes.Type.TEXT; import static org.junit.Assert.*; public class TestEvalTree extends ExprTestBase { @@ -385,4 +383,34 @@ private void assertCloneEqual(EvalNode eval) throws CloneNotSupportedException { assertEquals(eval, copy); assertFalse(eval == copy); } + + @Test + public final void testHashAndEqual() { + EvalNode e1 = new BinaryEval(EvalType.EQUAL, + new FieldEval("default.n1.n_nationkey", TypeConverter.convert(INT4)), + new FieldEval("default.n2.n_nationkey", TypeConverter.convert(INT4)) + ); + + EvalNode e2 = new BinaryEval(EvalType.EQUAL, + new FieldEval("default.n1.n_name", TypeConverter.convert(INT4)), + new FieldEval("default.n2.n_name", TypeConverter.convert(INT4)) + ); + + EvalNode e3 = new BinaryEval(EvalType.EQUAL, + new FieldEval("default.n1.n_regionkey", TypeConverter.convert(INT4)), + new FieldEval("default.n2.n_regionkey", TypeConverter.convert(INT4)) + ); + + EvalNode e4 = new BinaryEval(EvalType.EQUAL, + new FieldEval("default.n1.n_comment", TypeConverter.convert(TEXT)), + new FieldEval("default.n2.n_comment", TypeConverter.convert(TEXT)) + ); + + FieldEval f1 = new FieldEval("default.n1.n_nationkey", TypeConverter.convert(INT4)); + System.out.println(f1.hashCode()); + System.out.println(e1.hashCode()); + + System.out.println(">>>>" + e1.hashCode() + " >>>>" + e2.hashCode() + " >>>>" + e3.hashCode() + " >>>> " + e4.hashCode()); + System.out.println(">>>>" + e1.hashCode() + " >>>>" + e2.hashCode() + " >>>>" + e3.hashCode() + " >>>> " + e4.hashCode()); + } } diff --git a/tajo-core-tests/src/test/resources/results/TestCommonConditionReduce/test51.plan b/tajo-core-tests/src/test/resources/results/TestCommonConditionReduce/test51.plan index 5f1d6c0f0d..9dd7b846ae 100644 --- a/tajo-core-tests/src/test/resources/results/TestCommonConditionReduce/test51.plan +++ b/tajo-core-tests/src/test/resources/results/TestCommonConditionReduce/test51.plan @@ -1,7 +1,7 @@ explain ------------------------------- SCAN(0) on default.lineitem - => filter: (default.lineitem.l_linenumber (INT4) = 1 AND (default.lineitem.l_orderkey (INT4) = 1 AND (default.lineitem.l_suppkey (INT4) = 7706 AND (default.lineitem.l_comment (TEXT) IS NOT NULL AND default.lineitem.l_commitdate (TEXT) IS NOT NULL)))) + => filter: (default.lineitem.l_orderkey (INT4) = 1 AND (default.lineitem.l_linenumber (INT4) = 1 AND (default.lineitem.l_suppkey (INT4) = 7706 AND (default.lineitem.l_commitdate (TEXT) IS NOT NULL AND default.lineitem.l_comment (TEXT) IS NOT NULL)))) => target list: default.lineitem.l_comment (TEXT), default.lineitem.l_commitdate (TEXT), default.lineitem.l_discount (FLOAT8), default.lineitem.l_extendedprice (FLOAT8), default.lineitem.l_linenumber (INT4), default.lineitem.l_linestatus (TEXT), default.lineitem.l_orderkey (INT4), default.lineitem.l_partkey (INT4), default.lineitem.l_quantity (FLOAT8), default.lineitem.l_receiptdate (TEXT), default.lineitem.l_returnflag (TEXT), default.lineitem.l_shipdate (TEXT), default.lineitem.l_shipinstruct (TEXT), default.lineitem.l_shipmode (TEXT), default.lineitem.l_suppkey (INT4), default.lineitem.l_tax (FLOAT8) => out schema: {(16) default.lineitem.l_comment (TEXT), default.lineitem.l_commitdate (TEXT), default.lineitem.l_discount (FLOAT8), default.lineitem.l_extendedprice (FLOAT8), default.lineitem.l_linenumber (INT4), default.lineitem.l_linestatus (TEXT), default.lineitem.l_orderkey (INT4), default.lineitem.l_partkey (INT4), default.lineitem.l_quantity (FLOAT8), default.lineitem.l_receiptdate (TEXT), default.lineitem.l_returnflag (TEXT), default.lineitem.l_shipdate (TEXT), default.lineitem.l_shipinstruct (TEXT), default.lineitem.l_shipmode (TEXT), default.lineitem.l_suppkey (INT4), default.lineitem.l_tax (FLOAT8)} => in schema: {(16) default.lineitem.l_comment (TEXT), default.lineitem.l_commitdate (TEXT), default.lineitem.l_discount (FLOAT8), default.lineitem.l_extendedprice (FLOAT8), default.lineitem.l_linenumber (INT4), default.lineitem.l_linestatus (TEXT), default.lineitem.l_orderkey (INT4), default.lineitem.l_partkey (INT4), default.lineitem.l_quantity (FLOAT8), default.lineitem.l_receiptdate (TEXT), default.lineitem.l_returnflag (TEXT), default.lineitem.l_shipdate (TEXT), default.lineitem.l_shipinstruct (TEXT), default.lineitem.l_shipmode (TEXT), default.lineitem.l_suppkey (INT4), default.lineitem.l_tax (FLOAT8)} diff --git a/tajo-core-tests/src/test/resources/results/TestInnerJoinWithSubQuery/testJoinWithMultipleJoinQual2.Hash.plan b/tajo-core-tests/src/test/resources/results/TestInnerJoinWithSubQuery/testJoinWithMultipleJoinQual2.Hash.plan index 7848e95d5a..5558eb974b 100644 --- a/tajo-core-tests/src/test/resources/results/TestInnerJoinWithSubQuery/testJoinWithMultipleJoinQual2.Hash.plan +++ b/tajo-core-tests/src/test/resources/results/TestInnerJoinWithSubQuery/testJoinWithMultipleJoinQual2.Hash.plan @@ -20,7 +20,7 @@ JOIN(11)(INNER) => in schema: {(3) default.r.r_comment (TEXT), default.r.r_name (TEXT), default.r.r_regionkey (INT4)} SCAN(0) on default.nation as n => filter: default.n.n_name (TEXT) = MOROCCO - => target list: default.n.n_comment (TEXT), default.n.n_name (TEXT) as name, default.n.n_nationkey (INT4), default.n.n_regionkey (INT4) + => target list: default.n.n_comment (TEXT), default.n.n_nationkey (INT4), default.n.n_regionkey (INT4), default.n.n_name (TEXT) as name => out schema: {(4) default.n.n_comment (TEXT), default.n.n_nationkey (INT4), default.n.n_regionkey (INT4), name (TEXT)} => in schema: {(4) default.n.n_comment (TEXT), default.n.n_name (TEXT), default.n.n_nationkey (INT4), default.n.n_regionkey (INT4)} SCAN(5) on default.supplier as s @@ -73,7 +73,7 @@ JOIN(11)(INNER) => in schema: {(3) default.r.r_comment (TEXT), default.r.r_name (TEXT), default.r.r_regionkey (INT4)} SCAN(0) on default.nation as n => filter: default.n.n_name (TEXT) = MOROCCO - => target list: default.n.n_comment (TEXT), default.n.n_name (TEXT) as name, default.n.n_nationkey (INT4), default.n.n_regionkey (INT4) + => target list: default.n.n_comment (TEXT), default.n.n_nationkey (INT4), default.n.n_regionkey (INT4), default.n.n_name (TEXT) as name => out schema: {(4) default.n.n_comment (TEXT), default.n.n_nationkey (INT4), default.n.n_regionkey (INT4), name (TEXT)} => in schema: {(4) default.n.n_comment (TEXT), default.n.n_name (TEXT), default.n.n_nationkey (INT4), default.n.n_regionkey (INT4)} SCAN(5) on default.supplier as s diff --git a/tajo-core-tests/src/test/resources/results/TestInnerJoinWithSubQuery/testJoinWithMultipleJoinQual2.Hash_NoBroadcast.plan b/tajo-core-tests/src/test/resources/results/TestInnerJoinWithSubQuery/testJoinWithMultipleJoinQual2.Hash_NoBroadcast.plan index 2f7da9984d..65f8b1c597 100644 --- a/tajo-core-tests/src/test/resources/results/TestInnerJoinWithSubQuery/testJoinWithMultipleJoinQual2.Hash_NoBroadcast.plan +++ b/tajo-core-tests/src/test/resources/results/TestInnerJoinWithSubQuery/testJoinWithMultipleJoinQual2.Hash_NoBroadcast.plan @@ -20,7 +20,7 @@ JOIN(11)(INNER) => in schema: {(3) default.r.r_comment (TEXT), default.r.r_name (TEXT), default.r.r_regionkey (INT4)} SCAN(0) on default.nation as n => filter: default.n.n_name (TEXT) = MOROCCO - => target list: default.n.n_comment (TEXT), default.n.n_name (TEXT) as name, default.n.n_nationkey (INT4), default.n.n_regionkey (INT4) + => target list: default.n.n_comment (TEXT), default.n.n_nationkey (INT4), default.n.n_regionkey (INT4), default.n.n_name (TEXT) as name => out schema: {(4) default.n.n_comment (TEXT), default.n.n_nationkey (INT4), default.n.n_regionkey (INT4), name (TEXT)} => in schema: {(4) default.n.n_comment (TEXT), default.n.n_name (TEXT), default.n.n_nationkey (INT4), default.n.n_regionkey (INT4)} SCAN(5) on default.supplier as s @@ -58,7 +58,7 @@ Block Id: eb_0000000000000_0000_000002 [LEAF] SCAN(0) on default.nation as n => filter: default.n.n_name (TEXT) = MOROCCO - => target list: default.n.n_comment (TEXT), default.n.n_name (TEXT) as name, default.n.n_nationkey (INT4), default.n.n_regionkey (INT4) + => target list: default.n.n_comment (TEXT), default.n.n_nationkey (INT4), default.n.n_regionkey (INT4), default.n.n_name (TEXT) as name => out schema: {(4) default.n.n_comment (TEXT), default.n.n_nationkey (INT4), default.n.n_regionkey (INT4), name (TEXT)} => in schema: {(4) default.n.n_comment (TEXT), default.n.n_name (TEXT), default.n.n_nationkey (INT4), default.n.n_regionkey (INT4)} diff --git a/tajo-core-tests/src/test/resources/results/TestInnerJoinWithSubQuery/testJoinWithMultipleJoinQual2.Sort.plan b/tajo-core-tests/src/test/resources/results/TestInnerJoinWithSubQuery/testJoinWithMultipleJoinQual2.Sort.plan index 7848e95d5a..5558eb974b 100644 --- a/tajo-core-tests/src/test/resources/results/TestInnerJoinWithSubQuery/testJoinWithMultipleJoinQual2.Sort.plan +++ b/tajo-core-tests/src/test/resources/results/TestInnerJoinWithSubQuery/testJoinWithMultipleJoinQual2.Sort.plan @@ -20,7 +20,7 @@ JOIN(11)(INNER) => in schema: {(3) default.r.r_comment (TEXT), default.r.r_name (TEXT), default.r.r_regionkey (INT4)} SCAN(0) on default.nation as n => filter: default.n.n_name (TEXT) = MOROCCO - => target list: default.n.n_comment (TEXT), default.n.n_name (TEXT) as name, default.n.n_nationkey (INT4), default.n.n_regionkey (INT4) + => target list: default.n.n_comment (TEXT), default.n.n_nationkey (INT4), default.n.n_regionkey (INT4), default.n.n_name (TEXT) as name => out schema: {(4) default.n.n_comment (TEXT), default.n.n_nationkey (INT4), default.n.n_regionkey (INT4), name (TEXT)} => in schema: {(4) default.n.n_comment (TEXT), default.n.n_name (TEXT), default.n.n_nationkey (INT4), default.n.n_regionkey (INT4)} SCAN(5) on default.supplier as s @@ -73,7 +73,7 @@ JOIN(11)(INNER) => in schema: {(3) default.r.r_comment (TEXT), default.r.r_name (TEXT), default.r.r_regionkey (INT4)} SCAN(0) on default.nation as n => filter: default.n.n_name (TEXT) = MOROCCO - => target list: default.n.n_comment (TEXT), default.n.n_name (TEXT) as name, default.n.n_nationkey (INT4), default.n.n_regionkey (INT4) + => target list: default.n.n_comment (TEXT), default.n.n_nationkey (INT4), default.n.n_regionkey (INT4), default.n.n_name (TEXT) as name => out schema: {(4) default.n.n_comment (TEXT), default.n.n_nationkey (INT4), default.n.n_regionkey (INT4), name (TEXT)} => in schema: {(4) default.n.n_comment (TEXT), default.n.n_name (TEXT), default.n.n_nationkey (INT4), default.n.n_regionkey (INT4)} SCAN(5) on default.supplier as s diff --git a/tajo-core-tests/src/test/resources/results/TestInnerJoinWithSubQuery/testJoinWithMultipleJoinQual2.Sort_NoBroadcast.plan b/tajo-core-tests/src/test/resources/results/TestInnerJoinWithSubQuery/testJoinWithMultipleJoinQual2.Sort_NoBroadcast.plan index 2f7da9984d..65f8b1c597 100644 --- a/tajo-core-tests/src/test/resources/results/TestInnerJoinWithSubQuery/testJoinWithMultipleJoinQual2.Sort_NoBroadcast.plan +++ b/tajo-core-tests/src/test/resources/results/TestInnerJoinWithSubQuery/testJoinWithMultipleJoinQual2.Sort_NoBroadcast.plan @@ -20,7 +20,7 @@ JOIN(11)(INNER) => in schema: {(3) default.r.r_comment (TEXT), default.r.r_name (TEXT), default.r.r_regionkey (INT4)} SCAN(0) on default.nation as n => filter: default.n.n_name (TEXT) = MOROCCO - => target list: default.n.n_comment (TEXT), default.n.n_name (TEXT) as name, default.n.n_nationkey (INT4), default.n.n_regionkey (INT4) + => target list: default.n.n_comment (TEXT), default.n.n_nationkey (INT4), default.n.n_regionkey (INT4), default.n.n_name (TEXT) as name => out schema: {(4) default.n.n_comment (TEXT), default.n.n_nationkey (INT4), default.n.n_regionkey (INT4), name (TEXT)} => in schema: {(4) default.n.n_comment (TEXT), default.n.n_name (TEXT), default.n.n_nationkey (INT4), default.n.n_regionkey (INT4)} SCAN(5) on default.supplier as s @@ -58,7 +58,7 @@ Block Id: eb_0000000000000_0000_000002 [LEAF] SCAN(0) on default.nation as n => filter: default.n.n_name (TEXT) = MOROCCO - => target list: default.n.n_comment (TEXT), default.n.n_name (TEXT) as name, default.n.n_nationkey (INT4), default.n.n_regionkey (INT4) + => target list: default.n.n_comment (TEXT), default.n.n_nationkey (INT4), default.n.n_regionkey (INT4), default.n.n_name (TEXT) as name => out schema: {(4) default.n.n_comment (TEXT), default.n.n_nationkey (INT4), default.n.n_regionkey (INT4), name (TEXT)} => in schema: {(4) default.n.n_comment (TEXT), default.n.n_name (TEXT), default.n.n_nationkey (INT4), default.n.n_regionkey (INT4)} diff --git a/tajo-core-tests/src/test/resources/results/TestOuterJoinQuery/testLeftOuterJoinPredicationCaseByCase3.1.Hash.plan b/tajo-core-tests/src/test/resources/results/TestOuterJoinQuery/testLeftOuterJoinPredicationCaseByCase3.1.Hash.plan index d5e35ee5d2..785a0eb66d 100644 --- a/tajo-core-tests/src/test/resources/results/TestOuterJoinQuery/testLeftOuterJoinPredicationCaseByCase3.1.Hash.plan +++ b/tajo-core-tests/src/test/resources/results/TestOuterJoinQuery/testLeftOuterJoinPredicationCaseByCase3.1.Hash.plan @@ -10,7 +10,7 @@ JOIN(8)(LEFT_OUTER) => out schema: {(1) default.t3.id (INT4)} => in schema: {(2) default.t3.id (INT4), default.t3.name (TEXT)} JOIN(7)(LEFT_OUTER) - => Join Cond: (default.t1.id (INT4) = default.t2.id (INT4) AND (concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-11 OR concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-33)) + => Join Cond: ((concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-11 OR concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-33) AND default.t1.id (INT4) = default.t2.id (INT4)) => target list: default.t1.id (INT4), default.t1.name (TEXT), default.t2.id (INT4) => out schema: {(3) default.t1.id (INT4), default.t1.name (TEXT), default.t2.id (INT4)} => in schema: {(3) default.t1.id (INT4), default.t1.name (TEXT), default.t2.id (INT4)} @@ -54,7 +54,7 @@ JOIN(8)(LEFT_OUTER) => out schema: {(1) default.t3.id (INT4)} => in schema: {(2) default.t3.id (INT4), default.t3.name (TEXT)} JOIN(7)(LEFT_OUTER) - => Join Cond: (default.t1.id (INT4) = default.t2.id (INT4) AND (concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-11 OR concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-33)) + => Join Cond: ((concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-11 OR concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-33) AND default.t1.id (INT4) = default.t2.id (INT4)) => target list: default.t1.id (INT4), default.t1.name (TEXT), default.t2.id (INT4) => out schema: {(3) default.t1.id (INT4), default.t1.name (TEXT), default.t2.id (INT4)} => in schema: {(3) default.t1.id (INT4), default.t1.name (TEXT), default.t2.id (INT4)} diff --git a/tajo-core-tests/src/test/resources/results/TestOuterJoinQuery/testLeftOuterJoinPredicationCaseByCase3.1.Hash_NoBroadcast.plan b/tajo-core-tests/src/test/resources/results/TestOuterJoinQuery/testLeftOuterJoinPredicationCaseByCase3.1.Hash_NoBroadcast.plan index d1a96c9f66..cd1484dc5d 100644 --- a/tajo-core-tests/src/test/resources/results/TestOuterJoinQuery/testLeftOuterJoinPredicationCaseByCase3.1.Hash_NoBroadcast.plan +++ b/tajo-core-tests/src/test/resources/results/TestOuterJoinQuery/testLeftOuterJoinPredicationCaseByCase3.1.Hash_NoBroadcast.plan @@ -10,7 +10,7 @@ JOIN(8)(LEFT_OUTER) => out schema: {(1) default.t3.id (INT4)} => in schema: {(2) default.t3.id (INT4), default.t3.name (TEXT)} JOIN(7)(LEFT_OUTER) - => Join Cond: (default.t1.id (INT4) = default.t2.id (INT4) AND (concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-11 OR concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-33)) + => Join Cond: ((concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-11 OR concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-33) AND default.t1.id (INT4) = default.t2.id (INT4)) => target list: default.t1.id (INT4), default.t1.name (TEXT), default.t2.id (INT4) => out schema: {(3) default.t1.id (INT4), default.t1.name (TEXT), default.t2.id (INT4)} => in schema: {(3) default.t1.id (INT4), default.t1.name (TEXT), default.t2.id (INT4)} @@ -80,7 +80,7 @@ Block Id: eb_0000000000000_0000_000003 [INTERMEDIATE] [q_0000000000000_0000] 3 => 5 (type=HASH_SHUFFLE, key=default.t1.id (INT4), num=32) JOIN(7)(LEFT_OUTER) - => Join Cond: (default.t1.id (INT4) = default.t2.id (INT4) AND (concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-11 OR concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-33)) + => Join Cond: ((concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-11 OR concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-33) AND default.t1.id (INT4) = default.t2.id (INT4)) => target list: default.t1.id (INT4), default.t1.name (TEXT), default.t2.id (INT4) => out schema: {(3) default.t1.id (INT4), default.t1.name (TEXT), default.t2.id (INT4)} => in schema: {(3) default.t1.id (INT4), default.t1.name (TEXT), default.t2.id (INT4)} diff --git a/tajo-core-tests/src/test/resources/results/TestOuterJoinQuery/testLeftOuterJoinPredicationCaseByCase3.1.Sort.plan b/tajo-core-tests/src/test/resources/results/TestOuterJoinQuery/testLeftOuterJoinPredicationCaseByCase3.1.Sort.plan index d5e35ee5d2..785a0eb66d 100644 --- a/tajo-core-tests/src/test/resources/results/TestOuterJoinQuery/testLeftOuterJoinPredicationCaseByCase3.1.Sort.plan +++ b/tajo-core-tests/src/test/resources/results/TestOuterJoinQuery/testLeftOuterJoinPredicationCaseByCase3.1.Sort.plan @@ -10,7 +10,7 @@ JOIN(8)(LEFT_OUTER) => out schema: {(1) default.t3.id (INT4)} => in schema: {(2) default.t3.id (INT4), default.t3.name (TEXT)} JOIN(7)(LEFT_OUTER) - => Join Cond: (default.t1.id (INT4) = default.t2.id (INT4) AND (concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-11 OR concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-33)) + => Join Cond: ((concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-11 OR concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-33) AND default.t1.id (INT4) = default.t2.id (INT4)) => target list: default.t1.id (INT4), default.t1.name (TEXT), default.t2.id (INT4) => out schema: {(3) default.t1.id (INT4), default.t1.name (TEXT), default.t2.id (INT4)} => in schema: {(3) default.t1.id (INT4), default.t1.name (TEXT), default.t2.id (INT4)} @@ -54,7 +54,7 @@ JOIN(8)(LEFT_OUTER) => out schema: {(1) default.t3.id (INT4)} => in schema: {(2) default.t3.id (INT4), default.t3.name (TEXT)} JOIN(7)(LEFT_OUTER) - => Join Cond: (default.t1.id (INT4) = default.t2.id (INT4) AND (concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-11 OR concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-33)) + => Join Cond: ((concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-11 OR concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-33) AND default.t1.id (INT4) = default.t2.id (INT4)) => target list: default.t1.id (INT4), default.t1.name (TEXT), default.t2.id (INT4) => out schema: {(3) default.t1.id (INT4), default.t1.name (TEXT), default.t2.id (INT4)} => in schema: {(3) default.t1.id (INT4), default.t1.name (TEXT), default.t2.id (INT4)} diff --git a/tajo-core-tests/src/test/resources/results/TestOuterJoinQuery/testLeftOuterJoinPredicationCaseByCase3.1.Sort_NoBroadcast.plan b/tajo-core-tests/src/test/resources/results/TestOuterJoinQuery/testLeftOuterJoinPredicationCaseByCase3.1.Sort_NoBroadcast.plan index d1a96c9f66..cd1484dc5d 100644 --- a/tajo-core-tests/src/test/resources/results/TestOuterJoinQuery/testLeftOuterJoinPredicationCaseByCase3.1.Sort_NoBroadcast.plan +++ b/tajo-core-tests/src/test/resources/results/TestOuterJoinQuery/testLeftOuterJoinPredicationCaseByCase3.1.Sort_NoBroadcast.plan @@ -10,7 +10,7 @@ JOIN(8)(LEFT_OUTER) => out schema: {(1) default.t3.id (INT4)} => in schema: {(2) default.t3.id (INT4), default.t3.name (TEXT)} JOIN(7)(LEFT_OUTER) - => Join Cond: (default.t1.id (INT4) = default.t2.id (INT4) AND (concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-11 OR concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-33)) + => Join Cond: ((concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-11 OR concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-33) AND default.t1.id (INT4) = default.t2.id (INT4)) => target list: default.t1.id (INT4), default.t1.name (TEXT), default.t2.id (INT4) => out schema: {(3) default.t1.id (INT4), default.t1.name (TEXT), default.t2.id (INT4)} => in schema: {(3) default.t1.id (INT4), default.t1.name (TEXT), default.t2.id (INT4)} @@ -80,7 +80,7 @@ Block Id: eb_0000000000000_0000_000003 [INTERMEDIATE] [q_0000000000000_0000] 3 => 5 (type=HASH_SHUFFLE, key=default.t1.id (INT4), num=32) JOIN(7)(LEFT_OUTER) - => Join Cond: (default.t1.id (INT4) = default.t2.id (INT4) AND (concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-11 OR concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-33)) + => Join Cond: ((concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-11 OR concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-33) AND default.t1.id (INT4) = default.t2.id (INT4)) => target list: default.t1.id (INT4), default.t1.name (TEXT), default.t2.id (INT4) => out schema: {(3) default.t1.id (INT4), default.t1.name (TEXT), default.t2.id (INT4)} => in schema: {(3) default.t1.id (INT4), default.t1.name (TEXT), default.t2.id (INT4)} diff --git a/tajo-core-tests/src/test/resources/results/TestOuterJoinQuery/testOuterJoinAndCaseWhen1.Hash.plan b/tajo-core-tests/src/test/resources/results/TestOuterJoinQuery/testOuterJoinAndCaseWhen1.Hash.plan index 43fc9f740e..87b13bfa48 100644 --- a/tajo-core-tests/src/test/resources/results/TestOuterJoinQuery/testOuterJoinAndCaseWhen1.Hash.plan +++ b/tajo-core-tests/src/test/resources/results/TestOuterJoinQuery/testOuterJoinAndCaseWhen1.Hash.plan @@ -17,7 +17,7 @@ SORT(5) => out schema: {(2) default.c.id (INT4), default.c.name (TEXT)} => in schema: {(4) default.c.id (INT4), default.c.name (TEXT), default.c.score (FLOAT4), default.c.type (TEXT)} SCAN(1) on default.testouterjoinandcasewhen2 as b - => target list: default.b.id (INT4), default.b.id (INT4) as id2, default.b.name (TEXT), default.b.name (TEXT) as name2 + => target list: default.b.id (INT4), default.b.name (TEXT), default.b.id (INT4) as id2, default.b.name (TEXT) as name2 => out schema: {(4) default.b.id (INT4), default.b.name (TEXT), id2 (INT4), name2 (TEXT)} => in schema: {(4) default.b.id (INT4), default.b.name (TEXT), default.b.score (FLOAT4), default.b.type (TEXT)} SCAN(0) on default.testouterjoinandcasewhen1 as a @@ -68,7 +68,7 @@ SORT(15) => out schema: {(2) default.c.id (INT4), default.c.name (TEXT)} => in schema: {(4) default.c.id (INT4), default.c.name (TEXT), default.c.score (FLOAT4), default.c.type (TEXT)} SCAN(1) on default.testouterjoinandcasewhen2 as b - => target list: default.b.id (INT4), default.b.id (INT4) as id2, default.b.name (TEXT), default.b.name (TEXT) as name2 + => target list: default.b.id (INT4), default.b.name (TEXT), default.b.id (INT4) as id2, default.b.name (TEXT) as name2 => out schema: {(4) default.b.id (INT4), default.b.name (TEXT), id2 (INT4), name2 (TEXT)} => in schema: {(4) default.b.id (INT4), default.b.name (TEXT), default.b.score (FLOAT4), default.b.type (TEXT)} SCAN(0) on default.testouterjoinandcasewhen1 as a diff --git a/tajo-core-tests/src/test/resources/results/TestOuterJoinQuery/testOuterJoinAndCaseWhen1.Hash_NoBroadcast.plan b/tajo-core-tests/src/test/resources/results/TestOuterJoinQuery/testOuterJoinAndCaseWhen1.Hash_NoBroadcast.plan index 749c8c0b5b..4134f9ba60 100644 --- a/tajo-core-tests/src/test/resources/results/TestOuterJoinQuery/testOuterJoinAndCaseWhen1.Hash_NoBroadcast.plan +++ b/tajo-core-tests/src/test/resources/results/TestOuterJoinQuery/testOuterJoinAndCaseWhen1.Hash_NoBroadcast.plan @@ -17,7 +17,7 @@ SORT(5) => out schema: {(2) default.c.id (INT4), default.c.name (TEXT)} => in schema: {(4) default.c.id (INT4), default.c.name (TEXT), default.c.score (FLOAT4), default.c.type (TEXT)} SCAN(1) on default.testouterjoinandcasewhen2 as b - => target list: default.b.id (INT4), default.b.id (INT4) as id2, default.b.name (TEXT), default.b.name (TEXT) as name2 + => target list: default.b.id (INT4), default.b.name (TEXT), default.b.id (INT4) as id2, default.b.name (TEXT) as name2 => out schema: {(4) default.b.id (INT4), default.b.name (TEXT), id2 (INT4), name2 (TEXT)} => in schema: {(4) default.b.id (INT4), default.b.name (TEXT), default.b.score (FLOAT4), default.b.type (TEXT)} SCAN(0) on default.testouterjoinandcasewhen1 as a @@ -56,7 +56,7 @@ Block Id: eb_0000000000000_0000_000002 [LEAF] [q_0000000000000_0000] 2 => 4 (type=HASH_SHUFFLE, key=default.b.id (INT4), num=32) SCAN(1) on default.testouterjoinandcasewhen2 as b - => target list: default.b.id (INT4), default.b.id (INT4) as id2, default.b.name (TEXT), default.b.name (TEXT) as name2 + => target list: default.b.id (INT4), default.b.name (TEXT), default.b.id (INT4) as id2, default.b.name (TEXT) as name2 => out schema: {(4) default.b.id (INT4), default.b.name (TEXT), id2 (INT4), name2 (TEXT)} => in schema: {(4) default.b.id (INT4), default.b.name (TEXT), default.b.score (FLOAT4), default.b.type (TEXT)} diff --git a/tajo-core-tests/src/test/resources/results/TestOuterJoinQuery/testOuterJoinAndCaseWhen1.Sort.plan b/tajo-core-tests/src/test/resources/results/TestOuterJoinQuery/testOuterJoinAndCaseWhen1.Sort.plan index 43fc9f740e..87b13bfa48 100644 --- a/tajo-core-tests/src/test/resources/results/TestOuterJoinQuery/testOuterJoinAndCaseWhen1.Sort.plan +++ b/tajo-core-tests/src/test/resources/results/TestOuterJoinQuery/testOuterJoinAndCaseWhen1.Sort.plan @@ -17,7 +17,7 @@ SORT(5) => out schema: {(2) default.c.id (INT4), default.c.name (TEXT)} => in schema: {(4) default.c.id (INT4), default.c.name (TEXT), default.c.score (FLOAT4), default.c.type (TEXT)} SCAN(1) on default.testouterjoinandcasewhen2 as b - => target list: default.b.id (INT4), default.b.id (INT4) as id2, default.b.name (TEXT), default.b.name (TEXT) as name2 + => target list: default.b.id (INT4), default.b.name (TEXT), default.b.id (INT4) as id2, default.b.name (TEXT) as name2 => out schema: {(4) default.b.id (INT4), default.b.name (TEXT), id2 (INT4), name2 (TEXT)} => in schema: {(4) default.b.id (INT4), default.b.name (TEXT), default.b.score (FLOAT4), default.b.type (TEXT)} SCAN(0) on default.testouterjoinandcasewhen1 as a @@ -68,7 +68,7 @@ SORT(15) => out schema: {(2) default.c.id (INT4), default.c.name (TEXT)} => in schema: {(4) default.c.id (INT4), default.c.name (TEXT), default.c.score (FLOAT4), default.c.type (TEXT)} SCAN(1) on default.testouterjoinandcasewhen2 as b - => target list: default.b.id (INT4), default.b.id (INT4) as id2, default.b.name (TEXT), default.b.name (TEXT) as name2 + => target list: default.b.id (INT4), default.b.name (TEXT), default.b.id (INT4) as id2, default.b.name (TEXT) as name2 => out schema: {(4) default.b.id (INT4), default.b.name (TEXT), id2 (INT4), name2 (TEXT)} => in schema: {(4) default.b.id (INT4), default.b.name (TEXT), default.b.score (FLOAT4), default.b.type (TEXT)} SCAN(0) on default.testouterjoinandcasewhen1 as a diff --git a/tajo-core-tests/src/test/resources/results/TestOuterJoinQuery/testOuterJoinAndCaseWhen1.Sort_NoBroadcast.plan b/tajo-core-tests/src/test/resources/results/TestOuterJoinQuery/testOuterJoinAndCaseWhen1.Sort_NoBroadcast.plan index 749c8c0b5b..4134f9ba60 100644 --- a/tajo-core-tests/src/test/resources/results/TestOuterJoinQuery/testOuterJoinAndCaseWhen1.Sort_NoBroadcast.plan +++ b/tajo-core-tests/src/test/resources/results/TestOuterJoinQuery/testOuterJoinAndCaseWhen1.Sort_NoBroadcast.plan @@ -17,7 +17,7 @@ SORT(5) => out schema: {(2) default.c.id (INT4), default.c.name (TEXT)} => in schema: {(4) default.c.id (INT4), default.c.name (TEXT), default.c.score (FLOAT4), default.c.type (TEXT)} SCAN(1) on default.testouterjoinandcasewhen2 as b - => target list: default.b.id (INT4), default.b.id (INT4) as id2, default.b.name (TEXT), default.b.name (TEXT) as name2 + => target list: default.b.id (INT4), default.b.name (TEXT), default.b.id (INT4) as id2, default.b.name (TEXT) as name2 => out schema: {(4) default.b.id (INT4), default.b.name (TEXT), id2 (INT4), name2 (TEXT)} => in schema: {(4) default.b.id (INT4), default.b.name (TEXT), default.b.score (FLOAT4), default.b.type (TEXT)} SCAN(0) on default.testouterjoinandcasewhen1 as a @@ -56,7 +56,7 @@ Block Id: eb_0000000000000_0000_000002 [LEAF] [q_0000000000000_0000] 2 => 4 (type=HASH_SHUFFLE, key=default.b.id (INT4), num=32) SCAN(1) on default.testouterjoinandcasewhen2 as b - => target list: default.b.id (INT4), default.b.id (INT4) as id2, default.b.name (TEXT), default.b.name (TEXT) as name2 + => target list: default.b.id (INT4), default.b.name (TEXT), default.b.id (INT4) as id2, default.b.name (TEXT) as name2 => out schema: {(4) default.b.id (INT4), default.b.name (TEXT), id2 (INT4), name2 (TEXT)} => in schema: {(4) default.b.id (INT4), default.b.name (TEXT), default.b.score (FLOAT4), default.b.type (TEXT)} diff --git a/tajo-core-tests/src/test/resources/results/TestOuterJoinQuery/testRightOuterJoinPredicationCaseByCase3.1.Hash.plan b/tajo-core-tests/src/test/resources/results/TestOuterJoinQuery/testRightOuterJoinPredicationCaseByCase3.1.Hash.plan index 8e7074e40c..0f8ca51e8f 100644 --- a/tajo-core-tests/src/test/resources/results/TestOuterJoinQuery/testRightOuterJoinPredicationCaseByCase3.1.Hash.plan +++ b/tajo-core-tests/src/test/resources/results/TestOuterJoinQuery/testRightOuterJoinPredicationCaseByCase3.1.Hash.plan @@ -10,7 +10,7 @@ JOIN(8)(RIGHT_OUTER) => out schema: {(1) default.t3.id (INT4)} => in schema: {(2) default.t3.id (INT4), default.t3.name (TEXT)} JOIN(7)(RIGHT_OUTER) - => Join Cond: (default.t1.id (INT4) = default.t2.id (INT4) AND (concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-11 OR concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-33)) + => Join Cond: ((concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-11 OR concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-33) AND default.t1.id (INT4) = default.t2.id (INT4)) => target list: default.t1.id (INT4), default.t1.name (TEXT), default.t2.id (INT4) => out schema: {(3) default.t1.id (INT4), default.t1.name (TEXT), default.t2.id (INT4)} => in schema: {(3) default.t1.id (INT4), default.t1.name (TEXT), default.t2.id (INT4)} @@ -54,7 +54,7 @@ JOIN(8)(RIGHT_OUTER) => out schema: {(1) default.t3.id (INT4)} => in schema: {(2) default.t3.id (INT4), default.t3.name (TEXT)} JOIN(7)(RIGHT_OUTER) - => Join Cond: (default.t1.id (INT4) = default.t2.id (INT4) AND (concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-11 OR concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-33)) + => Join Cond: ((concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-11 OR concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-33) AND default.t1.id (INT4) = default.t2.id (INT4)) => target list: default.t1.id (INT4), default.t1.name (TEXT), default.t2.id (INT4) => out schema: {(3) default.t1.id (INT4), default.t1.name (TEXT), default.t2.id (INT4)} => in schema: {(3) default.t1.id (INT4), default.t1.name (TEXT), default.t2.id (INT4)} diff --git a/tajo-core-tests/src/test/resources/results/TestOuterJoinQuery/testRightOuterJoinPredicationCaseByCase3.1.Hash_NoBroadcast.plan b/tajo-core-tests/src/test/resources/results/TestOuterJoinQuery/testRightOuterJoinPredicationCaseByCase3.1.Hash_NoBroadcast.plan index fab38091f8..dfb22875a5 100644 --- a/tajo-core-tests/src/test/resources/results/TestOuterJoinQuery/testRightOuterJoinPredicationCaseByCase3.1.Hash_NoBroadcast.plan +++ b/tajo-core-tests/src/test/resources/results/TestOuterJoinQuery/testRightOuterJoinPredicationCaseByCase3.1.Hash_NoBroadcast.plan @@ -10,7 +10,7 @@ JOIN(8)(RIGHT_OUTER) => out schema: {(1) default.t3.id (INT4)} => in schema: {(2) default.t3.id (INT4), default.t3.name (TEXT)} JOIN(7)(RIGHT_OUTER) - => Join Cond: (default.t1.id (INT4) = default.t2.id (INT4) AND (concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-11 OR concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-33)) + => Join Cond: ((concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-11 OR concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-33) AND default.t1.id (INT4) = default.t2.id (INT4)) => target list: default.t1.id (INT4), default.t1.name (TEXT), default.t2.id (INT4) => out schema: {(3) default.t1.id (INT4), default.t1.name (TEXT), default.t2.id (INT4)} => in schema: {(3) default.t1.id (INT4), default.t1.name (TEXT), default.t2.id (INT4)} @@ -80,7 +80,7 @@ Block Id: eb_0000000000000_0000_000003 [INTERMEDIATE] [q_0000000000000_0000] 3 => 5 (type=HASH_SHUFFLE, key=default.t1.id (INT4), num=32) JOIN(7)(RIGHT_OUTER) - => Join Cond: (default.t1.id (INT4) = default.t2.id (INT4) AND (concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-11 OR concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-33)) + => Join Cond: ((concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-11 OR concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-33) AND default.t1.id (INT4) = default.t2.id (INT4)) => target list: default.t1.id (INT4), default.t1.name (TEXT), default.t2.id (INT4) => out schema: {(3) default.t1.id (INT4), default.t1.name (TEXT), default.t2.id (INT4)} => in schema: {(3) default.t1.id (INT4), default.t1.name (TEXT), default.t2.id (INT4)} diff --git a/tajo-core-tests/src/test/resources/results/TestOuterJoinQuery/testRightOuterJoinPredicationCaseByCase3.1.Sort.plan b/tajo-core-tests/src/test/resources/results/TestOuterJoinQuery/testRightOuterJoinPredicationCaseByCase3.1.Sort.plan index 8e7074e40c..0f8ca51e8f 100644 --- a/tajo-core-tests/src/test/resources/results/TestOuterJoinQuery/testRightOuterJoinPredicationCaseByCase3.1.Sort.plan +++ b/tajo-core-tests/src/test/resources/results/TestOuterJoinQuery/testRightOuterJoinPredicationCaseByCase3.1.Sort.plan @@ -10,7 +10,7 @@ JOIN(8)(RIGHT_OUTER) => out schema: {(1) default.t3.id (INT4)} => in schema: {(2) default.t3.id (INT4), default.t3.name (TEXT)} JOIN(7)(RIGHT_OUTER) - => Join Cond: (default.t1.id (INT4) = default.t2.id (INT4) AND (concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-11 OR concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-33)) + => Join Cond: ((concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-11 OR concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-33) AND default.t1.id (INT4) = default.t2.id (INT4)) => target list: default.t1.id (INT4), default.t1.name (TEXT), default.t2.id (INT4) => out schema: {(3) default.t1.id (INT4), default.t1.name (TEXT), default.t2.id (INT4)} => in schema: {(3) default.t1.id (INT4), default.t1.name (TEXT), default.t2.id (INT4)} @@ -54,7 +54,7 @@ JOIN(8)(RIGHT_OUTER) => out schema: {(1) default.t3.id (INT4)} => in schema: {(2) default.t3.id (INT4), default.t3.name (TEXT)} JOIN(7)(RIGHT_OUTER) - => Join Cond: (default.t1.id (INT4) = default.t2.id (INT4) AND (concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-11 OR concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-33)) + => Join Cond: ((concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-11 OR concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-33) AND default.t1.id (INT4) = default.t2.id (INT4)) => target list: default.t1.id (INT4), default.t1.name (TEXT), default.t2.id (INT4) => out schema: {(3) default.t1.id (INT4), default.t1.name (TEXT), default.t2.id (INT4)} => in schema: {(3) default.t1.id (INT4), default.t1.name (TEXT), default.t2.id (INT4)} diff --git a/tajo-core-tests/src/test/resources/results/TestOuterJoinQuery/testRightOuterJoinPredicationCaseByCase3.1.Sort_NoBroadcast.plan b/tajo-core-tests/src/test/resources/results/TestOuterJoinQuery/testRightOuterJoinPredicationCaseByCase3.1.Sort_NoBroadcast.plan index fab38091f8..dfb22875a5 100644 --- a/tajo-core-tests/src/test/resources/results/TestOuterJoinQuery/testRightOuterJoinPredicationCaseByCase3.1.Sort_NoBroadcast.plan +++ b/tajo-core-tests/src/test/resources/results/TestOuterJoinQuery/testRightOuterJoinPredicationCaseByCase3.1.Sort_NoBroadcast.plan @@ -10,7 +10,7 @@ JOIN(8)(RIGHT_OUTER) => out schema: {(1) default.t3.id (INT4)} => in schema: {(2) default.t3.id (INT4), default.t3.name (TEXT)} JOIN(7)(RIGHT_OUTER) - => Join Cond: (default.t1.id (INT4) = default.t2.id (INT4) AND (concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-11 OR concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-33)) + => Join Cond: ((concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-11 OR concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-33) AND default.t1.id (INT4) = default.t2.id (INT4)) => target list: default.t1.id (INT4), default.t1.name (TEXT), default.t2.id (INT4) => out schema: {(3) default.t1.id (INT4), default.t1.name (TEXT), default.t2.id (INT4)} => in schema: {(3) default.t1.id (INT4), default.t1.name (TEXT), default.t2.id (INT4)} @@ -80,7 +80,7 @@ Block Id: eb_0000000000000_0000_000003 [INTERMEDIATE] [q_0000000000000_0000] 3 => 5 (type=HASH_SHUFFLE, key=default.t1.id (INT4), num=32) JOIN(7)(RIGHT_OUTER) - => Join Cond: (default.t1.id (INT4) = default.t2.id (INT4) AND (concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-11 OR concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-33)) + => Join Cond: ((concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-11 OR concat(default.t1.name (TEXT),CAST (default.t2.id (INT4) AS TEXT)) = table11-33) AND default.t1.id (INT4) = default.t2.id (INT4)) => target list: default.t1.id (INT4), default.t1.name (TEXT), default.t2.id (INT4) => out schema: {(3) default.t1.id (INT4), default.t1.name (TEXT), default.t2.id (INT4)} => in schema: {(3) default.t1.id (INT4), default.t1.name (TEXT), default.t2.id (INT4)} diff --git a/tajo-core-tests/src/test/resources/results/TestTPCH/testFirstJoinInQ7.plan b/tajo-core-tests/src/test/resources/results/TestTPCH/testFirstJoinInQ7.plan index d17f26cc65..416f13b544 100644 --- a/tajo-core-tests/src/test/resources/results/TestTPCH/testFirstJoinInQ7.plan +++ b/tajo-core-tests/src/test/resources/results/TestTPCH/testFirstJoinInQ7.plan @@ -11,12 +11,12 @@ UNION(8) => in schema: {(4) c_nationkey (INT4), cust_nation (TEXT), s_nationkey (INT4), supp_nation (TEXT)} SCAN(4) on default.nation as n1 => filter: default.n1.n_name (TEXT) = GERMANY - => target list: default.n1.n_name (TEXT) as supp_nation, default.n1.n_nationkey (INT4) as s_nationkey + => target list: default.n1.n_nationkey (INT4) as s_nationkey, default.n1.n_name (TEXT) as supp_nation => out schema: {(2) s_nationkey (INT4), supp_nation (TEXT)} => in schema: {(4) default.n1.n_comment (TEXT), default.n1.n_name (TEXT), default.n1.n_nationkey (INT4), default.n1.n_regionkey (INT4)} SCAN(5) on default.nation as n2 => filter: default.n2.n_name (TEXT) = FRANCE - => target list: default.n2.n_name (TEXT) as cust_nation, default.n2.n_nationkey (INT4) as c_nationkey + => target list: default.n2.n_nationkey (INT4) as c_nationkey, default.n2.n_name (TEXT) as cust_nation => out schema: {(2) c_nationkey (INT4), cust_nation (TEXT)} => in schema: {(4) default.n2.n_comment (TEXT), default.n2.n_name (TEXT), default.n2.n_nationkey (INT4), default.n2.n_regionkey (INT4)} PROJECTION(3) @@ -29,12 +29,12 @@ UNION(8) => in schema: {(4) c_nationkey (INT4), cust_nation (TEXT), s_nationkey (INT4), supp_nation (TEXT)} SCAN(0) on default.nation as n1 => filter: default.n1.n_name (TEXT) = FRANCE - => target list: default.n1.n_name (TEXT) as supp_nation, default.n1.n_nationkey (INT4) as s_nationkey + => target list: default.n1.n_nationkey (INT4) as s_nationkey, default.n1.n_name (TEXT) as supp_nation => out schema: {(2) s_nationkey (INT4), supp_nation (TEXT)} => in schema: {(4) default.n1.n_comment (TEXT), default.n1.n_name (TEXT), default.n1.n_nationkey (INT4), default.n1.n_regionkey (INT4)} SCAN(1) on default.nation as n2 => filter: default.n2.n_name (TEXT) = GERMANY - => target list: default.n2.n_name (TEXT) as cust_nation, default.n2.n_nationkey (INT4) as c_nationkey + => target list: default.n2.n_nationkey (INT4) as c_nationkey, default.n2.n_name (TEXT) as cust_nation => out schema: {(2) c_nationkey (INT4), cust_nation (TEXT)} => in schema: {(4) default.n2.n_comment (TEXT), default.n2.n_name (TEXT), default.n2.n_nationkey (INT4), default.n2.n_regionkey (INT4)} explain @@ -70,7 +70,7 @@ Block Id: eb_0000000000000_0000_000001 [LEAF] SCAN(1) on default.nation as n2 => filter: default.n2.n_name (TEXT) = GERMANY - => target list: default.n2.n_name (TEXT) as cust_nation, default.n2.n_nationkey (INT4) as c_nationkey + => target list: default.n2.n_nationkey (INT4) as c_nationkey, default.n2.n_name (TEXT) as cust_nation => out schema: {(2) c_nationkey (INT4), cust_nation (TEXT)} => in schema: {(4) default.n2.n_comment (TEXT), default.n2.n_name (TEXT), default.n2.n_nationkey (INT4), default.n2.n_regionkey (INT4)} @@ -83,7 +83,7 @@ Block Id: eb_0000000000000_0000_000002 [LEAF] SCAN(0) on default.nation as n1 => filter: default.n1.n_name (TEXT) = FRANCE - => target list: default.n1.n_name (TEXT) as supp_nation, default.n1.n_nationkey (INT4) as s_nationkey + => target list: default.n1.n_nationkey (INT4) as s_nationkey, default.n1.n_name (TEXT) as supp_nation => out schema: {(2) s_nationkey (INT4), supp_nation (TEXT)} => in schema: {(4) default.n1.n_comment (TEXT), default.n1.n_name (TEXT), default.n1.n_nationkey (INT4), default.n1.n_regionkey (INT4)} @@ -96,7 +96,7 @@ Block Id: eb_0000000000000_0000_000004 [LEAF] SCAN(5) on default.nation as n2 => filter: default.n2.n_name (TEXT) = FRANCE - => target list: default.n2.n_name (TEXT) as cust_nation, default.n2.n_nationkey (INT4) as c_nationkey + => target list: default.n2.n_nationkey (INT4) as c_nationkey, default.n2.n_name (TEXT) as cust_nation => out schema: {(2) c_nationkey (INT4), cust_nation (TEXT)} => in schema: {(4) default.n2.n_comment (TEXT), default.n2.n_name (TEXT), default.n2.n_nationkey (INT4), default.n2.n_regionkey (INT4)} @@ -109,7 +109,7 @@ Block Id: eb_0000000000000_0000_000005 [LEAF] SCAN(4) on default.nation as n1 => filter: default.n1.n_name (TEXT) = GERMANY - => target list: default.n1.n_name (TEXT) as supp_nation, default.n1.n_nationkey (INT4) as s_nationkey + => target list: default.n1.n_nationkey (INT4) as s_nationkey, default.n1.n_name (TEXT) as supp_nation => out schema: {(2) s_nationkey (INT4), supp_nation (TEXT)} => in schema: {(4) default.n1.n_comment (TEXT), default.n1.n_name (TEXT), default.n1.n_nationkey (INT4), default.n1.n_regionkey (INT4)} diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalNode.java index 665a770d18..8a501c03f5 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/EvalNode.java @@ -22,7 +22,6 @@ import org.apache.tajo.annotation.Nullable; import org.apache.tajo.catalog.Schema; import org.apache.tajo.common.ProtoObject; -import org.apache.tajo.common.TajoDataTypes.DataType; import org.apache.tajo.datum.Datum; import org.apache.tajo.json.GsonObject; import org.apache.tajo.plan.serder.EvalNodeSerializer; diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/FieldEval.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/FieldEval.java index d905e4b1ef..396832964b 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/FieldEval.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/FieldEval.java @@ -18,7 +18,10 @@ package org.apache.tajo.plan.expr; -import org.apache.tajo.catalog.*; +import org.apache.tajo.catalog.Column; +import org.apache.tajo.catalog.FieldConverter; +import org.apache.tajo.catalog.Schema; +import org.apache.tajo.catalog.TypeConverter; import org.apache.tajo.common.TajoDataTypes.DataType; import org.apache.tajo.datum.Datum; import org.apache.tajo.schema.Field; @@ -113,12 +116,11 @@ public String toString() { } public boolean equals(Object obj) { - if (obj instanceof FieldEval) { - FieldEval other = (FieldEval) obj; - - return field.equals(other.field); + if (this == obj) { + return true; } - return false; + + return obj instanceof FieldEval && field.equals(((FieldEval) obj).field); } @Override diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/JoinSpec.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/JoinSpec.java index 643aa3d5bc..06afd684c3 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/JoinSpec.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/JoinSpec.java @@ -31,7 +31,7 @@ private static class EvalNodeComparator implements Comparator { @Override public int compare(EvalNode e1, EvalNode e2) { - return e1.toJson().compareTo(e2.toJson()); + return e1.toString().compareTo(e2.toString()); } } @@ -69,7 +69,7 @@ public boolean hasPredicates() { return predicates.size() > 0; } - public void setPredicates(Set predicates) { + public void setPredicates(Collection predicates) { this.predicates.clear(); if (predicates == null || predicates.isEmpty()) { if (type == JoinType.INNER) { @@ -81,7 +81,7 @@ public void setPredicates(Set predicates) { } public void setSingletonPredicate(EvalNode predicates) { - this.setPredicates(new HashSet<>(Arrays.asList(AlgebraicUtil.toConjunctiveNormalFormArray(predicates)))); + this.setPredicates(Arrays.asList(AlgebraicUtil.toConjunctiveNormalFormArray(predicates))); } public EvalNode getSingletonPredicate() { From 6f1e5fc879c45046d6c7c34ae0493f1000f91c65 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Mon, 25 Apr 2016 15:37:23 -0700 Subject: [PATCH 11/23] Fix failed unit tests. --- .../main/java/org/apache/tajo/type/Char.java | 2 +- .../java/org/apache/tajo/type/Numeric.java | 2 +- .../java/org/apache/tajo/type/Varchar.java | 2 +- .../org/apache/tajo/schema/TestSchema.java | 6 ++-- .../java/org/apache/tajo/type/TestType.java | 34 +++++++++---------- 5 files changed, 23 insertions(+), 23 deletions(-) diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Char.java b/tajo-common/src/main/java/org/apache/tajo/type/Char.java index 348e041f10..827ef132c1 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Char.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Char.java @@ -45,7 +45,7 @@ public TajoDataTypes.Type baseType() { @Override public String toString() { - return "char(" + length + ")"; + return "CHAR(" + length + ")"; } @Override diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Numeric.java b/tajo-common/src/main/java/org/apache/tajo/type/Numeric.java index 75ef3929a2..6bb51cb2e7 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Numeric.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Numeric.java @@ -46,7 +46,7 @@ public TajoDataTypes.Type baseType() { @Override public String toString() { - return "numeric(" + precision + "," + scale + ")"; + return "NUMERIC(" + precision + "," + scale + ")"; } @Override diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Varchar.java b/tajo-common/src/main/java/org/apache/tajo/type/Varchar.java index fbfac173ae..077de41956 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Varchar.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Varchar.java @@ -45,7 +45,7 @@ public boolean hasParam() { @Override public String toString() { - return "varchar(" + length + ")"; + return "VARCHAR(" + length + ")"; } @Override diff --git a/tajo-common/src/test/java/org/apache/tajo/schema/TestSchema.java b/tajo-common/src/test/java/org/apache/tajo/schema/TestSchema.java index 09b47cd630..7d716306b7 100644 --- a/tajo-common/src/test/java/org/apache/tajo/schema/TestSchema.java +++ b/tajo-common/src/test/java/org/apache/tajo/schema/TestSchema.java @@ -36,7 +36,7 @@ public final void testSchema1() { Field struct2 = Record($("f34"), Field($("f3"), Int8), Field($("f4"), Int4)); Schema schema = Schema(struct1, struct2); - assertEquals("f12 record(f1 int8, f2 int4), f34 record(f3 int8, f4 int4)", schema.toString()); + assertEquals("f12 (RECORD(f1 (INT8), f2 (INT4))), f34 (RECORD(f3 (INT8), f4 (INT4)))", schema.toString()); } @Test @@ -46,7 +46,7 @@ public final void testSchema2() { Field f3 = Record($("z"), Field($("z-1"), Time), Field($("z-2"), Array(Int8))); Schema schema = Schema(f1, f2, f3); - assertEquals("x array, y int8, z record(z-1 time, z-2 array)", schema.toString()); + assertEquals("x (ARRAY), y (INT8), z (RECORD(z-1 (TIME), z-2 (ARRAY)))", schema.toString()); } @Test @@ -56,6 +56,6 @@ public final void testSchemaWithIdentifiers() { Field f3 = Record($("z"), Field($("z-1"), Time), Field($("z-2"), Array(Int8))); Schema schema = Schema(f1, f2, f3); - assertEquals("x.y array, y.B int8, z record(z-1 time, z-2 array)", schema.toString()); + assertEquals("x.y (ARRAY), y.B (INT8), z (RECORD(z-1 (TIME), z-2 (ARRAY)))", schema.toString()); } } \ No newline at end of file diff --git a/tajo-common/src/test/java/org/apache/tajo/type/TestType.java b/tajo-common/src/test/java/org/apache/tajo/type/TestType.java index 318f5447a8..a399286888 100644 --- a/tajo-common/src/test/java/org/apache/tajo/type/TestType.java +++ b/tajo-common/src/test/java/org/apache/tajo/type/TestType.java @@ -90,35 +90,35 @@ public final void testPrimitiveTypes() { @Test public final void testToString() { - assertEquals("boolean", Bool.toString()); - assertEquals("int1", Int1.toString()); - assertEquals("int2", Int2.toString()); - assertEquals("int4", Int4.toString()); - assertEquals("int8", Int8.toString()); - assertEquals("float4", Float4.toString()); - assertEquals("float8", Float8.toString()); - assertEquals("date", Date.toString()); - assertEquals("time", Time.toString()); - assertEquals("timestamp", Timestamp.toString()); + assertEquals("BOOLEAN", Bool.toString()); + assertEquals("INT1", Int1.toString()); + assertEquals("INT2", Int2.toString()); + assertEquals("INT4", Int4.toString()); + assertEquals("INT8", Int8.toString()); + assertEquals("FLOAT4", Float4.toString()); + assertEquals("FLOAT8", Float8.toString()); + assertEquals("DATE", Date.toString()); + assertEquals("TIME", Time.toString()); + assertEquals("TIMESTAMP", Timestamp.toString()); Numeric n = Numeric(4, 2); - assertEquals("numeric(4,2)", n.toString()); + assertEquals("NUMERIC(4,2)", n.toString()); - assertEquals("blob", Blob.toString()); + assertEquals("BLOB", Blob.toString()); Char c = Char(2); - assertEquals("char(2)", c.toString()); + assertEquals("CHAR(2)", c.toString()); Varchar varchar = Varchar(2); - assertEquals("varchar(2)", varchar.toString()); + assertEquals("VARCHAR(2)", varchar.toString()); Record record = Struct(Field("x", Int8), Field("y", Array(Float8))); - assertEquals("record(x int8, y array)", record.toString()); + assertEquals("RECORD(x (INT8), y (ARRAY))", record.toString()); Map map = Map(Int8, Array(Timestamp)); - assertEquals("map>", map.toString()); + assertEquals("MAP>", map.toString()); Array array = Array(Int8); - assertEquals("array", array.toString()); + assertEquals("ARRAY", array.toString()); } } \ No newline at end of file From 6702ead3ec86bae5a0eca915c406e76868b19ce8 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Mon, 25 Apr 2016 20:49:04 -0700 Subject: [PATCH 12/23] Add array unit test to TestCreateTable. --- .../org/apache/tajo/algebra/ColumnDefinition.java | 10 +++++----- .../main/java/org/apache/tajo/catalog/Column.java | 4 ++-- .../apache/tajo/engine/query/TestCreateTable.java | 14 ++++++++++++++ .../TestCreateTable/create_table_various_types.sql | 3 ++- .../queries/TestCreateTable/testArrayType1.sql | 1 + 5 files changed, 24 insertions(+), 8 deletions(-) create mode 100644 tajo-core-tests/src/test/resources/queries/TestCreateTable/testArrayType1.sql diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/ColumnDefinition.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/ColumnDefinition.java index 2a829e1950..7ed9ade723 100644 --- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/ColumnDefinition.java +++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/ColumnDefinition.java @@ -24,11 +24,6 @@ public class ColumnDefinition extends DataTypeExpr { @Expose @SerializedName("ColumnDefName") String columnName; - public ColumnDefinition(String columnName, String dataType) { - super(dataType); - this.columnName = columnName; - } - public ColumnDefinition(String columnName, DataTypeExpr dataType) { super(dataType.getTypeName()); @@ -51,6 +46,11 @@ public ColumnDefinition(String columnName, DataTypeExpr dataType) { if (dataType.isMapType()) { this.mapType = dataType.mapType; } + + // map type + if (dataType.isArrayType()) { + this.arrayType = dataType.arrayType; + } } public String getColumnName() { diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java index a8ab86f02c..92dccba481 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java @@ -33,8 +33,8 @@ * Describes a column. It is an immutable object. */ public class Column implements ProtoObject, GsonObject { - @Expose protected String name; - @Expose protected TypeDesc typeDesc; + protected String name; + protected TypeDesc typeDesc; /** * Column Constructor diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java index 3978db019f..d733abbafe 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java @@ -679,4 +679,18 @@ public final void testSelfDescTable2() throws Exception { executeString("drop table d10.schemaless").close(); executeString("drop database d10").close(); } + + @Test + public final void testArrayType1() throws Exception { + try { + executeString("CREATE DATABASE D11;").close(); + + assertTableNotExists("d11.array_table1"); + executeQuery().close(); + assertTableExists("d11.array_table1"); + } finally { + executeString("DROP TABLE IF EXISTS D11.array_table1"); + executeString("DROP DATABASE IF EXISTS D11").close(); + } + } } diff --git a/tajo-core-tests/src/test/resources/queries/TestCreateTable/create_table_various_types.sql b/tajo-core-tests/src/test/resources/queries/TestCreateTable/create_table_various_types.sql index bfa5e97551..45cd7a21b6 100644 --- a/tajo-core-tests/src/test/resources/queries/TestCreateTable/create_table_various_types.sql +++ b/tajo-core-tests/src/test/resources/queries/TestCreateTable/create_table_various_types.sql @@ -46,5 +46,6 @@ create table various_types ( -- col42 binary(10), -- col43 varbinary(10), -- col44 binary Varying(10), - col45 blOb + col45 blOb, + col46 array ); \ No newline at end of file diff --git a/tajo-core-tests/src/test/resources/queries/TestCreateTable/testArrayType1.sql b/tajo-core-tests/src/test/resources/queries/TestCreateTable/testArrayType1.sql new file mode 100644 index 0000000000..f4d387445d --- /dev/null +++ b/tajo-core-tests/src/test/resources/queries/TestCreateTable/testArrayType1.sql @@ -0,0 +1 @@ +CREATE TABLE array_table1 (vec array); \ No newline at end of file From c74af8adfad0b075618c990d16b674a6395448b4 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Mon, 25 Apr 2016 21:04:07 -0700 Subject: [PATCH 13/23] Change TypeDesc to Type. --- .../java/org/apache/tajo/catalog/Column.java | 37 +++++++------------ .../apache/tajo/catalog/NestedPathUtil.java | 2 +- .../org/apache/tajo/catalog/SchemaLegacy.java | 8 ++-- .../org/apache/tajo/catalog/SchemaUtil.java | 2 +- .../java/org/apache/tajo/schema/Schema.java | 1 - .../org/apache/tajo/plan/LogicalPlanner.java | 2 +- 6 files changed, 20 insertions(+), 32 deletions(-) diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java index 92dccba481..c685b8c60a 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java @@ -19,7 +19,6 @@ package org.apache.tajo.catalog; import com.google.common.base.Objects; -import com.google.gson.annotations.Expose; import org.apache.tajo.catalog.json.CatalogGsonHelper; import org.apache.tajo.catalog.proto.CatalogProtos.ColumnProto; import org.apache.tajo.common.ProtoObject; @@ -34,17 +33,17 @@ */ public class Column implements ProtoObject, GsonObject { protected String name; - protected TypeDesc typeDesc; + protected Type type; /** * Column Constructor * * @param name field name - * @param typeDesc Type description + * @param type Type description */ - public Column(String name, TypeDesc typeDesc) { + public Column(String name, TypeDesc type) { this.name = name; - this.typeDesc = typeDesc; + this.type = TypeConverter.convert(type); } /** @@ -71,22 +70,13 @@ public Column(String name, TajoDataTypes.Type type) { * @param type Type */ public Column(String name, Type type) { - this(name, TypeConverter.convert(type)); - } - - /** - * - * @param name Column name - * @param type Data Type - * @param typeLength The length of type - */ - public Column(String name, TajoDataTypes.Type type, int typeLength) { - this(name, CatalogUtil.newDataTypeWithLen(type, typeLength)); + this.name = name; + this.type = type; } public Column(ColumnProto proto) { name = proto.getName(); - typeDesc = new TypeDesc(proto.getDataType()); + type = TypeConverter.convert(new TypeDesc(proto.getDataType())); } /** @@ -127,7 +117,7 @@ public String getSimpleName() { * @return TypeDesc */ public TypeDesc getTypeDesc() { - return this.typeDesc; + return TypeConverter.convert(this.type); } /** @@ -135,21 +125,20 @@ public TypeDesc getTypeDesc() { * @return DataType which includes domain type and scale. */ public DataType getDataType() { - return this.typeDesc.dataType; + return TypeConverter.convert(this.type).getDataType(); } @Override public boolean equals(Object o) { if (o instanceof Column) { Column another = (Column)o; - return name.equals(another.name) && typeDesc.equals(another.typeDesc); + return name.equals(another.name) && type.equals(another.type); } return false; } public int hashCode() { - return Objects.hashCode(name, typeDesc); - + return Objects.hashCode(name, type); } /** @@ -161,13 +150,13 @@ public ColumnProto getProto() { ColumnProto.Builder builder = ColumnProto.newBuilder(); builder .setName(this.name) - .setDataType(this.typeDesc.getDataType()); + .setDataType(TypeConverter.convert(this.type).dataType); return builder.build(); } public String toString() { StringBuilder sb = new StringBuilder(getQualifiedName()); - sb.append(" (").append(typeDesc.toString()).append(")"); + sb.append(" (").append(type.toString()).append(")"); return sb.toString(); } diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/NestedPathUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/NestedPathUtil.java index bf874b95b0..afe7e3112d 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/NestedPathUtil.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/NestedPathUtil.java @@ -73,7 +73,7 @@ private static Column lookupColumnInternal(Column currentColumn, String [] paths Column found = null; if (currentColumn.getDataType().getType() == Type.RECORD) { - found = currentColumn.typeDesc.nestedRecordSchema.getColumn(paths[depth]); + found = TypeConverter.convert(currentColumn.type).nestedRecordSchema.getColumn(paths[depth]); } if (found != null) { diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaLegacy.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaLegacy.java index 2a16fa6e83..0c3032a6b5 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaLegacy.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaLegacy.java @@ -139,7 +139,7 @@ public void setQualifier(String qualifier) { Column newColumn; for (Column c : columns) { - newColumn = new Column(qualifier + "." + c.getSimpleName(), c.typeDesc); + newColumn = new Column(qualifier + "." + c.getSimpleName(), c.type); addColumn(newColumn); } } @@ -200,7 +200,7 @@ public Column getColumn(String name) { Column columnPath = new Column( column.getQualifiedName() + NestedPathUtil.makePath(paths, 1), - actualColumn.typeDesc); + actualColumn.type); return columnPath; } else { @@ -441,7 +441,7 @@ private SchemaLegacy addColumn(String name, TypeDesc typeDesc) { } private synchronized void addColumn(Column column) { - addColumn(column.getQualifiedName(), column.typeDesc); + addColumn(column.getQualifiedName(), TypeConverter.convert(column.type)); } @Override @@ -488,7 +488,7 @@ public void visit(int depth, List path, Column column) { if (column.getDataType().getType() == Type.RECORD) { DataType.Builder updatedType = DataType.newBuilder(column.getDataType()); - updatedType.setNumNestedFields(column.typeDesc.nestedRecordSchema.size()); + updatedType.setNumNestedFields(TypeConverter.convert(column.type).nestedRecordSchema.size()); ColumnProto.Builder updatedColumn = ColumnProto.newBuilder(column.getProto()); updatedColumn.setDataType(updatedType); diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java index c0b60a31bc..fc6d3538d9 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaUtil.java @@ -185,7 +185,7 @@ private static void visitInDepthFirstOrder(int depth, Column column) { if (column.getDataType().getType() == Type.RECORD) { - for (Column nestedColumn : column.typeDesc.nestedRecordSchema.getRootColumns()) { + for (Column nestedColumn : TypeConverter.convert(column.type).nestedRecordSchema.getRootColumns()) { List newPath = new ArrayList<>(path); newPath.add(column.getQualifiedName()); diff --git a/tajo-common/src/main/java/org/apache/tajo/schema/Schema.java b/tajo-common/src/main/java/org/apache/tajo/schema/Schema.java index b3b28b7e89..771e03b5d6 100644 --- a/tajo-common/src/main/java/org/apache/tajo/schema/Schema.java +++ b/tajo-common/src/main/java/org/apache/tajo/schema/Schema.java @@ -52,5 +52,4 @@ public String toString() { public Iterator iterator() { return fields.iterator(); } - } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java index fce4e69d31..08081a567b 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java @@ -2103,7 +2103,7 @@ private static String[] convertExprsToStrings(Expr[] exprs) { } private static Column convertColumn(ColumnDefinition columnDefinition) { - return new Column(columnDefinition.getColumnName(), convert(convertDataType(columnDefinition))); + return new Column(columnDefinition.getColumnName(), convertDataType(columnDefinition)); } public static Type convertDataType(DataTypeExpr dataType) { From 4a5056edb00da5eb59a49892f79b5d03bb1bccf7 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Mon, 2 May 2016 13:37:37 -0700 Subject: [PATCH 14/23] Implement TypeSerializer and its relevant classes. --- .../apache/tajo/catalog/FieldConverter.java | 26 +- .../apache/tajo/catalog/SchemaBuilder.java | 7 +- .../org/apache/tajo/catalog/SchemaLegacy.java | 6 +- .../apache/tajo/catalog/TypeConverter.java | 45 ++- .../tajo/schema/ANSISQLIdentifierPolicy.java | 4 +- .../java/org/apache/tajo/schema/Field.java | 8 +- .../org/apache/tajo/schema/Identifier.java | 26 +- .../apache/tajo/schema/IdentifierPolicy.java | 8 +- .../apache/tajo/schema/IdentifierUtil.java | 68 +++- .../tajo/schema/PgSQLIdentifierPolicy.java | 4 +- .../tajo/schema/QualifiedIdentifier.java | 38 +- .../org/apache/tajo/schema/package-info.java | 22 ++ .../main/java/org/apache/tajo/type/Any.java | 12 +- .../main/java/org/apache/tajo/type/Array.java | 40 +- .../main/java/org/apache/tajo/type/Blob.java | 7 +- .../main/java/org/apache/tajo/type/Bool.java | 7 +- .../main/java/org/apache/tajo/type/Char.java | 41 +- .../main/java/org/apache/tajo/type/Date.java | 7 +- .../java/org/apache/tajo/type/Float4.java | 7 +- .../java/org/apache/tajo/type/Float8.java | 7 +- .../main/java/org/apache/tajo/type/Inet4.java | 7 +- .../main/java/org/apache/tajo/type/Int1.java | 7 +- .../main/java/org/apache/tajo/type/Int2.java | 7 +- .../main/java/org/apache/tajo/type/Int4.java | 7 +- .../main/java/org/apache/tajo/type/Int8.java | 7 +- .../java/org/apache/tajo/type/Interval.java | 7 +- .../main/java/org/apache/tajo/type/Map.java | 38 +- .../main/java/org/apache/tajo/type/Null.java | 7 +- .../java/org/apache/tajo/type/Numeric.java | 40 +- .../java/org/apache/tajo/type/Protobuf.java | 12 +- .../java/org/apache/tajo/type/Record.java | 11 +- .../main/java/org/apache/tajo/type/Text.java | 7 +- .../main/java/org/apache/tajo/type/Time.java | 7 +- .../java/org/apache/tajo/type/Timestamp.java | 7 +- .../main/java/org/apache/tajo/type/Type.java | 33 +- .../tajo/type/TypeParamterizedType.java | 77 ++++ .../org/apache/tajo/type/TypeSerializer.java | 367 ++++++++++++++++++ .../tajo/type/ValueParamterizedType.java | 77 ++++ .../java/org/apache/tajo/type/Varchar.java | 40 +- .../org/apache/tajo/type/package-info.java | 29 ++ .../tajo/schema/TestTajoIdentifierPolicy.java | 4 +- .../java/org/apache/tajo/type/TestType.java | 5 +- .../apache/tajo/type/TestTypeSerializer.java | 126 ++++++ .../TestCreateTable/testArrayType1.sql | 2 +- .../org/apache/tajo/plan/LogicalPlanner.java | 5 +- .../org/apache/tajo/plan/expr/FieldEval.java | 10 +- 46 files changed, 987 insertions(+), 349 deletions(-) create mode 100644 tajo-common/src/main/java/org/apache/tajo/schema/package-info.java create mode 100644 tajo-common/src/main/java/org/apache/tajo/type/TypeParamterizedType.java create mode 100644 tajo-common/src/main/java/org/apache/tajo/type/TypeSerializer.java create mode 100644 tajo-common/src/main/java/org/apache/tajo/type/ValueParamterizedType.java create mode 100644 tajo-common/src/main/java/org/apache/tajo/type/package-info.java create mode 100644 tajo-common/src/test/java/org/apache/tajo/type/TestTypeSerializer.java diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FieldConverter.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FieldConverter.java index fbff4063c9..1b34429b78 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FieldConverter.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/FieldConverter.java @@ -21,14 +21,12 @@ import com.google.common.base.Function; import com.google.common.collect.Collections2; import com.google.common.collect.ImmutableList; -import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.exception.NotImplementedException; import org.apache.tajo.exception.TajoRuntimeException; import org.apache.tajo.schema.Field; import org.apache.tajo.schema.Identifier; import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.schema.QualifiedIdentifier; -import org.apache.tajo.type.Record; import javax.annotation.Nullable; import java.util.Collection; @@ -47,29 +45,13 @@ public Identifier apply(@Nullable String input) { } public static Field convert(Column column) { - if (column.getTypeDesc().getDataType().getType() == TajoDataTypes.Type.RECORD) { - - if (column.getTypeDesc().getNestedSchema() == null) { - throw new TajoRuntimeException(new NotImplementedException("record type projection")); - } + if (column.type.isStruct() && column.getTypeDesc().getNestedSchema() == null) { + throw new TajoRuntimeException(new NotImplementedException("record type projection")); } - - return new Field(TypeConverter.convert(column.getTypeDesc()), toQualifiedIdentifier(column.getQualifiedName())); + return new Field(toQualifiedIdentifier(column.getQualifiedName()), column.type); } public static Column convert(Field field) { - if (field.isStruct()) { - Record record = field.type(); - Collection converted = Collections2 - .transform(record.fields(), new Function() { - @Override - public Column apply(@Nullable Field namedType) { - return FieldConverter.convert(namedType); - } - }); - return new Column(field.name().raw(), new TypeDesc(new SchemaLegacy(converted))); - } else { - return new Column(field.name().raw(), TypeConverter.convert(field)); - } + return new Column(field.name().interned(), field.type()); } } diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaBuilder.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaBuilder.java index 1beb792fc2..582b5ee300 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaBuilder.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaBuilder.java @@ -22,15 +22,14 @@ import com.google.common.collect.ImmutableCollection; import com.google.common.collect.ImmutableList; import org.apache.tajo.common.TajoDataTypes; -import org.apache.tajo.schema.QualifiedIdentifier; import org.apache.tajo.schema.Field; +import org.apache.tajo.schema.QualifiedIdentifier; import org.apache.tajo.type.Type; import javax.annotation.Nullable; import java.util.Iterator; import static org.apache.tajo.catalog.FieldConverter.toQualifiedIdentifier; -import static org.apache.tajo.schema.IdentifierPolicy.DefaultPolicy; /** * Builder for Schema @@ -67,7 +66,7 @@ public SchemaBuilder add(Field field) { } public SchemaBuilder add(QualifiedIdentifier id, Type type) { - add(new Field(type, id)); + add(new Field(id, type)); return this; } @@ -171,7 +170,7 @@ public SchemaBuilder addAll2(Iterator fields, Function fn) { public SchemaLegacy build() { ImmutableList.Builder columns = new ImmutableList.Builder(); for (Field field : fields.build()) { - columns.add(new Column(field.name().raw(), TypeConverter.convert(field))); + columns.add(new Column(field.name().interned(), field.type())); } return new SchemaLegacy(columns.build()); diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaLegacy.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaLegacy.java index 0c3032a6b5..8cda938706 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaLegacy.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaLegacy.java @@ -424,13 +424,13 @@ public boolean containsAny(Collection columns) { return false; } - private SchemaLegacy addColumn(String name, TypeDesc typeDesc) { + private SchemaLegacy addColumn(String name, org.apache.tajo.type.Type type) { String normalized = name; if(fieldsByQualifiedName.containsKey(normalized)) { throw new TajoRuntimeException(new DuplicateColumnException(normalized)); } - Column newCol = new Column(normalized, typeDesc); + Column newCol = new Column(normalized, type); fields.add(newCol); fieldsByQualifiedName.put(newCol.getQualifiedName(), fields.size() - 1); List inputList = new ArrayList<>(); @@ -441,7 +441,7 @@ private SchemaLegacy addColumn(String name, TypeDesc typeDesc) { } private synchronized void addColumn(Column column) { - addColumn(column.getQualifiedName(), TypeConverter.convert(column.type)); + addColumn(column.getQualifiedName(), column.type); } @Override diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TypeConverter.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TypeConverter.java index 64489251c5..b42b6affca 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TypeConverter.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TypeConverter.java @@ -27,6 +27,8 @@ import org.apache.tajo.type.*; import static org.apache.tajo.catalog.CatalogUtil.newDataTypeWithLen; +import static org.apache.tajo.catalog.CatalogUtil.newSimpleDataType; +import static org.apache.tajo.common.TajoDataTypes.Type.*; import static org.apache.tajo.type.Type.*; public class TypeConverter { @@ -37,7 +39,7 @@ public static Type convert(TypeDesc type) { for (Column c : type.getNestedSchema().getRootColumns()) { fields.add(FieldConverter.convert(c)); } - return Struct(fields.build()); + return Record(fields.build()); } else { return convert(type.dataType); } @@ -105,6 +107,22 @@ public static Type convert(TajoDataTypes.Type legacyBaseType) { return Null; case ANY: return Any; + + case BOOLEAN_ARRAY: + return Array(Bool); + case INT1_ARRAY: + return Array(Int1); + case INT2_ARRAY: + return Array(Int2); + case INT4_ARRAY: + return Array(Int4); + case INT8_ARRAY: + return Array(Int8); + case FLOAT4_ARRAY: + return Array(Float4); + case FLOAT8_ARRAY: + return Array(Float8); + default: throw new TajoRuntimeException(new UnsupportedException(legacyBaseType.name())); } @@ -132,11 +150,32 @@ public static TypeDesc convert(Type type) { Record record = (Record) type; ImmutableList.Builder fields = ImmutableList.builder(); for (Field t: record.fields()) { - fields.add(new Column(t.name().raw(), convert(t))); + fields.add(new Column(t.name().interned(), convert(t))); } return new TypeDesc(SchemaBuilder.builder().addAll(fields.build()).build()); + + case ARRAY: + Array array = (Array) type; + Type elemType = array.elementType(); + switch (elemType.baseType()) { + case INT1: + return new TypeDesc(newSimpleDataType(INT1_ARRAY)); + case INT2: + return new TypeDesc(newSimpleDataType(INT2_ARRAY)); + case INT4: + return new TypeDesc(newSimpleDataType(INT4_ARRAY)); + case INT8: + return new TypeDesc(newSimpleDataType(INT8_ARRAY)); + case FLOAT4: + return new TypeDesc(newSimpleDataType(FLOAT4_ARRAY)); + case FLOAT8: + return new TypeDesc(newSimpleDataType(FLOAT8_ARRAY)); + default: + return new TypeDesc(newSimpleDataType(type.baseType())); + } + default: - return new TypeDesc(CatalogUtil.newSimpleDataType(type.baseType())); + return new TypeDesc(newSimpleDataType(type.baseType())); } } } diff --git a/tajo-common/src/main/java/org/apache/tajo/schema/ANSISQLIdentifierPolicy.java b/tajo-common/src/main/java/org/apache/tajo/schema/ANSISQLIdentifierPolicy.java index c195bb6e9d..cfe91c1291 100644 --- a/tajo-common/src/main/java/org/apache/tajo/schema/ANSISQLIdentifierPolicy.java +++ b/tajo-common/src/main/java/org/apache/tajo/schema/ANSISQLIdentifierPolicy.java @@ -29,12 +29,12 @@ String getName() { } @Override - public String getIdentifierQuoteString() { + public char getIdentifierQuoteString() { return ANSI_SQL_QUOTE_STRING; } @Override - String getIdentifierSeperator() { + char getIdentifierSeperator() { return ANSI_SQL_SEPERATOR_STRING; } diff --git a/tajo-common/src/main/java/org/apache/tajo/schema/Field.java b/tajo-common/src/main/java/org/apache/tajo/schema/Field.java index 24909bb349..f3d93e363e 100644 --- a/tajo-common/src/main/java/org/apache/tajo/schema/Field.java +++ b/tajo-common/src/main/java/org/apache/tajo/schema/Field.java @@ -33,7 +33,7 @@ public class Field implements Cloneable { protected final Type type; protected final QualifiedIdentifier name; - public Field(Type type, QualifiedIdentifier name) { + public Field(QualifiedIdentifier name, Type type) { this.type = type; this.name = name; } @@ -43,16 +43,16 @@ public static Field Record(QualifiedIdentifier name, Field ... fields) { } public static Field Record(QualifiedIdentifier name, Collection fields) { - return new Field(Type.Struct(fields), name); + return new Field(name, Type.Record(fields)); } @VisibleForTesting public static Field Field(String name, Type type) { - return new Field(type, QualifiedIdentifier.$(name)); + return new Field(QualifiedIdentifier.$(name), type); } public static Field Field(QualifiedIdentifier name, Type type) { - return new Field(type, name); + return new Field(name, type); } public QualifiedIdentifier name() { diff --git a/tajo-common/src/main/java/org/apache/tajo/schema/Identifier.java b/tajo-common/src/main/java/org/apache/tajo/schema/Identifier.java index 6fb3693bd8..dba9588ece 100644 --- a/tajo-common/src/main/java/org/apache/tajo/schema/Identifier.java +++ b/tajo-common/src/main/java/org/apache/tajo/schema/Identifier.java @@ -22,13 +22,20 @@ import java.util.Objects; +import static org.apache.tajo.schema.IdentifierPolicy.DefaultPolicy; + /** - * Identifier Element + * Represents an identifier part */ public class Identifier { private String name; private boolean quoted; + /** + * Identifier constructor + * @param name Identifier part string + * @param quoted quoted or not + */ private Identifier(String name, boolean quoted) { this.name = name; this.quoted = quoted; @@ -42,7 +49,12 @@ public static Identifier _(String name, boolean quoted) { return new Identifier(name, quoted); } - public String displayString(IdentifierPolicy policy) { + /** + * Raw string for an identifier, which is equivalent to an identifier directly used in SQL statements. + * @param policy IdentifierPolicy + * @return Raw string + */ + public String raw(IdentifierPolicy policy) { StringBuilder sb = new StringBuilder(); if (quoted) { appendByCase(sb, policy.storesQuotedIdentifierAs()); @@ -56,11 +68,11 @@ public String displayString(IdentifierPolicy policy) { } /** - * Raw string of an identifier - * @param policy Identifier Policy - * @return raw string + * Interned string of an identifier + * @param policy IdentifierPolicy + * @return interned string */ - public String raw(IdentifierPolicy policy) { + public String interned(IdentifierPolicy policy) { StringBuilder sb = new StringBuilder(); if (quoted) { appendByCase(sb, policy.storesQuotedIdentifierAs()); @@ -87,7 +99,7 @@ private void appendByCase(StringBuilder sb, IdentifierCase c) { @Override public String toString() { - return displayString(IdentifierPolicy.DefaultPolicy()); + return raw(DefaultPolicy()); } public int hashCode() { diff --git a/tajo-common/src/main/java/org/apache/tajo/schema/IdentifierPolicy.java b/tajo-common/src/main/java/org/apache/tajo/schema/IdentifierPolicy.java index 56762e779d..75da1860ee 100644 --- a/tajo-common/src/main/java/org/apache/tajo/schema/IdentifierPolicy.java +++ b/tajo-common/src/main/java/org/apache/tajo/schema/IdentifierPolicy.java @@ -24,9 +24,9 @@ public abstract class IdentifierPolicy { /** Quote String; e.g., 'abc' */ - public static final String ANSI_SQL_QUOTE_STRING = "'"; + public static final char ANSI_SQL_QUOTE_STRING = '\''; /** Separator; e.g., abc.xyz */ - public static final String ANSI_SQL_SEPERATOR_STRING = "."; + public static final char ANSI_SQL_SEPERATOR_STRING = '.'; /** Maximum length of identifiers */ public final static int MAX_IDENTIFIER_LENGTH = 128; @@ -49,7 +49,7 @@ public enum IdentifierCase { * * @return the quoting string or a space if quoting is not supported */ - abstract String getIdentifierQuoteString(); + abstract char getIdentifierQuoteString(); /** * Retrieves the String that this policy uses as the separator between @@ -57,7 +57,7 @@ public enum IdentifierCase { * * @return the separator string */ - abstract String getIdentifierSeperator(); + abstract char getIdentifierSeperator(); /** * Retrieves the maximum number of characters this policy allows for a column name. diff --git a/tajo-common/src/main/java/org/apache/tajo/schema/IdentifierUtil.java b/tajo-common/src/main/java/org/apache/tajo/schema/IdentifierUtil.java index 78b4a3a21e..f086410f99 100644 --- a/tajo-common/src/main/java/org/apache/tajo/schema/IdentifierUtil.java +++ b/tajo-common/src/main/java/org/apache/tajo/schema/IdentifierUtil.java @@ -19,13 +19,20 @@ package org.apache.tajo.schema; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; import org.apache.tajo.TajoConstants; +import org.apache.tajo.schema.IdentifierPolicy.IdentifierCase; import org.apache.tajo.util.Pair; import org.apache.tajo.util.StringUtils; import java.util.HashSet; +import java.util.Locale; import java.util.Set; +import static org.apache.tajo.schema.Identifier._; +import static org.apache.tajo.schema.IdentifierPolicy.DefaultPolicy; +import static org.apache.tajo.schema.QualifiedIdentifier.$; + /** * Util methods for Identifiers */ @@ -59,6 +66,38 @@ public class IdentifierUtil { "WHEN", "WHERE", "WINDOW", "WITH" }; + public static QualifiedIdentifier makeIdentifier(String raw, IdentifierPolicy policy) { + if (raw == null || raw.equals("")) { + return $(raw); + } + + final String [] splitted = raw.split(IDENTIFIER_DELIMITER_REGEXP); + final ImmutableList.Builder builder = ImmutableList.builder(); + + for (String part : splitted) { + boolean quoted = isDelimited(part, policy); + if (quoted) { + builder.add(_(stripQuote(part), true)); + } else { + builder.add(internIdentifierPart(part, policy)); + } + } + return $(builder.build()); + } + + public static Identifier internIdentifierPart(String rawPart, IdentifierPolicy policy) { + IdentifierCase unquotedIdentifierCase = policy.storesUnquotedIdentifierAs(); + final String interned; + if (unquotedIdentifierCase == IdentifierCase.LowerCase) { + interned = rawPart.toLowerCase(Locale.ENGLISH); + } else if (unquotedIdentifierCase == IdentifierCase.UpperCase) { + interned = rawPart.toUpperCase(Locale.ENGLISH); + } else { + interned = rawPart; + } + + return _(interned, false); + } /** * Normalize an identifier. Normalization means a translation from a identifier to be a refined identifier name. @@ -142,8 +181,8 @@ public static String denormalizePart(String identifier) { } } - public static boolean isShouldBeQuoted(String columnName) { - for (char character : columnName.toCharArray()) { + public static boolean isShouldBeQuoted(String interned) { + for (char character : interned.toCharArray()) { if (Character.isUpperCase(character)) { return true; } @@ -152,7 +191,7 @@ public static boolean isShouldBeQuoted(String columnName) { return true; } - if (RESERVED_KEYWORDS_SET.contains(columnName.toUpperCase())) { + if (RESERVED_KEYWORDS_SET.contains(interned.toUpperCase())) { return true; } } @@ -160,22 +199,27 @@ public static boolean isShouldBeQuoted(String columnName) { return false; } - public static String stripQuote(String str) { - return str.substring(1, str.length() - 1); + public static String stripQuote(String raw) { + return raw.substring(1, raw.length() - 1); + } + + public static boolean isDelimited(String raw) { + return isDelimited(raw, DefaultPolicy()); } - public static boolean isDelimited(String identifier) { - boolean openQuote = identifier.charAt(0) == '"'; - boolean closeQuote = identifier.charAt(identifier.length() - 1) == '"'; + public static boolean isDelimited(String raw, IdentifierPolicy policy) { + char quoteString = policy.getIdentifierQuoteString(); + boolean openQuote = raw.charAt(0) == quoteString; + boolean closeQuote = raw.charAt(raw.length() - 1) == quoteString; // if at least one quote mark exists, the identifier must be grater than equal to 2 characters, - if (openQuote ^ closeQuote && identifier.length() < 2) { - throw new IllegalArgumentException("Invalid Identifier: " + identifier); + if (openQuote ^ closeQuote && raw.length() < 2) { + throw new IllegalArgumentException("Invalid Identifier: " + raw); } // does not allow the empty identifier (''), - if (openQuote && closeQuote && identifier.length() == 2) { - throw new IllegalArgumentException("zero-length delimited identifier: " + identifier); + if (openQuote && closeQuote && raw.length() == 2) { + throw new IllegalArgumentException("zero-length delimited identifier: " + raw); } // Ensure the quote open and close diff --git a/tajo-common/src/main/java/org/apache/tajo/schema/PgSQLIdentifierPolicy.java b/tajo-common/src/main/java/org/apache/tajo/schema/PgSQLIdentifierPolicy.java index 0aca2d7500..31f6d086ff 100644 --- a/tajo-common/src/main/java/org/apache/tajo/schema/PgSQLIdentifierPolicy.java +++ b/tajo-common/src/main/java/org/apache/tajo/schema/PgSQLIdentifierPolicy.java @@ -29,12 +29,12 @@ String getName() { } @Override - public String getIdentifierQuoteString() { + public char getIdentifierQuoteString() { return ANSI_SQL_QUOTE_STRING; } @Override - String getIdentifierSeperator() { + char getIdentifierSeperator() { return ANSI_SQL_SEPERATOR_STRING; } diff --git a/tajo-common/src/main/java/org/apache/tajo/schema/QualifiedIdentifier.java b/tajo-common/src/main/java/org/apache/tajo/schema/QualifiedIdentifier.java index b82057f3eb..c2759f3b05 100644 --- a/tajo-common/src/main/java/org/apache/tajo/schema/QualifiedIdentifier.java +++ b/tajo-common/src/main/java/org/apache/tajo/schema/QualifiedIdentifier.java @@ -20,7 +20,6 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Function; -import com.google.common.collect.Collections2; import com.google.common.collect.ImmutableList; import org.apache.tajo.util.StringUtils; @@ -30,6 +29,10 @@ import static org.apache.tajo.schema.IdentifierPolicy.DefaultPolicy; +/** + * Represents a qualified identifier. In other words, it represents identifiers for + * all kinds of of database objects, such as databases, tables, and columns. + */ public class QualifiedIdentifier { private ImmutableList names; @@ -37,17 +40,23 @@ private QualifiedIdentifier(ImmutableList names) { this.names = names; } - public String displayString(final IdentifierPolicy policy) { - return StringUtils.join(names, policy.getIdentifierSeperator(), new Function() { + /** + * Return a raw string representation for an identifier, which is equivalent to ones used in SQL statements. + * + * @param policy IdentifierPolicy + * @return Raw identifier string + */ + public String raw(final IdentifierPolicy policy) { + return StringUtils.join(names, "" + policy.getIdentifierSeperator(), new Function() { @Override public String apply(@Nullable Identifier identifier) { - return identifier.displayString(policy); + return identifier.raw(policy); } }); } - public String raw() { - return raw(DefaultPolicy()); + public String interned() { + return interned(DefaultPolicy()); } /** @@ -55,18 +64,18 @@ public String raw() { * @param policy Identifier Policy * @return raw string */ - public String raw(final IdentifierPolicy policy) { - return StringUtils.join(names, policy.getIdentifierSeperator(), new Function() { + public String interned(final IdentifierPolicy policy) { + return StringUtils.join(names, "" + policy.getIdentifierSeperator(), new Function() { @Override public String apply(@Nullable Identifier identifier) { - return identifier.raw(policy); + return identifier.interned(policy); } }); } @Override public String toString() { - return raw(DefaultPolicy()); + return interned(DefaultPolicy()); } @Override @@ -94,11 +103,16 @@ public boolean equals(Object obj) { return new QualifiedIdentifier(ImmutableList.copyOf(names)); } + /** + * It takes interned strings. It assumes all parameters already stripped. It is used only for tests. + * @param names interned strings + * @return QualifiedIdentifier + */ @VisibleForTesting public static QualifiedIdentifier $(String...names) { - ImmutableList.Builder builder = new ImmutableList.Builder(); + final ImmutableList.Builder builder = ImmutableList.builder(); for (String n : names) { - for (String split : n.split(StringUtils.escapeRegexp(DefaultPolicy().getIdentifierSeperator()))) { + for (String split : n.split(StringUtils.escapeRegexp(""+DefaultPolicy().getIdentifierSeperator()))) { builder.add(Identifier._(split, IdentifierUtil.isShouldBeQuoted(split))); } } diff --git a/tajo-common/src/main/java/org/apache/tajo/schema/package-info.java b/tajo-common/src/main/java/org/apache/tajo/schema/package-info.java new file mode 100644 index 0000000000..88ce44050b --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/schema/package-info.java @@ -0,0 +1,22 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Provide Schema and its relevant classes and utilities. + */ +package org.apache.tajo.schema; \ No newline at end of file diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Any.java b/tajo-common/src/main/java/org/apache/tajo/type/Any.java index d61afae04e..342fdb68f7 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Any.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Any.java @@ -18,16 +18,10 @@ package org.apache.tajo.type; -import org.apache.tajo.common.TajoDataTypes; - -import java.util.Objects; +import static org.apache.tajo.common.TajoDataTypes.Type.ANY; public class Any extends Type { - - public static Any INSTANCE = new Any(); - - @Override - public TajoDataTypes.Type baseType() { - return TajoDataTypes.Type.ANY; + public Any() { + super(ANY); } } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Array.java b/tajo-common/src/main/java/org/apache/tajo/type/Array.java index b6cf069329..3432238115 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Array.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Array.java @@ -18,47 +18,17 @@ package org.apache.tajo.type; -import org.apache.tajo.common.TajoDataTypes; +import com.google.common.collect.ImmutableList; -import java.util.Objects; +import static org.apache.tajo.common.TajoDataTypes.Type.ARRAY; -public class Array extends Type { - private final Type elementType; +public class Array extends TypeParamterizedType { public Array(Type elementType) { - this.elementType = elementType; + super(ARRAY, ImmutableList.of(elementType)); } public Type elementType() { - return this.elementType; - } - - @Override - public boolean hasParam() { - return true; - } - - @Override - public TajoDataTypes.Type baseType() { - return TajoDataTypes.Type.ARRAY; - } - - @Override - public String toString() { - return "ARRAY<" + elementType + ">"; - } - - @Override - public int hashCode() { - return Objects.hash(baseType(), elementType); - } - - @Override - public boolean equals(Object object) { - if (object instanceof Array) { - return elementType.equals(((Array)object).elementType); - } - - return false; + return this.params.get(0); } } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Blob.java b/tajo-common/src/main/java/org/apache/tajo/type/Blob.java index 104dd120f7..360ea13fba 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Blob.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Blob.java @@ -18,11 +18,10 @@ package org.apache.tajo.type; -import org.apache.tajo.common.TajoDataTypes; +import static org.apache.tajo.common.TajoDataTypes.Type.BLOB; public class Blob extends Type { - @Override - public TajoDataTypes.Type baseType() { - return TajoDataTypes.Type.BLOB; + public Blob() { + super(BLOB); } } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Bool.java b/tajo-common/src/main/java/org/apache/tajo/type/Bool.java index ddd7300a09..923e8e90ec 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Bool.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Bool.java @@ -18,11 +18,10 @@ package org.apache.tajo.type; -import org.apache.tajo.common.TajoDataTypes; +import static org.apache.tajo.common.TajoDataTypes.Type.BOOLEAN; public class Bool extends Type { - @Override - public TajoDataTypes.Type baseType() { - return TajoDataTypes.Type.BOOLEAN; + public Bool() { + super(BOOLEAN); } } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Char.java b/tajo-common/src/main/java/org/apache/tajo/type/Char.java index 827ef132c1..555df64874 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Char.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Char.java @@ -18,47 +18,16 @@ package org.apache.tajo.type; -import org.apache.tajo.common.TajoDataTypes; +import com.google.common.collect.ImmutableList; -import java.util.Objects; - -public class Char extends Type { - private final int length; +import static org.apache.tajo.common.TajoDataTypes.Type.CHAR; +public class Char extends ValueParamterizedType { public Char(int length) { - this.length = length; + super(CHAR, ImmutableList.of((Object)length)); } public int length() { - return length; - } - - @Override - public boolean hasParam() { - return true; - } - - @Override - public TajoDataTypes.Type baseType() { - return TajoDataTypes.Type.CHAR; - } - - @Override - public String toString() { - return "CHAR(" + length + ")"; - } - - @Override - public int hashCode() { - return Objects.hash(baseType(), length); - } - - @Override - public boolean equals(Object object) { - if (object instanceof Char) { - return length == ((Char) object).length; - } - - return false; + return (int) params.get(0); } } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Date.java b/tajo-common/src/main/java/org/apache/tajo/type/Date.java index 0febc1a123..a68ab97a49 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Date.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Date.java @@ -18,11 +18,10 @@ package org.apache.tajo.type; -import org.apache.tajo.common.TajoDataTypes; +import static org.apache.tajo.common.TajoDataTypes.Type.DATE; public class Date extends Type { - @Override - public TajoDataTypes.Type baseType() { - return TajoDataTypes.Type.DATE; + public Date() { + super(DATE); } } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Float4.java b/tajo-common/src/main/java/org/apache/tajo/type/Float4.java index 47ccd6b15c..b8c82181ee 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Float4.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Float4.java @@ -18,11 +18,10 @@ package org.apache.tajo.type; -import org.apache.tajo.common.TajoDataTypes; +import static org.apache.tajo.common.TajoDataTypes.Type.FLOAT4; public class Float4 extends Type { - @Override - public TajoDataTypes.Type baseType() { - return TajoDataTypes.Type.FLOAT4; + public Float4() { + super(FLOAT4); } } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Float8.java b/tajo-common/src/main/java/org/apache/tajo/type/Float8.java index 6d2702ca31..f36fac87c8 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Float8.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Float8.java @@ -18,11 +18,10 @@ package org.apache.tajo.type; -import org.apache.tajo.common.TajoDataTypes; +import static org.apache.tajo.common.TajoDataTypes.Type.FLOAT8; public class Float8 extends Type { - @Override - public TajoDataTypes.Type baseType() { - return TajoDataTypes.Type.FLOAT8; + public Float8() { + super(FLOAT8); } } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Inet4.java b/tajo-common/src/main/java/org/apache/tajo/type/Inet4.java index 05dc0cf198..541841ec07 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Inet4.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Inet4.java @@ -18,11 +18,10 @@ package org.apache.tajo.type; -import org.apache.tajo.common.TajoDataTypes; +import static org.apache.tajo.common.TajoDataTypes.Type.INET4; public class Inet4 extends Type { - @Override - public TajoDataTypes.Type baseType() { - return TajoDataTypes.Type.INET4; + public Inet4() { + super(INET4); } } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Int1.java b/tajo-common/src/main/java/org/apache/tajo/type/Int1.java index 9288328d36..d86d36e196 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Int1.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Int1.java @@ -18,11 +18,10 @@ package org.apache.tajo.type; -import org.apache.tajo.common.TajoDataTypes; +import static org.apache.tajo.common.TajoDataTypes.Type.INT1; public class Int1 extends Type { - @Override - public TajoDataTypes.Type baseType() { - return TajoDataTypes.Type.INT1; + public Int1() { + super(INT1); } } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Int2.java b/tajo-common/src/main/java/org/apache/tajo/type/Int2.java index 44db4cbfc5..e7c221541d 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Int2.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Int2.java @@ -18,11 +18,10 @@ package org.apache.tajo.type; -import org.apache.tajo.common.TajoDataTypes; +import static org.apache.tajo.common.TajoDataTypes.Type.INT2; public class Int2 extends Type { - @Override - public TajoDataTypes.Type baseType() { - return TajoDataTypes.Type.INT2; + public Int2() { + super(INT2); } } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Int4.java b/tajo-common/src/main/java/org/apache/tajo/type/Int4.java index 0837e2ce60..e8db7841a0 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Int4.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Int4.java @@ -18,11 +18,10 @@ package org.apache.tajo.type; -import org.apache.tajo.common.TajoDataTypes; +import static org.apache.tajo.common.TajoDataTypes.Type.INT4; public class Int4 extends Type { - @Override - public TajoDataTypes.Type baseType() { - return TajoDataTypes.Type.INT4; + public Int4() { + super(INT4); } } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Int8.java b/tajo-common/src/main/java/org/apache/tajo/type/Int8.java index 3d4049c667..999e9acd97 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Int8.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Int8.java @@ -18,11 +18,10 @@ package org.apache.tajo.type; -import org.apache.tajo.common.TajoDataTypes; +import static org.apache.tajo.common.TajoDataTypes.Type.INT8; public class Int8 extends Type { - @Override - public TajoDataTypes.Type baseType() { - return TajoDataTypes.Type.INT8; + public Int8() { + super(INT8); } } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Interval.java b/tajo-common/src/main/java/org/apache/tajo/type/Interval.java index 6c9952628b..d6dc17eda8 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Interval.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Interval.java @@ -18,11 +18,10 @@ package org.apache.tajo.type; -import org.apache.tajo.common.TajoDataTypes; +import static org.apache.tajo.common.TajoDataTypes.Type.INTERVAL; public class Interval extends Type { - @Override - public TajoDataTypes.Type baseType() { - return TajoDataTypes.Type.INTERVAL; + public Interval() { + super(INTERVAL); } } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Map.java b/tajo-common/src/main/java/org/apache/tajo/type/Map.java index 1978cb6434..07e6c7da27 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Map.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Map.java @@ -18,49 +18,23 @@ package org.apache.tajo.type; +import com.google.common.collect.ImmutableList; import org.apache.tajo.common.TajoDataTypes; import java.util.Objects; -public class Map extends Type { - private final Type keyType; - private final Type valueType; +import static org.apache.tajo.common.TajoDataTypes.Type.MAP; +public class Map extends TypeParamterizedType { public Map(Type keyType, Type valueType) { - this.keyType = keyType; - this.valueType = valueType; + super(MAP, ImmutableList.of(keyType, valueType)); } public Type keyType() { - return this.keyType; + return this.params.get(0); } public Type valueType() { - return this.valueType; - } - - @Override - public TajoDataTypes.Type baseType() { - return TajoDataTypes.Type.MAP; - } - - @Override - public String toString() { - return "MAP<" + keyType + "," + valueType + ">"; - } - - @Override - public int hashCode() { - return Objects.hash(baseType(), keyType, valueType); - } - - @Override - public boolean equals(Object object) { - if (object instanceof Map) { - Map other = (Map) object; - return keyType.equals(other.keyType) && valueType.equals(other.valueType); - } - - return false; + return this.params.get(1); } } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Null.java b/tajo-common/src/main/java/org/apache/tajo/type/Null.java index fe823e133a..74cf052a0b 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Null.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Null.java @@ -18,11 +18,10 @@ package org.apache.tajo.type; -import org.apache.tajo.common.TajoDataTypes; +import static org.apache.tajo.common.TajoDataTypes.Type.NULL_TYPE; public class Null extends Type { - @Override - public TajoDataTypes.Type baseType() { - return TajoDataTypes.Type.NULL_TYPE; + public Null() { + super(NULL_TYPE); } } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Numeric.java b/tajo-common/src/main/java/org/apache/tajo/type/Numeric.java index 6bb51cb2e7..04497976d0 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Numeric.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Numeric.java @@ -18,49 +18,21 @@ package org.apache.tajo.type; -import org.apache.tajo.common.TajoDataTypes; +import com.google.common.collect.ImmutableList; -import java.util.Objects; +import static org.apache.tajo.common.TajoDataTypes.Type.NUMERIC; -public class Numeric extends Type { - private final int precision; - private final int scale; +public class Numeric extends ValueParamterizedType { public Numeric(int precision, int scale) { - this.precision = precision; - this.scale = scale; + super(NUMERIC, ImmutableList.of((Object)precision, scale)); } public int precision() { - return this.precision; + return (int) this.params.get(0); } public int scale() { - return this.scale; - } - - @Override - public TajoDataTypes.Type baseType() { - return TajoDataTypes.Type.NUMERIC; - } - - @Override - public String toString() { - return "NUMERIC(" + precision + "," + scale + ")"; - } - - @Override - public int hashCode() { - return Objects.hash(baseType(), precision, scale); - } - - @Override - public boolean equals(Object object) { - if (object instanceof Numeric) { - Numeric other = (Numeric) object; - return precision == other.precision && scale == other.scale; - } - - return false; + return (int) this.params.get(1); } } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Protobuf.java b/tajo-common/src/main/java/org/apache/tajo/type/Protobuf.java index 59d3c1a94e..e286724644 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Protobuf.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Protobuf.java @@ -18,25 +18,17 @@ package org.apache.tajo.type; -import org.apache.tajo.common.TajoDataTypes; +import static org.apache.tajo.common.TajoDataTypes.Type.PROTOBUF; public class Protobuf extends Type { private String msgName; public Protobuf(String msgName) { + super(PROTOBUF); this.msgName = msgName; } public String getMessageName() { return this.msgName; } - - @Override - public TajoDataTypes.Type baseType() { - return TajoDataTypes.Type.PROTOBUF; - } - - public boolean hasParam() { - return true; - } } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Record.java b/tajo-common/src/main/java/org/apache/tajo/type/Record.java index e8fb6a83ba..ba9298d37b 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Record.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Record.java @@ -27,10 +27,16 @@ import java.util.List; import java.util.Objects; +import static org.apache.tajo.common.TajoDataTypes.Type.RECORD; + +/** + * Represents Record type + */ public class Record extends Type { private final ImmutableList memberTypes; public Record(Collection memberTypes) { + super(RECORD); this.memberTypes = ImmutableList.copyOf(memberTypes); } @@ -46,11 +52,6 @@ public List fields() { return this.memberTypes; } - @Override - public TajoDataTypes.Type baseType() { - return TajoDataTypes.Type.RECORD; - } - @Override public String toString() { return "RECORD(" + StringUtils.join(memberTypes, ", ") + ")"; diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Text.java b/tajo-common/src/main/java/org/apache/tajo/type/Text.java index e523afe952..1c3083427e 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Text.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Text.java @@ -20,9 +20,10 @@ import org.apache.tajo.common.TajoDataTypes; +import static org.apache.tajo.common.TajoDataTypes.Type.TEXT; + public class Text extends Type { - @Override - public TajoDataTypes.Type baseType() { - return TajoDataTypes.Type.TEXT; + public Text() { + super(TEXT); } } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Time.java b/tajo-common/src/main/java/org/apache/tajo/type/Time.java index 3b1414c5f9..12fe5f82db 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Time.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Time.java @@ -18,11 +18,10 @@ package org.apache.tajo.type; -import org.apache.tajo.common.TajoDataTypes; +import static org.apache.tajo.common.TajoDataTypes.Type.TIME; public class Time extends Type { - @Override - public TajoDataTypes.Type baseType() { - return TajoDataTypes.Type.TIME; + public Time() { + super(TIME); } } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Timestamp.java b/tajo-common/src/main/java/org/apache/tajo/type/Timestamp.java index 5b02314fa1..577f877419 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Timestamp.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Timestamp.java @@ -18,11 +18,10 @@ package org.apache.tajo.type; -import org.apache.tajo.common.TajoDataTypes; +import static org.apache.tajo.common.TajoDataTypes.Type.TIMESTAMP; public class Timestamp extends Type { - @Override - public TajoDataTypes.Type baseType() { - return TajoDataTypes.Type.TIMESTAMP; + public Timestamp() { + super(TIMESTAMP); } } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Type.java b/tajo-common/src/main/java/org/apache/tajo/type/Type.java index 89f9b8dc58..58ccc71a60 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Type.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Type.java @@ -19,11 +19,16 @@ package org.apache.tajo.type; import org.apache.tajo.common.TajoDataTypes; +import org.apache.tajo.exception.TajoRuntimeException; +import org.apache.tajo.exception.UnsupportedException; import org.apache.tajo.schema.Field; import java.util.Arrays; import java.util.Collection; +/** + * Represents Type + */ public abstract class Type implements Cloneable { // No paramter types @@ -44,12 +49,32 @@ public abstract class Type implements Cloneable { public static final Blob Blob = new Blob(); public static final Inet4 Inet4 = new Inet4(); - public abstract TajoDataTypes.Type baseType(); + protected TajoDataTypes.Type baseType; + + public Type(TajoDataTypes.Type baseType) { + this.baseType = baseType; + } - public boolean hasParam() { + public TajoDataTypes.Type baseType() { + return baseType; + } + + public boolean isTypeParameterized() { return false; } + public boolean isValueParameterized() { + return false; + } + + public Collection getTypeParameters() { + throw new TajoRuntimeException(new UnsupportedException()); + } + + public Collection getValueParameters() { + throw new TajoRuntimeException(new UnsupportedException()); + } + protected static String typeName(TajoDataTypes.Type type) { return type.name().toUpperCase(); } @@ -98,7 +123,7 @@ public static Varchar Varchar(int len) { return new Varchar(len); } - public static Record Struct(Collection types) { + public static Record Record(Collection types) { return new Record(types); } @@ -106,7 +131,7 @@ public static Array Array(Type type) { return new Array(type); } - public static Record Struct(Field... types) { + public static Record Record(Field... types) { return new Record(Arrays.asList(types)); } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/TypeParamterizedType.java b/tajo-common/src/main/java/org/apache/tajo/type/TypeParamterizedType.java new file mode 100644 index 0000000000..9f84f41eb0 --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/type/TypeParamterizedType.java @@ -0,0 +1,77 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.type; + +import com.google.common.collect.ImmutableList; +import org.apache.tajo.common.TajoDataTypes; +import org.apache.tajo.util.StringUtils; + +import java.util.Collection; +import java.util.Objects; + +/** + * Represents a type-parameterized Type (e.g., Map<Int, String> and Array<Double>) + */ +public abstract class TypeParamterizedType extends Type { + protected ImmutableList params; + + public TypeParamterizedType(TajoDataTypes.Type type, ImmutableList params) { + super(type); + this.params = params; + } + + @Override + public boolean isTypeParameterized() { + return true; + } + + @Override + public Collection getTypeParameters() { + return params; + } + + @Override + public boolean equals(Object object) { + if (object == this) { + return true; + } + + if (object instanceof TypeParamterizedType) { + TypeParamterizedType other = (TypeParamterizedType) object; + return this.baseType.equals(other.baseType) && params.equals(other.params); + } + + return false; + } + + @Override + public int hashCode() { + return Objects.hash(baseType(), params); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(typeName(this.baseType)); + sb.append("<"); + sb.append(StringUtils.join(params, ",")); + sb.append(">"); + return sb.toString(); + } +} diff --git a/tajo-common/src/main/java/org/apache/tajo/type/TypeSerializer.java b/tajo-common/src/main/java/org/apache/tajo/type/TypeSerializer.java new file mode 100644 index 0000000000..5806bdbac6 --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/type/TypeSerializer.java @@ -0,0 +1,367 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.type; + +import com.google.common.base.Function; +import com.google.common.collect.ImmutableList; +import org.apache.tajo.common.TajoDataTypes; +import org.apache.tajo.exception.TajoInternalError; +import org.apache.tajo.exception.UnsupportedException; +import org.apache.tajo.schema.Field; +import org.apache.tajo.schema.IdentifierUtil; +import org.apache.tajo.schema.QualifiedIdentifier; +import org.apache.tajo.util.StringUtils; + +import javax.annotation.Nullable; +import java.util.List; +import java.util.Stack; + +import static org.apache.tajo.schema.IdentifierPolicy.DefaultPolicy; +import static org.apache.tajo.type.Type.Char; +import static org.apache.tajo.type.Type.Varchar; + +/** + * This class enables to serialize a type into a string representation and vice versa. + */ +public class TypeSerializer { + + /** + * Serialize a type into a string representation + * @param type A type + * @return A type string representation + */ + public static String serialize(Type type) { + StringBuilder sb = new StringBuilder(type.baseType().name()); + + if (type.isTypeParameterized()) { + sb.append("<"); + sb.append(StringUtils.join(type.getTypeParameters(), ",", new Function() { + @Override + public String apply(@Nullable Type type) { + return TypeSerializer.serialize(type); + } + })); + sb.append(">"); + } + + // Assume all parameter values are integers. + if (type.isValueParameterized()) { + sb.append("("); + sb.append(StringUtils.join(type.getValueParameters(), ",")); + sb.append(")"); + } + + if (type.isStruct()) { + Record record = (Record) type; + sb.append("["); + sb.append(StringUtils.join(record.fields(), ",", new Function() { + @Override + public String apply(@Nullable Field field) { + return serializeField(field); + } + })); + sb.append("]"); + } + + return sb.toString(); + } + + /** + * Make a string from a field + * @param field A field + * @return String representation for a field + */ + static String serializeField(Field field) { + return field.name().raw(DefaultPolicy()) + " " + serialize(field.type()); + } + + /** + * Parse and transform a type string representation to a Type. + * @param signature Type string representation + * @return Type + */ + public static Type deserialize(String signature) { + + // termination condition in this recursion + if (!(signature.contains("<") || signature.contains("(") || signature.contains("["))) { + return createType(signature, + ImmutableList.of(), + ImmutableList.of(), + ImmutableList.of()); + } + + final Stack stack = new Stack<>(); + final Stack spanStack = new Stack<>(); + String baseType = null; + for (int i = 0; i < signature.length(); i++) { + char c = signature.charAt(i); + + if (c == '<') { + if (stack.isEmpty()) { + assertArgument(baseType == null, "Expected baseName to be null"); + baseType = signature.substring(0, i); + } + stack.push('<'); + spanStack.push(i + 1); + + } else if (c == '>') { + assertArgument(stack.pop() == '<', "Bad signature: '%s'", signature); + int paramStartIdx = spanStack.pop(); + + if (stack.isEmpty()) { // ensure outermost parameters + return createType(baseType, + parseList(signature.substring(paramStartIdx, i), new Function() { + @Override + public Type apply(@Nullable String s) { + return deserialize(s); + } + }), + ImmutableList.of(), + ImmutableList.of()); + } + + } else if (c == '[') { + if (stack.isEmpty()) { + assertArgument(baseType == null, "Expected baseName to be null"); + baseType = signature.substring(0, i); + } + + stack.push('['); + spanStack.push(i + 1); + + } else if (c == ']') { + assertArgument(stack.pop() == '[', "Bad signature: '%s'", signature); + + int paramStartIdx = spanStack.pop(); + if (stack.isEmpty()) { // ensure outermost parameters + return createType(baseType, + ImmutableList.of(), + ImmutableList.of(), + parseList(signature.substring(paramStartIdx, i), new Function() { + @Override + public Field apply(@Nullable String s) { + return parseField(s); + } + })); + } + + } else if (c == '(') { + if (stack.isEmpty()) { + assertArgument(baseType == null, "Expected baseName to be null"); + baseType = signature.substring(0, i); + } + stack.push('('); + spanStack.push(i + 1); + + } else if (c == ')') { + assertArgument(stack.pop() == '(', "Bad signature: '%s'", signature); + int paramStartIdx = spanStack.pop(); + + if (stack.isEmpty()) { // ensure outermost parameters + return createType(baseType, + ImmutableList.of(), + parseList(signature.substring(paramStartIdx, i), new Function() { + @Override + public Integer apply(@Nullable String s) { + return parseValue(s); + } + }), + ImmutableList.of()); + } + } + } + + return null; + } + + public static int parseValue(String literal) { + try { + return Integer.parseInt(literal); + } catch (NumberFormatException e) { + throw new TajoInternalError(e); + } + } + + /** + * Parse a string delimited by comma into a list of object instances depending on 
itemParser
. + * @param str String delimited by comma + * @param itemParser A function to transform a string to an object. + * @param Type to be transformed from a string + * @return List of object instances + */ + static List parseList(String str, Function itemParser) { + if (!str.contains(",")) { // if just one item + return ImmutableList.of(itemParser.apply(str)); + } + + final ImmutableList.Builder fields = ImmutableList.builder(); + final Stack stack = new Stack<>(); + int paramStartIdx = 0; + for (int i = 0; i < str.length(); i++) { + char c = str.charAt(i); + + if (c == '<' || c == '[' || c == '(') { + stack.push(c); + } else if (c == '>') { + assertArgument(stack.pop() == '<', "Bad signature: '%s'", str); + } else if (c == ']') { + assertArgument(stack.pop() == '[', "Bad signature: '%s'", str); + } else if (c == ')') { + assertArgument(stack.pop() == '(', "Bad signature: '%s'", str); + } else if (c == ',') { + if (stack.isEmpty()) { // ensure outermost type parameters + fields.add(itemParser.apply(str.substring(paramStartIdx, i))); + paramStartIdx = i + 1; + } + } + } + + assertArgument(stack.empty(), "Bad signature: '%s'", str); + if (paramStartIdx < str.length()) { + fields.add(itemParser.apply(str.substring(paramStartIdx, str.length()))); + } + + return fields.build(); + } + + /** + * Make a field from a string representation + * @param str String + * @return Field + */ + static Field parseField(String str) { + // A field consists of an identifier and a type, and they are delimited by space. + if (!str.contains(" ")) { + assertArgument(false, "Bad field signature: '%s'", str); + } + + // Stack to track the nested bracket depth + Stack stack = new Stack<>(); + int paramStartIdx = 0; + for (int i = 0; i < str.length(); i++) { + char c = str.charAt(i); + + if (c == '<' || c == '[' || c == '(') { + stack.push(c); + } else if (c == '>') { // for validation + assertArgument(stack.pop() == '<', "Bad field signature: '%s'", str); + } else if (c == ']') { // for validation + assertArgument(stack.pop() == '[', "Bad field signature: '%s'", str); + } else if (c == ')') { // for validation + assertArgument(stack.pop() == '(', "Bad field signature: '%s'", str); + + } else if (c == ' ') { + if (stack.isEmpty()) { // ensure outermost type parameters + QualifiedIdentifier identifier = + IdentifierUtil.makeIdentifier(str.substring(paramStartIdx, i), DefaultPolicy()); + String typePart = str.substring(i + 1, str.length()); + return new Field(identifier, deserialize(typePart)); + } + } + } + + return null; + } + + static Type createType(String baseTypeStr, + List typeParams, + List valueParams, + List fieldParams) { + final TajoDataTypes.Type baseType; + + try { + baseType = TajoDataTypes.Type.valueOf(baseTypeStr); + } catch (Throwable t) { + throw new TajoInternalError(new UnsupportedException(baseTypeStr)); + } + + switch (baseType) { + case CHAR: { + assertArgument(valueParams.size() == 1, + "Char type requires 1 integer parameters, but it takes (%s).", StringUtils.join(typeParams)); + return Char(valueParams.get(0)); + } + case VARCHAR: { + assertArgument(valueParams.size() == 1, + "Varchar type requires 1 integer parameters, but it takes (%s).", StringUtils.join(typeParams)); + return Varchar(valueParams.get(0)); + } + case TEXT: return Type.Text; + + case BOOLEAN: return Type.Bool; + case INT1: return Type.Int1; + case INT2: return Type.Int2; + case INT4: return Type.Int4; + case INT8: return Type.Int8; + case FLOAT4: return Type.Float4; + case FLOAT8: return Type.Float8; + case NUMERIC: { + if (valueParams.size() == 0) { + return Numeric.Numeric(); + + } else { + for (Object p : valueParams) { + assertArgument(p instanceof Integer, "Numeric type requires integer parameters"); + } + if (valueParams.size() == 1) { + return Numeric.Numeric(valueParams.get(0)); + } else if (valueParams.size() == 2) { + return Numeric.Numeric(valueParams.get(0), valueParams.get(1)); + } else { + assertArgument(false, + "Numeric type can take 2 or less integer parameters, but it takes (%s).", StringUtils.join(valueParams)); + } + } + } + + case DATE: return Type.Date; + case TIME: return Type.Time; + case TIMESTAMP: return Type.Timestamp; + case INTERVAL: return Type.Interval; + case BLOB: return Type.Blob; + case INET4: return Type.Inet4; + + case ARRAY: { + assertArgument(typeParams.size() == 1, + "Array Type requires 1 type parameters, but it takes (%s).", StringUtils.join(typeParams)); + return Type.Array(typeParams.get(0)); + } + case MAP: { + assertArgument(typeParams.size() == 2, + "Map Type requires 2 type parameters, but it takes (%s).", StringUtils.join(typeParams)); + + return Type.Map(typeParams.get(0), typeParams.get(1)); + } + case RECORD: { + assertArgument(fieldParams.size() >= 1, + "Record Type requires at least 1 field parameters, but it takes (%s).", StringUtils.join(fieldParams)); + return Type.Record(fieldParams); + } + + default: + throw new TajoInternalError(new UnsupportedException(baseTypeStr)); + } + } + + static void assertArgument(boolean condition, String message, Object ... arguments) { + if (!condition) { + throw new TajoInternalError(String.format(message, arguments)); + } + } +} diff --git a/tajo-common/src/main/java/org/apache/tajo/type/ValueParamterizedType.java b/tajo-common/src/main/java/org/apache/tajo/type/ValueParamterizedType.java new file mode 100644 index 0000000000..c53db46c90 --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/type/ValueParamterizedType.java @@ -0,0 +1,77 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.type; + +import com.google.common.collect.ImmutableList; +import org.apache.tajo.common.TajoDataTypes; +import org.apache.tajo.util.StringUtils; + +import java.util.Collection; +import java.util.Objects; + +/** + * Represents a type which takes value parameters (e.g., Numeric(10, 6)) + */ +public abstract class ValueParamterizedType extends Type { + protected ImmutableList params; + + public ValueParamterizedType(TajoDataTypes.Type type, ImmutableList params) { + super(type); + this.params = params; + } + + @Override + public boolean isValueParameterized() { + return true; + } + + @Override + public Collection getValueParameters() { + return params; + } + + @Override + public boolean equals(Object object) { + if (object == this) { + return true; + } + + if (object instanceof ValueParamterizedType) { + ValueParamterizedType other = (ValueParamterizedType) object; + return this.baseType.equals(other.baseType) && params.equals(other.params); + } + + return false; + } + + @Override + public int hashCode() { + return Objects.hash(baseType(), params); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(typeName(this.baseType)); + sb.append("("); + sb.append(StringUtils.join(params, ",")); + sb.append(")"); + return sb.toString(); + } +} diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Varchar.java b/tajo-common/src/main/java/org/apache/tajo/type/Varchar.java index 077de41956..e953e0236d 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Varchar.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Varchar.java @@ -18,47 +18,17 @@ package org.apache.tajo.type; -import org.apache.tajo.common.TajoDataTypes; +import com.google.common.collect.ImmutableList; -import java.util.Objects; +import static org.apache.tajo.common.TajoDataTypes.Type.VARCHAR; -public class Varchar extends Type { - private final int length; +public class Varchar extends ValueParamterizedType { public Varchar(int length) { - this.length = length; + super(VARCHAR, ImmutableList.of((Object)length)); } public int length() { - return length; - } - - @Override - public TajoDataTypes.Type baseType() { - return TajoDataTypes.Type.VARCHAR; - } - - @Override - public boolean hasParam() { - return true; - } - - @Override - public String toString() { - return "VARCHAR(" + length + ")"; - } - - @Override - public int hashCode() { - return Objects.hash(baseType(), length); - } - - @Override - public boolean equals(Object object) { - if (object instanceof Varchar) { - return length == ((Varchar) object).length; - } - - return false; + return (int) params.get(0); } } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/package-info.java b/tajo-common/src/main/java/org/apache/tajo/type/package-info.java new file mode 100644 index 0000000000..6ec63eb9d1 --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/type/package-info.java @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Design Consideration: + *
    + *
  • Parameterized Type
    • + *
+ * + * numeric (precise, scale) + * record (x int) + * map[int, text] + */ +package org.apache.tajo.type; \ No newline at end of file diff --git a/tajo-common/src/test/java/org/apache/tajo/schema/TestTajoIdentifierPolicy.java b/tajo-common/src/test/java/org/apache/tajo/schema/TestTajoIdentifierPolicy.java index 72027fe824..cb777a7a45 100644 --- a/tajo-common/src/test/java/org/apache/tajo/schema/TestTajoIdentifierPolicy.java +++ b/tajo-common/src/test/java/org/apache/tajo/schema/TestTajoIdentifierPolicy.java @@ -45,10 +45,10 @@ public void testQualifiedIdentifiers() throws Exception { } public static void assertIdentifier(IdentifierPolicy p, Identifier id, String expected) { - assertEquals(expected, id.displayString(p)); + assertEquals(expected, id.raw(p)); } public static void assertQualifiedIdentifier(IdentifierPolicy p, QualifiedIdentifier id, String expected) { - assertEquals(expected, id.displayString(p)); + assertEquals(expected, id.raw(p)); } } \ No newline at end of file diff --git a/tajo-common/src/test/java/org/apache/tajo/type/TestType.java b/tajo-common/src/test/java/org/apache/tajo/type/TestType.java index a399286888..e73ca864bc 100644 --- a/tajo-common/src/test/java/org/apache/tajo/type/TestType.java +++ b/tajo-common/src/test/java/org/apache/tajo/type/TestType.java @@ -36,7 +36,6 @@ import static org.apache.tajo.type.Type.Int8; import static org.apache.tajo.type.Type.Map; import static org.apache.tajo.type.Type.Numeric; -import static org.apache.tajo.type.Type.Struct; import static org.apache.tajo.type.Type.Time; import static org.apache.tajo.type.Type.Timestamp; import static org.apache.tajo.type.Type.Varchar; @@ -71,7 +70,7 @@ public final void testPrimitiveTypes() { assertEquals(varchar.baseType(), VARCHAR); assertEquals(varchar.length(), 2); - Record record = Struct(Field("x", Int8), Field("y", Array(Float8))); + Record record = Type.Record(Field("x", Int8), Field("y", Array(Float8))); assertEquals(record.baseType(), RECORD); assertEquals(record.field(0).baseType(), INT8); assertEquals(record.field(0).name(), $("x")); @@ -112,7 +111,7 @@ public final void testToString() { Varchar varchar = Varchar(2); assertEquals("VARCHAR(2)", varchar.toString()); - Record record = Struct(Field("x", Int8), Field("y", Array(Float8))); + Record record = Type.Record(Field("x", Int8), Field("y", Array(Float8))); assertEquals("RECORD(x (INT8), y (ARRAY))", record.toString()); Map map = Map(Int8, Array(Timestamp)); diff --git a/tajo-common/src/test/java/org/apache/tajo/type/TestTypeSerializer.java b/tajo-common/src/test/java/org/apache/tajo/type/TestTypeSerializer.java new file mode 100644 index 0000000000..fb32df81b5 --- /dev/null +++ b/tajo-common/src/test/java/org/apache/tajo/type/TestTypeSerializer.java @@ -0,0 +1,126 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.type; + +import com.google.common.base.Function; +import org.apache.tajo.schema.Field; +import org.junit.Test; + +import javax.annotation.Nullable; +import java.util.List; + +import static org.apache.tajo.schema.Field.Field; +import static org.apache.tajo.schema.Identifier._; +import static org.apache.tajo.schema.QualifiedIdentifier.$; +import static org.apache.tajo.type.Type.Array; +import static org.apache.tajo.type.Type.Blob; +import static org.apache.tajo.type.Type.Bool; +import static org.apache.tajo.type.Type.Char; +import static org.apache.tajo.type.Type.Float4; +import static org.apache.tajo.type.Type.Float8; +import static org.apache.tajo.type.Type.Inet4; +import static org.apache.tajo.type.Type.Int1; +import static org.apache.tajo.type.Type.Int2; +import static org.apache.tajo.type.Type.Int4; +import static org.apache.tajo.type.Type.Int8; +import static org.apache.tajo.type.Type.Map; +import static org.apache.tajo.type.Type.Numeric; +import static org.apache.tajo.type.Type.Record; +import static org.apache.tajo.type.Type.Text; +import static org.apache.tajo.type.Type.Varchar; +import static org.apache.tajo.type.TypeSerializer.*; +import static org.junit.Assert.assertEquals; + +public class TestTypeSerializer { + + public static final Record RECORD_POINT = Record(Field($("x"), Float8), Field($("y"), Float8)); + public static final Record RECORD_PERSON = + Record( + Field.Record($("name"), Field($(_("FirstName", true)), Text), Field($(_("LastName", true)), Text)), + Field($("age"), Int2), + Field.Record($("addr"), Field($("city"), Text), Field($("state"), Text)) + ); + + @Test + public final void testTypesWithoutParams() { + assertSerialize(Bool); + assertSerialize(Int1); + assertSerialize(Int2); + assertSerialize(Int4); + assertSerialize(Int8); + assertSerialize(Float4); + assertSerialize(Float8); + assertSerialize(Text); + assertSerialize(Blob); + assertSerialize(Inet4); + } + + @Test + public final void testValueParams() { + assertSerialize(Numeric()); + assertSerialize(Numeric(10)); + assertSerialize(Numeric(10, 12)); + + assertSerialize(Char(256)); + assertSerialize(Varchar(256)); + } + + @Test + public final void testTypeParams() { + assertSerialize(Array(Float8)); + assertSerialize(Map(Text, Float8)); + + // nested + assertSerialize(Array(Array(Array(Float8)))); + assertSerialize(Array(Map(Text, Array(Float8)))); + assertSerialize(Map(Text, Array(Float8))); + assertSerialize(Map(Text, Array(RECORD_POINT))); + } + + @Test + public final void testFieldParams() { + assertSerialize(RECORD_POINT); + assertSerialize(Record(Field.Record($("rectangle"), + Field($("left-bottom"), RECORD_POINT), + Field($("right-top"), RECORD_POINT)))); + + // nested with quoted identifiers + assertSerialize(RECORD_PERSON); + assertSerialize(Record(Field.Record($("couple"), + Field($("husband"), RECORD_PERSON), + Field($("wife"), RECORD_PERSON)))); + } + + @Test + public final void testParseTypeList() { + List types = parseList("TEXT,ARRAY", new Function() { + @Override + public Type apply(@Nullable String s) { + return deserialize(s); + } + }); + assertEquals(2, types.size()); + assertEquals(Text, types.get(0)); + assertEquals(Array(Float8), types.get(1)); + } + + public final void assertSerialize(Type type) { + assertEquals(type, deserialize(serialize(type))); + } +} \ No newline at end of file diff --git a/tajo-core-tests/src/test/resources/queries/TestCreateTable/testArrayType1.sql b/tajo-core-tests/src/test/resources/queries/TestCreateTable/testArrayType1.sql index f4d387445d..30ee94c01e 100644 --- a/tajo-core-tests/src/test/resources/queries/TestCreateTable/testArrayType1.sql +++ b/tajo-core-tests/src/test/resources/queries/TestCreateTable/testArrayType1.sql @@ -1 +1 @@ -CREATE TABLE array_table1 (vec array); \ No newline at end of file +CREATE TABLE d11.array_table1 (vec array); \ No newline at end of file diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java index 08081a567b..4996bf5fef 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java @@ -24,8 +24,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import com.google.common.collect.Sets; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.Path; import org.apache.tajo.BuiltinStorages; import org.apache.tajo.OverridableConf; @@ -72,7 +70,6 @@ * This class creates a logical plan from a nested tajo algebra expression ({@link org.apache.tajo.algebra}) */ public class LogicalPlanner extends BaseAlgebraVisitor { - private static Log LOG = LogFactory.getLog(LogicalPlanner.class); private final CatalogService catalog; private final StorageService storage; @@ -2126,7 +2123,7 @@ public static Type convertDataType(DataTypeExpr dataType) { } else if (type == TajoDataTypes.Type.ARRAY) { return Array(convertDataType(dataType.getElementType())); } else if (type == TajoDataTypes.Type.RECORD) { - return Struct(convertTableElementsSchema(dataType.getNestedRecordTypes())); + return Record(convertTableElementsSchema(dataType.getNestedRecordTypes())); } else if (type == TajoDataTypes.Type.NUMERIC) { if (dataType.hasLengthOrPrecision() && dataType.hasScale()) { return Numeric(dataType.getLengthOrPrecision(), dataType.getScale()); diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/FieldEval.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/FieldEval.java index 396832964b..b527780ac6 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/FieldEval.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/FieldEval.java @@ -56,10 +56,10 @@ public EvalNode bind(EvalContext evalContext, Schema schema) { super.bind(evalContext, schema); // TODO - column namespace should be improved to simplify name handling and resolving. - fieldId = schema.getColumnId(field.name().raw()); + fieldId = schema.getColumnId(field.name().interned()); if (fieldId == -1) { // fallback - fieldId = schema.getColumnIdByName(field.name().raw()); + fieldId = schema.getColumnIdByName(field.name().interned()); } if (fieldId == -1) { @@ -95,11 +95,11 @@ public Column getColumnRef() { } public String getQualifier() { - return IdentifierUtil.extractQualifier(field.name().raw()); + return IdentifierUtil.extractQualifier(field.name().interned()); } public String getColumnName() { - return IdentifierUtil.extractSimpleName(field.name().raw()); + return IdentifierUtil.extractSimpleName(field.name().interned()); } public void replaceColumnRef(String columnName) { @@ -108,7 +108,7 @@ public void replaceColumnRef(String columnName) { @Override public String getName() { - return this.field.name().raw(); + return this.field.name().interned(); } public String toString() { From 7645d450b196cc96d3d7e3d46ac6a3b3b03a08cf Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Mon, 2 May 2016 20:07:12 -0700 Subject: [PATCH 15/23] Add TypeAdapter to transform Type to Json documentation. --- .../java/org/apache/tajo/catalog/Column.java | 5 ++- .../tajo/catalog/json/CatalogGsonHelper.java | 1 + .../apache/tajo/json/CommonGsonHelper.java | 1 + .../org/apache/tajo/json/TypeAdapter.java | 44 +++++++++++++++++++ .../plan/rewrite/BaseSchemaBuildPhase.java | 7 +++ .../tajo/plan/serder/PlanGsonHelper.java | 1 + 6 files changed, 57 insertions(+), 2 deletions(-) create mode 100644 tajo-common/src/main/java/org/apache/tajo/json/TypeAdapter.java diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java index c685b8c60a..a79008394a 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java @@ -19,6 +19,7 @@ package org.apache.tajo.catalog; import com.google.common.base.Objects; +import com.google.gson.annotations.Expose; import org.apache.tajo.catalog.json.CatalogGsonHelper; import org.apache.tajo.catalog.proto.CatalogProtos.ColumnProto; import org.apache.tajo.common.ProtoObject; @@ -32,8 +33,8 @@ * Describes a column. It is an immutable object. */ public class Column implements ProtoObject, GsonObject { - protected String name; - protected Type type; + @Expose protected String name; + @Expose protected Type type; /** * Column Constructor diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/CatalogGsonHelper.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/CatalogGsonHelper.java index c145ecdd3c..27c2309043 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/CatalogGsonHelper.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/json/CatalogGsonHelper.java @@ -48,6 +48,7 @@ private static Map> registerAdapters() { adapters.put(Datum.class, new DatumAdapter()); adapters.put(DataType.class, new DataTypeAdapter()); adapters.put(Schema.class, new SchemaAdapter()); + adapters.put(org.apache.tajo.type.Type.class, new TypeAdapter()); return adapters; } diff --git a/tajo-common/src/main/java/org/apache/tajo/json/CommonGsonHelper.java b/tajo-common/src/main/java/org/apache/tajo/json/CommonGsonHelper.java index d838b6ad0c..68d44654e1 100644 --- a/tajo-common/src/main/java/org/apache/tajo/json/CommonGsonHelper.java +++ b/tajo-common/src/main/java/org/apache/tajo/json/CommonGsonHelper.java @@ -40,6 +40,7 @@ private CommonGsonHelper() { private static Map> registerAdapters() { Map> adapters = new HashMap<>(); adapters.put(Datum.class, new DatumAdapter()); + adapters.put(Type.class, new TypeAdapter()); return adapters; } diff --git a/tajo-common/src/main/java/org/apache/tajo/json/TypeAdapter.java b/tajo-common/src/main/java/org/apache/tajo/json/TypeAdapter.java new file mode 100644 index 0000000000..54462d2f54 --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/json/TypeAdapter.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.json; + +import com.google.gson.*; +import org.apache.tajo.common.TajoDataTypes; +import org.apache.tajo.common.TajoDataTypes.DataType; +import org.apache.tajo.type.TypeSerializer; + +import java.lang.reflect.Type; + + +public class TypeAdapter implements GsonSerDerAdapter { + + @Override + public org.apache.tajo.type.Type deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) + throws JsonParseException { + JsonObject obj = (JsonObject) json; + return TypeSerializer.deserialize(CommonGsonHelper.getOrDie(obj, "type").getAsString()); + } + + @Override + public JsonElement serialize(org.apache.tajo.type.Type src, Type typeOfSrc, JsonSerializationContext context) { + JsonObject json = new JsonObject(); + json.addProperty("type", TypeSerializer.serialize(src)); + return json; + } +} diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseSchemaBuildPhase.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseSchemaBuildPhase.java index a262e563e0..14fb3ef8dd 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseSchemaBuildPhase.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/BaseSchemaBuildPhase.java @@ -22,6 +22,7 @@ import org.apache.tajo.algebra.*; import org.apache.tajo.catalog.*; import org.apache.tajo.common.TajoDataTypes; +import org.apache.tajo.exception.NotImplementedException; import org.apache.tajo.exception.TajoException; import org.apache.tajo.exception.UndefinedColumnException; import org.apache.tajo.plan.*; @@ -40,6 +41,8 @@ import java.util.*; +import static org.apache.tajo.common.TajoDataTypes.Type.RECORD; + /** * BaseSchemaBuildPhase builds a basic schema information of tables which have pre-defined schema. * For example, tables like the below example have pre-defined schema. @@ -259,6 +262,10 @@ private List buildTargets(LogicalPlanner.PlanContext context, List> registerAdapters() { adapters.put(Datum.class, new DatumAdapter()); adapters.put(DataType.class, new DataTypeAdapter()); adapters.put(TimeZone.class, new TimeZoneGsonSerdeAdapter()); + adapters.put(org.apache.tajo.type.Type.class, new TypeAdapter()); adapters.put(Schema.class, new SchemaAdapter()); return adapters; From 129213d038be928469b3ed982584bc859aa7e3d0 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Fri, 6 May 2016 15:27:20 -0700 Subject: [PATCH 16/23] Fix all unit tests. --- .../main/java/org/apache/tajo/catalog/TypeConverter.java | 8 ++++++++ .../org/apache/tajo/cli/tsql/TestTajoCliNegatives.java | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TypeConverter.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TypeConverter.java index b42b6affca..1f3d2e7234 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TypeConverter.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TypeConverter.java @@ -122,6 +122,14 @@ public static Type convert(TajoDataTypes.Type legacyBaseType) { return Array(Float4); case FLOAT8_ARRAY: return Array(Float8); + case TIMESTAMP_ARRAY: + return Array(Timestamp); + case DATE_ARRAY: + return Array(Date); + case TIME_ARRAY: + return Array(Time); + case TEXT_ARRAY: + return Array(Text); default: throw new TajoRuntimeException(new UnsupportedException(legacyBaseType.name())); diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestTajoCliNegatives.java b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestTajoCliNegatives.java index b5d2e3795b..df6617207a 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestTajoCliNegatives.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/cli/tsql/TestTajoCliNegatives.java @@ -124,7 +124,7 @@ public void testQueryNotImplementedFeature() throws Exception { client.updateQuery("CREATE TABLE TestTajoCliNegatives.table12u79 ( name RECORD(last TEXT, first TEXT) )"); assertScriptFailure("select name FROM TestTajoCliNegatives.table12u79", - "ERROR: org.apache.tajo.exception.NotImplementedException: not implemented feature: record type projection\n"); + "ERROR: not implemented feature: record projection\n"); } finally { client.updateQuery("DROP TABLE IF EXISTS TestTajoCliNegatives.table12u79"); From a0924a4c612cadf3f6a50da0bcf707a888fdc8ff Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Mon, 9 May 2016 21:21:54 -0700 Subject: [PATCH 17/23] Add TypeStringEncoder and TypeProtobufEncoder. --- .../apache/tajo/catalog/TypeConverter.java | 79 +------- .../tajo/catalog/store/AbstractDBStore.java | 2 +- .../src/main/java/org/apache/tajo/Assert.java | 30 +++ .../org/apache/tajo/json/TypeAdapter.java | 8 +- .../java/org/apache/tajo/schema/Field.java | 2 +- .../org/apache/tajo/schema/Identifier.java | 16 +- .../tajo/schema/QualifiedIdentifier.java | 33 ++- .../main/java/org/apache/tajo/type/Any.java | 2 +- .../main/java/org/apache/tajo/type/Blob.java | 2 +- .../main/java/org/apache/tajo/type/Bool.java | 2 +- .../main/java/org/apache/tajo/type/Char.java | 4 +- .../main/java/org/apache/tajo/type/Date.java | 2 +- .../java/org/apache/tajo/type/Float4.java | 2 +- .../java/org/apache/tajo/type/Float8.java | 2 +- .../main/java/org/apache/tajo/type/Inet4.java | 2 +- .../main/java/org/apache/tajo/type/Int1.java | 2 +- .../main/java/org/apache/tajo/type/Int2.java | 2 +- .../main/java/org/apache/tajo/type/Int4.java | 2 +- .../main/java/org/apache/tajo/type/Int8.java | 2 +- .../java/org/apache/tajo/type/Interval.java | 2 +- .../main/java/org/apache/tajo/type/Null.java | 2 +- .../java/org/apache/tajo/type/Numeric.java | 6 +- .../org/apache/tajo/type/PrimitiveType.java | 27 +++ .../java/org/apache/tajo/type/Protobuf.java | 2 +- .../java/org/apache/tajo/type/Record.java | 31 ++- .../main/java/org/apache/tajo/type/Text.java | 4 +- .../main/java/org/apache/tajo/type/Time.java | 2 +- .../java/org/apache/tajo/type/Timestamp.java | 2 +- .../main/java/org/apache/tajo/type/Type.java | 14 +- .../org/apache/tajo/type/TypeFactory.java | 181 +++++++++++++++++ .../tajo/type/TypeParamterizedType.java | 4 +- .../apache/tajo/type/TypeProtobufEncoder.java | 188 ++++++++++++++++++ ...Serializer.java => TypeStringEncoder.java} | 109 ++-------- .../org/apache/tajo/type/TypeVisitor.java | 54 +++++ .../tajo/type/ValueParamterizedType.java | 18 +- .../java/org/apache/tajo/type/Varchar.java | 4 +- tajo-common/src/main/proto/DataTypes.proto | 20 ++ ...peSerializer.java => TestTypeEncoder.java} | 10 +- .../tajo/type/TestTypeProtobufEncoder.java | 30 +++ .../tajo/type/TestTypeStringEncoder.java | 29 +++ .../apache/tajo/engine/eval/TestEvalTree.java | 19 +- .../org/apache/tajo/plan/LogicalPlanner.java | 4 +- .../org/apache/tajo/plan/TypeDeterminant.java | 5 +- .../org/apache/tajo/plan/expr/ConstEval.java | 4 +- .../tajo/plan/expr/RowConstantEval.java | 4 +- 45 files changed, 724 insertions(+), 247 deletions(-) create mode 100644 tajo-common/src/main/java/org/apache/tajo/Assert.java create mode 100644 tajo-common/src/main/java/org/apache/tajo/type/PrimitiveType.java create mode 100644 tajo-common/src/main/java/org/apache/tajo/type/TypeFactory.java create mode 100644 tajo-common/src/main/java/org/apache/tajo/type/TypeProtobufEncoder.java rename tajo-common/src/main/java/org/apache/tajo/type/{TypeSerializer.java => TypeStringEncoder.java} (69%) create mode 100644 tajo-common/src/main/java/org/apache/tajo/type/TypeVisitor.java rename tajo-common/src/test/java/org/apache/tajo/type/{TestTypeSerializer.java => TestTypeEncoder.java} (94%) create mode 100644 tajo-common/src/test/java/org/apache/tajo/type/TestTypeProtobufEncoder.java create mode 100644 tajo-common/src/test/java/org/apache/tajo/type/TestTypeStringEncoder.java diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TypeConverter.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TypeConverter.java index 1f3d2e7234..a926d05a30 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TypeConverter.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TypeConverter.java @@ -20,9 +20,6 @@ import com.google.common.collect.ImmutableList; import org.apache.tajo.common.TajoDataTypes; -import org.apache.tajo.exception.NotImplementedException; -import org.apache.tajo.exception.TajoRuntimeException; -import org.apache.tajo.exception.UnsupportedException; import org.apache.tajo.schema.Field; import org.apache.tajo.type.*; @@ -58,81 +55,7 @@ public static Type convert(TajoDataTypes.DataType legacyType) { case PROTOBUF: return new Protobuf(legacyType.getCode()); default: - return convert(legacyType.getType()); - } - } - - /** - * This is for base types. - * - * @param legacyBaseType legacy base type - * @return Type - */ - public static Type convert(TajoDataTypes.Type legacyBaseType) { - switch (legacyBaseType) { - case BOOLEAN: - return Bool; - case INT1: - return Int1; - case INT2: - return Int2; - case INT4: - return Int4; - case INT8: - return Int8; - case FLOAT4: - return Float4; - case FLOAT8: - return Float8; - case DATE: - return Date; - case TIME: - return Time; - case TIMESTAMP: - return Timestamp; - case INTERVAL: - return Interval; - case CHAR: - return Char(1); // default len = 1 - case TEXT: - return Text; - case BLOB: - return Blob; - case INET4: - return Inet4; - case RECORD: - // for better exception - throw new TajoRuntimeException(new NotImplementedException("record projection")); - case NULL_TYPE: - return Null; - case ANY: - return Any; - - case BOOLEAN_ARRAY: - return Array(Bool); - case INT1_ARRAY: - return Array(Int1); - case INT2_ARRAY: - return Array(Int2); - case INT4_ARRAY: - return Array(Int4); - case INT8_ARRAY: - return Array(Int8); - case FLOAT4_ARRAY: - return Array(Float4); - case FLOAT8_ARRAY: - return Array(Float8); - case TIMESTAMP_ARRAY: - return Array(Timestamp); - case DATE_ARRAY: - return Array(Date); - case TIME_ARRAY: - return Array(Time); - case TEXT_ARRAY: - return Array(Text); - - default: - throw new TajoRuntimeException(new UnsupportedException(legacyBaseType.name())); + return TypeFactory.create(legacyType.getType()); } } diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java index ed4dbb22dd..972a85b6c4 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java @@ -757,7 +757,7 @@ public void createTable(final CatalogProtos.TableDescProto table) PreparedStatement pstmt = null; ResultSet res = null; - String[] splitted = IdentifierUtil.splitTableName(table.getTableName()); + final String[] splitted = IdentifierUtil.splitTableName(table.getTableName()); if (splitted.length == 1) { throw new TajoInternalError( "createTable() requires a qualified table name, but it is '" + table.getTableName() + "'"); diff --git a/tajo-common/src/main/java/org/apache/tajo/Assert.java b/tajo-common/src/main/java/org/apache/tajo/Assert.java new file mode 100644 index 0000000000..6ad74811a6 --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/Assert.java @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo; + +import org.apache.tajo.exception.TajoInternalError; + +public class Assert { + + public static void assertCondition(boolean condition, String message, Object... arguments) { + if (!condition) { + throw new TajoInternalError(String.format(message, arguments)); + } + } +} diff --git a/tajo-common/src/main/java/org/apache/tajo/json/TypeAdapter.java b/tajo-common/src/main/java/org/apache/tajo/json/TypeAdapter.java index 54462d2f54..ddd88a9742 100644 --- a/tajo-common/src/main/java/org/apache/tajo/json/TypeAdapter.java +++ b/tajo-common/src/main/java/org/apache/tajo/json/TypeAdapter.java @@ -19,9 +19,7 @@ package org.apache.tajo.json; import com.google.gson.*; -import org.apache.tajo.common.TajoDataTypes; -import org.apache.tajo.common.TajoDataTypes.DataType; -import org.apache.tajo.type.TypeSerializer; +import org.apache.tajo.type.TypeStringEncoder; import java.lang.reflect.Type; @@ -32,13 +30,13 @@ public class TypeAdapter implements GsonSerDerAdapter public org.apache.tajo.type.Type deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) throws JsonParseException { JsonObject obj = (JsonObject) json; - return TypeSerializer.deserialize(CommonGsonHelper.getOrDie(obj, "type").getAsString()); + return TypeStringEncoder.deserialize(CommonGsonHelper.getOrDie(obj, "type").getAsString()); } @Override public JsonElement serialize(org.apache.tajo.type.Type src, Type typeOfSrc, JsonSerializationContext context) { JsonObject json = new JsonObject(); - json.addProperty("type", TypeSerializer.serialize(src)); + json.addProperty("type", TypeStringEncoder.serialize(src)); return json; } } diff --git a/tajo-common/src/main/java/org/apache/tajo/schema/Field.java b/tajo-common/src/main/java/org/apache/tajo/schema/Field.java index f3d93e363e..ae0a3792cd 100644 --- a/tajo-common/src/main/java/org/apache/tajo/schema/Field.java +++ b/tajo-common/src/main/java/org/apache/tajo/schema/Field.java @@ -30,8 +30,8 @@ * Represent a field in a schema. */ public class Field implements Cloneable { - protected final Type type; protected final QualifiedIdentifier name; + protected final Type type; public Field(QualifiedIdentifier name, Type type) { this.type = type; diff --git a/tajo-common/src/main/java/org/apache/tajo/schema/Identifier.java b/tajo-common/src/main/java/org/apache/tajo/schema/Identifier.java index dba9588ece..23ea0fbbda 100644 --- a/tajo-common/src/main/java/org/apache/tajo/schema/Identifier.java +++ b/tajo-common/src/main/java/org/apache/tajo/schema/Identifier.java @@ -18,6 +18,8 @@ package org.apache.tajo.schema; +import org.apache.tajo.common.ProtoObject; +import org.apache.tajo.common.TajoDataTypes.IdentifierProto; import org.apache.tajo.schema.IdentifierPolicy.IdentifierCase; import java.util.Objects; @@ -27,7 +29,7 @@ /** * Represents an identifier part */ -public class Identifier { +public class Identifier implements ProtoObject { private String name; private boolean quoted; @@ -118,4 +120,16 @@ public boolean equals(Object obj) { return false; } + + @Override + public IdentifierProto getProto() { + return IdentifierProto.newBuilder() + .setName(name) + .setQuoted(quoted) + .build(); + } + + public static Identifier fromProto(IdentifierProto proto) { + return new Identifier(proto.getName(), proto.getQuoted()); + } } diff --git a/tajo-common/src/main/java/org/apache/tajo/schema/QualifiedIdentifier.java b/tajo-common/src/main/java/org/apache/tajo/schema/QualifiedIdentifier.java index c2759f3b05..761958ebad 100644 --- a/tajo-common/src/main/java/org/apache/tajo/schema/QualifiedIdentifier.java +++ b/tajo-common/src/main/java/org/apache/tajo/schema/QualifiedIdentifier.java @@ -20,7 +20,12 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Function; +import com.google.common.collect.Collections2; import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import org.apache.tajo.common.ProtoObject; +import org.apache.tajo.common.TajoDataTypes.IdentifierProto; +import org.apache.tajo.common.TajoDataTypes.QualifiedIdentifierProto; import org.apache.tajo.util.StringUtils; import javax.annotation.Nullable; @@ -33,11 +38,11 @@ * Represents a qualified identifier. In other words, it represents identifiers for * all kinds of of database objects, such as databases, tables, and columns. */ -public class QualifiedIdentifier { +public class QualifiedIdentifier implements ProtoObject { private ImmutableList names; - private QualifiedIdentifier(ImmutableList names) { - this.names = names; + private QualifiedIdentifier(Collection names) { + this.names = ImmutableList.copyOf(names); } /** @@ -103,6 +108,16 @@ public boolean equals(Object obj) { return new QualifiedIdentifier(ImmutableList.copyOf(names)); } + public static QualifiedIdentifier fromProto(QualifiedIdentifierProto proto) { + Collection ids = Collections2.transform(proto.getNamesList(), new Function() { + @Override + public Identifier apply(@Nullable IdentifierProto identifierProto) { + return Identifier.fromProto(identifierProto); + } + }); + return new QualifiedIdentifier(ids); + } + /** * It takes interned strings. It assumes all parameters already stripped. It is used only for tests. * @param names interned strings @@ -118,4 +133,16 @@ public boolean equals(Object obj) { } return new QualifiedIdentifier(builder.build()); } + + @Override + public QualifiedIdentifierProto getProto() { + return QualifiedIdentifierProto.newBuilder() + .addAllNames(Iterables.transform(names, new Function() { + @Override + public IdentifierProto apply(@Nullable Identifier identifier) { + return identifier.getProto(); + } + })) + .build(); + } } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Any.java b/tajo-common/src/main/java/org/apache/tajo/type/Any.java index 342fdb68f7..04dfcebc0e 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Any.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Any.java @@ -20,7 +20,7 @@ import static org.apache.tajo.common.TajoDataTypes.Type.ANY; -public class Any extends Type { +public class Any extends PrimitiveType { public Any() { super(ANY); } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Blob.java b/tajo-common/src/main/java/org/apache/tajo/type/Blob.java index 360ea13fba..5ae9a83edc 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Blob.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Blob.java @@ -20,7 +20,7 @@ import static org.apache.tajo.common.TajoDataTypes.Type.BLOB; -public class Blob extends Type { +public class Blob extends PrimitiveType { public Blob() { super(BLOB); } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Bool.java b/tajo-common/src/main/java/org/apache/tajo/type/Bool.java index 923e8e90ec..b32752592a 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Bool.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Bool.java @@ -20,7 +20,7 @@ import static org.apache.tajo.common.TajoDataTypes.Type.BOOLEAN; -public class Bool extends Type { +public class Bool extends PrimitiveType { public Bool() { super(BOOLEAN); } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Char.java b/tajo-common/src/main/java/org/apache/tajo/type/Char.java index 555df64874..586775ea0d 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Char.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Char.java @@ -24,10 +24,10 @@ public class Char extends ValueParamterizedType { public Char(int length) { - super(CHAR, ImmutableList.of((Object)length)); + super(CHAR, ImmutableList.of(length)); } public int length() { - return (int) params.get(0); + return params.get(0); } } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Date.java b/tajo-common/src/main/java/org/apache/tajo/type/Date.java index a68ab97a49..1ff38cd5de 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Date.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Date.java @@ -20,7 +20,7 @@ import static org.apache.tajo.common.TajoDataTypes.Type.DATE; -public class Date extends Type { +public class Date extends PrimitiveType { public Date() { super(DATE); } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Float4.java b/tajo-common/src/main/java/org/apache/tajo/type/Float4.java index b8c82181ee..a6dc610b79 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Float4.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Float4.java @@ -20,7 +20,7 @@ import static org.apache.tajo.common.TajoDataTypes.Type.FLOAT4; -public class Float4 extends Type { +public class Float4 extends PrimitiveType { public Float4() { super(FLOAT4); } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Float8.java b/tajo-common/src/main/java/org/apache/tajo/type/Float8.java index f36fac87c8..a804654b66 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Float8.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Float8.java @@ -20,7 +20,7 @@ import static org.apache.tajo.common.TajoDataTypes.Type.FLOAT8; -public class Float8 extends Type { +public class Float8 extends PrimitiveType { public Float8() { super(FLOAT8); } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Inet4.java b/tajo-common/src/main/java/org/apache/tajo/type/Inet4.java index 541841ec07..020509be1b 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Inet4.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Inet4.java @@ -20,7 +20,7 @@ import static org.apache.tajo.common.TajoDataTypes.Type.INET4; -public class Inet4 extends Type { +public class Inet4 extends PrimitiveType { public Inet4() { super(INET4); } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Int1.java b/tajo-common/src/main/java/org/apache/tajo/type/Int1.java index d86d36e196..2a4e3996ae 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Int1.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Int1.java @@ -20,7 +20,7 @@ import static org.apache.tajo.common.TajoDataTypes.Type.INT1; -public class Int1 extends Type { +public class Int1 extends PrimitiveType { public Int1() { super(INT1); } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Int2.java b/tajo-common/src/main/java/org/apache/tajo/type/Int2.java index e7c221541d..a8f4eec79a 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Int2.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Int2.java @@ -20,7 +20,7 @@ import static org.apache.tajo.common.TajoDataTypes.Type.INT2; -public class Int2 extends Type { +public class Int2 extends PrimitiveType { public Int2() { super(INT2); } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Int4.java b/tajo-common/src/main/java/org/apache/tajo/type/Int4.java index e8db7841a0..61c673f5b4 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Int4.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Int4.java @@ -20,7 +20,7 @@ import static org.apache.tajo.common.TajoDataTypes.Type.INT4; -public class Int4 extends Type { +public class Int4 extends PrimitiveType { public Int4() { super(INT4); } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Int8.java b/tajo-common/src/main/java/org/apache/tajo/type/Int8.java index 999e9acd97..2e257d349c 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Int8.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Int8.java @@ -20,7 +20,7 @@ import static org.apache.tajo.common.TajoDataTypes.Type.INT8; -public class Int8 extends Type { +public class Int8 extends PrimitiveType { public Int8() { super(INT8); } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Interval.java b/tajo-common/src/main/java/org/apache/tajo/type/Interval.java index d6dc17eda8..5bc0c69641 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Interval.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Interval.java @@ -20,7 +20,7 @@ import static org.apache.tajo.common.TajoDataTypes.Type.INTERVAL; -public class Interval extends Type { +public class Interval extends PrimitiveType { public Interval() { super(INTERVAL); } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Null.java b/tajo-common/src/main/java/org/apache/tajo/type/Null.java index 74cf052a0b..0896e8274b 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Null.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Null.java @@ -20,7 +20,7 @@ import static org.apache.tajo.common.TajoDataTypes.Type.NULL_TYPE; -public class Null extends Type { +public class Null extends PrimitiveType { public Null() { super(NULL_TYPE); } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Numeric.java b/tajo-common/src/main/java/org/apache/tajo/type/Numeric.java index 04497976d0..c21325dde1 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Numeric.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Numeric.java @@ -25,14 +25,14 @@ public class Numeric extends ValueParamterizedType { public Numeric(int precision, int scale) { - super(NUMERIC, ImmutableList.of((Object)precision, scale)); + super(NUMERIC, ImmutableList.of(precision, scale)); } public int precision() { - return (int) this.params.get(0); + return this.params.get(0); } public int scale() { - return (int) this.params.get(1); + return this.params.get(1); } } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/PrimitiveType.java b/tajo-common/src/main/java/org/apache/tajo/type/PrimitiveType.java new file mode 100644 index 0000000000..bda946718f --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/type/PrimitiveType.java @@ -0,0 +1,27 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.type; + +import org.apache.tajo.common.TajoDataTypes; + +public abstract class PrimitiveType extends Type { + public PrimitiveType(TajoDataTypes.Type baseType) { + super(baseType); + } +} diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Protobuf.java b/tajo-common/src/main/java/org/apache/tajo/type/Protobuf.java index e286724644..2cc08f2f9e 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Protobuf.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Protobuf.java @@ -20,7 +20,7 @@ import static org.apache.tajo.common.TajoDataTypes.Type.PROTOBUF; -public class Protobuf extends Type { +public class Protobuf extends PrimitiveType { private String msgName; public Protobuf(String msgName) { diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Record.java b/tajo-common/src/main/java/org/apache/tajo/type/Record.java index ba9298d37b..c6526dcaac 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Record.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Record.java @@ -24,6 +24,7 @@ import org.apache.tajo.util.StringUtils; import java.util.Collection; +import java.util.Iterator; import java.util.List; import java.util.Objects; @@ -32,43 +33,53 @@ /** * Represents Record type */ -public class Record extends Type { - private final ImmutableList memberTypes; +public class Record extends Type implements Iterable { + private final ImmutableList fields; - public Record(Collection memberTypes) { + public Record(Collection fields) { super(RECORD); - this.memberTypes = ImmutableList.copyOf(memberTypes); + this.fields = ImmutableList.copyOf(fields); } public int size() { - return memberTypes.size(); + return fields.size(); } public Field field(int idx) { - return memberTypes.get(idx); + return fields.get(idx); } public List fields() { - return this.memberTypes; + return this.fields; } @Override public String toString() { - return "RECORD(" + StringUtils.join(memberTypes, ", ") + ")"; + return "RECORD(" + StringUtils.join(fields, ", ") + ")"; } @Override public int hashCode() { - return Objects.hash(baseType(), Objects.hash(memberTypes)); + return Objects.hash(baseType(), Objects.hash(fields)); } @Override public boolean equals(Object object) { if (object instanceof Record) { Record other = (Record) object; - return memberTypes.equals(other.memberTypes); + return fields.equals(other.fields); } return false; } + + @Override + public TajoDataTypes.TypeProto getProto() { + return null; + } + + @Override + public Iterator iterator() { + return fields.iterator(); + } } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Text.java b/tajo-common/src/main/java/org/apache/tajo/type/Text.java index 1c3083427e..066c839e33 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Text.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Text.java @@ -18,11 +18,9 @@ package org.apache.tajo.type; -import org.apache.tajo.common.TajoDataTypes; - import static org.apache.tajo.common.TajoDataTypes.Type.TEXT; -public class Text extends Type { +public class Text extends PrimitiveType { public Text() { super(TEXT); } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Time.java b/tajo-common/src/main/java/org/apache/tajo/type/Time.java index 12fe5f82db..6d624d7086 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Time.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Time.java @@ -20,7 +20,7 @@ import static org.apache.tajo.common.TajoDataTypes.Type.TIME; -public class Time extends Type { +public class Time extends PrimitiveType { public Time() { super(TIME); } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Timestamp.java b/tajo-common/src/main/java/org/apache/tajo/type/Timestamp.java index 577f877419..17a0cb3384 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Timestamp.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Timestamp.java @@ -20,7 +20,7 @@ import static org.apache.tajo.common.TajoDataTypes.Type.TIMESTAMP; -public class Timestamp extends Type { +public class Timestamp extends PrimitiveType { public Timestamp() { super(TIMESTAMP); } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Type.java b/tajo-common/src/main/java/org/apache/tajo/type/Type.java index 58ccc71a60..33334c2493 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Type.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Type.java @@ -18,18 +18,21 @@ package org.apache.tajo.type; +import org.apache.tajo.common.ProtoObject; import org.apache.tajo.common.TajoDataTypes; +import org.apache.tajo.common.TajoDataTypes.TypeProto; import org.apache.tajo.exception.TajoRuntimeException; import org.apache.tajo.exception.UnsupportedException; import org.apache.tajo.schema.Field; import java.util.Arrays; import java.util.Collection; +import java.util.List; /** * Represents Type */ -public abstract class Type implements Cloneable { +public abstract class Type implements Cloneable, ProtoObject { // No paramter types public static final Any Any = new Any(); @@ -67,11 +70,11 @@ public boolean isValueParameterized() { return false; } - public Collection getTypeParameters() { + public List getTypeParameters() { throw new TajoRuntimeException(new UnsupportedException()); } - public Collection getValueParameters() { + public List getValueParameters() { throw new TajoRuntimeException(new UnsupportedException()); } @@ -142,4 +145,9 @@ public static Map Map(Type keyType, Type valueType) { public static Null Null() { return new Null(); } + + @Override + public TypeProto getProto() { + return TypeProtobufEncoder.serialize(this); + } } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/TypeFactory.java b/tajo-common/src/main/java/org/apache/tajo/type/TypeFactory.java new file mode 100644 index 0000000000..b9c04bef6c --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/type/TypeFactory.java @@ -0,0 +1,181 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.type; + +import org.apache.tajo.Assert; +import org.apache.tajo.common.TajoDataTypes; +import org.apache.tajo.exception.NotImplementedException; +import org.apache.tajo.exception.TajoInternalError; +import org.apache.tajo.exception.TajoRuntimeException; +import org.apache.tajo.exception.UnsupportedException; +import org.apache.tajo.schema.Field; +import org.apache.tajo.util.StringUtils; + +import java.util.List; + +import static org.apache.tajo.type.Type.Char; +import static org.apache.tajo.type.Type.Varchar; + +public class TypeFactory { + /** + * This is for base types. + * + * @param baseType legacy base type + * @return Type + */ + public static Type create(TajoDataTypes.Type baseType) { + switch (baseType) { + case BOOLEAN: + return Type.Bool; + case INT1: + return Type.Int1; + case INT2: + return Type.Int2; + case INT4: + return Type.Int4; + case INT8: + return Type.Int8; + case FLOAT4: + return Type.Float4; + case FLOAT8: + return Type.Float8; + case DATE: + return Type.Date; + case TIME: + return Type.Time; + case TIMESTAMP: + return Type.Timestamp; + case INTERVAL: + return Type.Interval; + case CHAR: + return Type.Char(1); // default len = 1 + case TEXT: + return Type.Text; + case BLOB: + return Type.Blob; + case INET4: + return Type.Inet4; + case RECORD: + // for better exception + throw new TajoRuntimeException(new NotImplementedException("record projection")); + case NULL_TYPE: + return Type.Null; + case ANY: + return Type.Any; + + case BOOLEAN_ARRAY: + return Type.Array(Type.Bool); + case INT1_ARRAY: + return Type.Array(Type.Int1); + case INT2_ARRAY: + return Type.Array(Type.Int2); + case INT4_ARRAY: + return Type.Array(Type.Int4); + case INT8_ARRAY: + return Type.Array(Type.Int8); + case FLOAT4_ARRAY: + return Type.Array(Type.Float4); + case FLOAT8_ARRAY: + return Type.Array(Type.Float8); + case TIMESTAMP_ARRAY: + return Type.Array(Type.Timestamp); + case DATE_ARRAY: + return Type.Array(Type.Date); + case TIME_ARRAY: + return Type.Array(Type.Time); + case TEXT_ARRAY: + return Type.Array(Type.Text); + + default: + throw new TajoRuntimeException(new UnsupportedException(baseType.name())); + } + } + + public static Type create(TajoDataTypes.Type baseType, + List typeParams, + List valueParams, + List fieldParams) { + switch (baseType) { + case CHAR: { + Assert.assertCondition(valueParams.size() == 1, + "Char type requires 1 integer parameters, but it takes (%s).", StringUtils.join(typeParams)); + return Char(valueParams.get(0)); + } + case VARCHAR: { + Assert.assertCondition(valueParams.size() == 1, + "Varchar type requires 1 integer parameters, but it takes (%s).", StringUtils.join(typeParams)); + return Varchar(valueParams.get(0)); + } + case TEXT: return Type.Text; + + case BOOLEAN: return Type.Bool; + case INT1: return Type.Int1; + case INT2: return Type.Int2; + case INT4: return Type.Int4; + case INT8: return Type.Int8; + case FLOAT4: return Type.Float4; + case FLOAT8: return Type.Float8; + case NUMERIC: { + if (valueParams.size() == 0) { + return Numeric.Numeric(); + + } else { + for (Object p : valueParams) { + Assert.assertCondition(p instanceof Integer, "Numeric type requires integer parameters"); + } + if (valueParams.size() == 1) { + return Numeric.Numeric(valueParams.get(0)); + } else if (valueParams.size() == 2) { + return Numeric.Numeric(valueParams.get(0), valueParams.get(1)); + } else { + Assert.assertCondition(false, + "Numeric type can take 2 or less integer parameters, but it takes (%s).", StringUtils.join(valueParams)); + } + } + } + + case DATE: return Type.Date; + case TIME: return Type.Time; + case TIMESTAMP: return Type.Timestamp; + case INTERVAL: return Type.Interval; + case BLOB: return Type.Blob; + case INET4: return Type.Inet4; + + case ARRAY: { + Assert.assertCondition(typeParams.size() == 1, + "Array Type requires 1 type parameters, but it takes (%s).", StringUtils.join(typeParams)); + return Type.Array(typeParams.get(0)); + } + case MAP: { + Assert.assertCondition(typeParams.size() == 2, + "Map Type requires 2 type parameters, but it takes (%s).", StringUtils.join(typeParams)); + + return Type.Map(typeParams.get(0), typeParams.get(1)); + } + case RECORD: { + Assert.assertCondition(fieldParams.size() >= 1, + "Record Type requires at least 1 field parameters, but it takes (%s).", StringUtils.join(fieldParams)); + return Type.Record(fieldParams); + } + + default: + throw new TajoInternalError(new UnsupportedException(baseType.name())); + } + } +} diff --git a/tajo-common/src/main/java/org/apache/tajo/type/TypeParamterizedType.java b/tajo-common/src/main/java/org/apache/tajo/type/TypeParamterizedType.java index 9f84f41eb0..1abd4616fb 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/TypeParamterizedType.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/TypeParamterizedType.java @@ -22,7 +22,7 @@ import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.util.StringUtils; -import java.util.Collection; +import java.util.List; import java.util.Objects; /** @@ -42,7 +42,7 @@ public boolean isTypeParameterized() { } @Override - public Collection getTypeParameters() { + public List getTypeParameters() { return params; } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/TypeProtobufEncoder.java b/tajo-common/src/main/java/org/apache/tajo/type/TypeProtobufEncoder.java new file mode 100644 index 0000000000..5711a26f50 --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/type/TypeProtobufEncoder.java @@ -0,0 +1,188 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.type; + +import com.google.common.base.Function; +import com.google.common.collect.Collections2; +import com.google.common.collect.ImmutableList; +import org.apache.tajo.common.TajoDataTypes; +import org.apache.tajo.common.TajoDataTypes.QualifiedIdentifierProto; +import org.apache.tajo.common.TajoDataTypes.TypeElement; +import org.apache.tajo.common.TajoDataTypes.TypeProto; +import org.apache.tajo.exception.TajoInternalError; +import org.apache.tajo.schema.Field; +import org.apache.tajo.schema.QualifiedIdentifier; + +import javax.annotation.Nullable; +import java.util.*; + +import static java.util.Collections.EMPTY_LIST; +import static org.apache.tajo.Assert.assertCondition; +import static org.apache.tajo.common.TajoDataTypes.Type.*; +import static org.apache.tajo.type.Type.Array; +import static org.apache.tajo.type.Type.Map; +import static org.apache.tajo.type.Type.Record; + +public class TypeProtobufEncoder { + public static TypeProto serialize(Type type) { + + final TypeProto.Builder builder = TypeProto.newBuilder(); + new Visitor(builder).visit(type); + return builder.build(); + } + + public static Type deserialize(TypeProto proto) { + Stack stack = new Stack<>(); + + for (int curIdx = 0; curIdx < proto.getElementsCount(); curIdx++) { + TypeElement e = proto.getElements(curIdx); + + if (e.hasChildNum()) { // if it is a type-parameterized, that is + List childTypes = popMultiItems(stack, e.getChildNum()); + + if (e.getKind() == ARRAY || e.getKind() == MAP) { + stack.push(createTypeParameterizedType(e, childTypes)); + + } else { // record + assertCondition(e.getKind() == RECORD, + "This type must be RECORD type."); + assertCondition(childTypes.size() == e.getFieldNamesCount(), + "The number of Field types and names must be equal."); + + ImmutableList.Builder fields = ImmutableList.builder(); + for (int i = 0; i < childTypes.size(); i++) { + fields.add(new Field(QualifiedIdentifier.fromProto(e.getFieldNames(i)), childTypes.get(i))); + } + stack.push(Record(fields.build())); + } + + } else { + stack.push(createPrimitiveType(e)); + } + } + + assertCondition(stack.size() == 1, "Stack size has two or more items."); + return stack.pop(); + } + + static List popMultiItems(Stack stack, int num) { + List typeParams = new ArrayList<>(); + for (int i = 0; i < num; i++) { + typeParams.add(0, stack.pop()); + } + return typeParams; + } + + static boolean isValueParameterized(TajoDataTypes.Type baseType) { + switch (baseType) { + case CHAR: + case VARCHAR: + case NUMERIC: + return true; + default: + return false; + } + } + + static Type createPrimitiveType(TypeElement element) { + assertPrimitiveType(element); + + if (isValueParameterized(element.getKind())) { + return TypeFactory.create(element.getKind(), EMPTY_LIST, element.getValueParamsList(), EMPTY_LIST); + } else { + return TypeFactory.create(element.getKind()); + } + } + + static Type createTypeParameterizedType(TypeElement element, List childTypes) { + switch (element.getKind()) { + case ARRAY: + return Array(childTypes.get(0)); + case MAP: + return Map(childTypes.get(0), childTypes.get(1)); + default: + throw new TajoInternalError(element.getKind().name() + " is not a type-parameterized type."); + } + } + + static void assertPrimitiveType(TypeElement element) { + TajoDataTypes.Type baseType = element.getKind(); + if (baseType == MAP || baseType == RECORD || baseType == ARRAY) { + throw new TajoInternalError(baseType.name() + " is not a primitive type."); + } + } + + static class Visitor extends TypeVisitor { + final TypeProto.Builder builder; + + Visitor(TypeProto.Builder builder) { + this.builder = builder; + } + + @Override + public void visitPrimitive(Type type) { + TypeElement.Builder typeElemBuilder = TypeElement.newBuilder() + .setKind(type.baseType); + + if (type.isValueParameterized()) { + typeElemBuilder.addAllValueParams(type.getValueParameters()); + } + + builder.addElements(typeElemBuilder); + } + + @Override + public void visitMap(Map map) { + super.visitMap(map); + builder.addElements(TypeElement.newBuilder() + .setKind(map.baseType) + .setChildNum(2) + ); + } + + @Override + public void visitArray(Array array) { + super.visitArray(array); + builder + .addElements(TypeElement.newBuilder() + .setKind(array.baseType) + .setChildNum(1) + ); + } + + @Override + public void visitRecord(Record record) { + super.visitRecord(record); + + Collection field_names = + Collections2.transform(record.fields(), new Function() { + @Override + public QualifiedIdentifierProto apply(@Nullable Field field) { + return field.name().getProto(); + } + }); + + builder + .addElements(TypeElement.newBuilder() + .setChildNum(record.size()) + .addAllFieldNames(field_names) + .setKind(RECORD)); + } + } +} diff --git a/tajo-common/src/main/java/org/apache/tajo/type/TypeSerializer.java b/tajo-common/src/main/java/org/apache/tajo/type/TypeStringEncoder.java similarity index 69% rename from tajo-common/src/main/java/org/apache/tajo/type/TypeSerializer.java rename to tajo-common/src/main/java/org/apache/tajo/type/TypeStringEncoder.java index 5806bdbac6..acda5a7dfb 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/TypeSerializer.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/TypeStringEncoder.java @@ -20,6 +20,7 @@ import com.google.common.base.Function; import com.google.common.collect.ImmutableList; +import org.apache.tajo.Assert; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.exception.TajoInternalError; import org.apache.tajo.exception.UnsupportedException; @@ -33,13 +34,11 @@ import java.util.Stack; import static org.apache.tajo.schema.IdentifierPolicy.DefaultPolicy; -import static org.apache.tajo.type.Type.Char; -import static org.apache.tajo.type.Type.Varchar; /** * This class enables to serialize a type into a string representation and vice versa. */ -public class TypeSerializer { +public class TypeStringEncoder { /** * Serialize a type into a string representation @@ -54,7 +53,7 @@ public static String serialize(Type type) { sb.append(StringUtils.join(type.getTypeParameters(), ",", new Function() { @Override public String apply(@Nullable Type type) { - return TypeSerializer.serialize(type); + return TypeStringEncoder.serialize(type); } })); sb.append(">"); @@ -114,14 +113,14 @@ public static Type deserialize(String signature) { if (c == '<') { if (stack.isEmpty()) { - assertArgument(baseType == null, "Expected baseName to be null"); + Assert.assertCondition(baseType == null, "Expected baseName to be null"); baseType = signature.substring(0, i); } stack.push('<'); spanStack.push(i + 1); } else if (c == '>') { - assertArgument(stack.pop() == '<', "Bad signature: '%s'", signature); + Assert.assertCondition(stack.pop() == '<', "Bad signature: '%s'", signature); int paramStartIdx = spanStack.pop(); if (stack.isEmpty()) { // ensure outermost parameters @@ -138,7 +137,7 @@ public Type apply(@Nullable String s) { } else if (c == '[') { if (stack.isEmpty()) { - assertArgument(baseType == null, "Expected baseName to be null"); + Assert.assertCondition(baseType == null, "Expected baseName to be null"); baseType = signature.substring(0, i); } @@ -146,7 +145,7 @@ public Type apply(@Nullable String s) { spanStack.push(i + 1); } else if (c == ']') { - assertArgument(stack.pop() == '[', "Bad signature: '%s'", signature); + Assert.assertCondition(stack.pop() == '[', "Bad signature: '%s'", signature); int paramStartIdx = spanStack.pop(); if (stack.isEmpty()) { // ensure outermost parameters @@ -163,14 +162,14 @@ public Field apply(@Nullable String s) { } else if (c == '(') { if (stack.isEmpty()) { - assertArgument(baseType == null, "Expected baseName to be null"); + Assert.assertCondition(baseType == null, "Expected baseName to be null"); baseType = signature.substring(0, i); } stack.push('('); spanStack.push(i + 1); } else if (c == ')') { - assertArgument(stack.pop() == '(', "Bad signature: '%s'", signature); + Assert.assertCondition(stack.pop() == '(', "Bad signature: '%s'", signature); int paramStartIdx = spanStack.pop(); if (stack.isEmpty()) { // ensure outermost parameters @@ -219,11 +218,11 @@ static List parseList(String str, Function itemParser) { if (c == '<' || c == '[' || c == '(') { stack.push(c); } else if (c == '>') { - assertArgument(stack.pop() == '<', "Bad signature: '%s'", str); + Assert.assertCondition(stack.pop() == '<', "Bad signature: '%s'", str); } else if (c == ']') { - assertArgument(stack.pop() == '[', "Bad signature: '%s'", str); + Assert.assertCondition(stack.pop() == '[', "Bad signature: '%s'", str); } else if (c == ')') { - assertArgument(stack.pop() == '(', "Bad signature: '%s'", str); + Assert.assertCondition(stack.pop() == '(', "Bad signature: '%s'", str); } else if (c == ',') { if (stack.isEmpty()) { // ensure outermost type parameters fields.add(itemParser.apply(str.substring(paramStartIdx, i))); @@ -232,7 +231,7 @@ static List parseList(String str, Function itemParser) { } } - assertArgument(stack.empty(), "Bad signature: '%s'", str); + Assert.assertCondition(stack.empty(), "Bad signature: '%s'", str); if (paramStartIdx < str.length()) { fields.add(itemParser.apply(str.substring(paramStartIdx, str.length()))); } @@ -248,7 +247,7 @@ static List parseList(String str, Function itemParser) { static Field parseField(String str) { // A field consists of an identifier and a type, and they are delimited by space. if (!str.contains(" ")) { - assertArgument(false, "Bad field signature: '%s'", str); + Assert.assertCondition(false, "Bad field signature: '%s'", str); } // Stack to track the nested bracket depth @@ -260,11 +259,11 @@ static Field parseField(String str) { if (c == '<' || c == '[' || c == '(') { stack.push(c); } else if (c == '>') { // for validation - assertArgument(stack.pop() == '<', "Bad field signature: '%s'", str); + Assert.assertCondition(stack.pop() == '<', "Bad field signature: '%s'", str); } else if (c == ']') { // for validation - assertArgument(stack.pop() == '[', "Bad field signature: '%s'", str); + Assert.assertCondition(stack.pop() == '[', "Bad field signature: '%s'", str); } else if (c == ')') { // for validation - assertArgument(stack.pop() == '(', "Bad field signature: '%s'", str); + Assert.assertCondition(stack.pop() == '(', "Bad field signature: '%s'", str); } else if (c == ' ') { if (stack.isEmpty()) { // ensure outermost type parameters @@ -279,7 +278,7 @@ static Field parseField(String str) { return null; } - static Type createType(String baseTypeStr, + public static Type createType(String baseTypeStr, List typeParams, List valueParams, List fieldParams) { @@ -291,77 +290,7 @@ static Type createType(String baseTypeStr, throw new TajoInternalError(new UnsupportedException(baseTypeStr)); } - switch (baseType) { - case CHAR: { - assertArgument(valueParams.size() == 1, - "Char type requires 1 integer parameters, but it takes (%s).", StringUtils.join(typeParams)); - return Char(valueParams.get(0)); - } - case VARCHAR: { - assertArgument(valueParams.size() == 1, - "Varchar type requires 1 integer parameters, but it takes (%s).", StringUtils.join(typeParams)); - return Varchar(valueParams.get(0)); - } - case TEXT: return Type.Text; - - case BOOLEAN: return Type.Bool; - case INT1: return Type.Int1; - case INT2: return Type.Int2; - case INT4: return Type.Int4; - case INT8: return Type.Int8; - case FLOAT4: return Type.Float4; - case FLOAT8: return Type.Float8; - case NUMERIC: { - if (valueParams.size() == 0) { - return Numeric.Numeric(); - - } else { - for (Object p : valueParams) { - assertArgument(p instanceof Integer, "Numeric type requires integer parameters"); - } - if (valueParams.size() == 1) { - return Numeric.Numeric(valueParams.get(0)); - } else if (valueParams.size() == 2) { - return Numeric.Numeric(valueParams.get(0), valueParams.get(1)); - } else { - assertArgument(false, - "Numeric type can take 2 or less integer parameters, but it takes (%s).", StringUtils.join(valueParams)); - } - } - } - - case DATE: return Type.Date; - case TIME: return Type.Time; - case TIMESTAMP: return Type.Timestamp; - case INTERVAL: return Type.Interval; - case BLOB: return Type.Blob; - case INET4: return Type.Inet4; - - case ARRAY: { - assertArgument(typeParams.size() == 1, - "Array Type requires 1 type parameters, but it takes (%s).", StringUtils.join(typeParams)); - return Type.Array(typeParams.get(0)); - } - case MAP: { - assertArgument(typeParams.size() == 2, - "Map Type requires 2 type parameters, but it takes (%s).", StringUtils.join(typeParams)); - - return Type.Map(typeParams.get(0), typeParams.get(1)); - } - case RECORD: { - assertArgument(fieldParams.size() >= 1, - "Record Type requires at least 1 field parameters, but it takes (%s).", StringUtils.join(fieldParams)); - return Type.Record(fieldParams); - } - - default: - throw new TajoInternalError(new UnsupportedException(baseTypeStr)); - } + return TypeFactory.create(baseType, typeParams, valueParams, fieldParams); } - static void assertArgument(boolean condition, String message, Object ... arguments) { - if (!condition) { - throw new TajoInternalError(String.format(message, arguments)); - } - } } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/TypeVisitor.java b/tajo-common/src/main/java/org/apache/tajo/type/TypeVisitor.java new file mode 100644 index 0000000000..ee067d1c48 --- /dev/null +++ b/tajo-common/src/main/java/org/apache/tajo/type/TypeVisitor.java @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.type; + +import org.apache.tajo.schema.Field; + +public abstract class TypeVisitor { + public void visit(Type type) { + switch (type.baseType) { + case ARRAY: + visitArray((Array) type); + break; + case RECORD: + visitRecord((Record) type); + break; + case MAP: + visitMap((Map) type); + break; + default: + visitPrimitive(type); + } + } + + void visitPrimitive(Type type) {} + + void visitMap(Map map) { + visit(map.keyType()); + visit(map.valueType()); + } + void visitArray(Array array) { + visit(array.elementType()); + } + void visitRecord(Record record) { + for (Field field : record.fields()) { + visit(field.type()); + } + } +} diff --git a/tajo-common/src/main/java/org/apache/tajo/type/ValueParamterizedType.java b/tajo-common/src/main/java/org/apache/tajo/type/ValueParamterizedType.java index c53db46c90..f5300b6c51 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/ValueParamterizedType.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/ValueParamterizedType.java @@ -19,19 +19,22 @@ package org.apache.tajo.type; import com.google.common.collect.ImmutableList; +import org.apache.tajo.common.ProtoObject; import org.apache.tajo.common.TajoDataTypes; +import org.apache.tajo.common.TajoDataTypes.TypeElement; +import org.apache.tajo.common.TajoDataTypes.TypeProto; import org.apache.tajo.util.StringUtils; -import java.util.Collection; +import java.util.List; import java.util.Objects; /** * Represents a type which takes value parameters (e.g., Numeric(10, 6)) */ -public abstract class ValueParamterizedType extends Type { - protected ImmutableList params; +public abstract class ValueParamterizedType extends Type implements ProtoObject { + protected ImmutableList params; - public ValueParamterizedType(TajoDataTypes.Type type, ImmutableList params) { + public ValueParamterizedType(TajoDataTypes.Type type, ImmutableList params) { super(type); this.params = params; } @@ -42,7 +45,7 @@ public boolean isValueParameterized() { } @Override - public Collection getValueParameters() { + public List getValueParameters() { return params; } @@ -74,4 +77,9 @@ public String toString() { sb.append(")"); return sb.toString(); } + + @Override + public TypeProto getProto() { + return TypeProto.newBuilder().addElements(TypeElement.newBuilder().addAllValueParams(params)).build(); + } } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Varchar.java b/tajo-common/src/main/java/org/apache/tajo/type/Varchar.java index e953e0236d..61e4e8563f 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Varchar.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Varchar.java @@ -25,10 +25,10 @@ public class Varchar extends ValueParamterizedType { public Varchar(int length) { - super(VARCHAR, ImmutableList.of((Object)length)); + super(VARCHAR, ImmutableList.of(length)); } public int length() { - return (int) params.get(0); + return params.get(0); } } diff --git a/tajo-common/src/main/proto/DataTypes.proto b/tajo-common/src/main/proto/DataTypes.proto index 7cf09d387e..11f1582c49 100644 --- a/tajo-common/src/main/proto/DataTypes.proto +++ b/tajo-common/src/main/proto/DataTypes.proto @@ -116,3 +116,23 @@ message DataType { optional int32 num_nested_fields = 4; } +message TypeProto { + repeated TypeElement elements = 1; +} + +message TypeElement { + required Type kind = 1; + repeated int32 value_params = 2; + repeated QualifiedIdentifierProto field_names = 3; + optional int32 child_num = 4; +} + +message QualifiedIdentifierProto { + repeated IdentifierProto names = 1; +} + +message IdentifierProto { + required string name = 1; + required bool quoted = 2; +} + diff --git a/tajo-common/src/test/java/org/apache/tajo/type/TestTypeSerializer.java b/tajo-common/src/test/java/org/apache/tajo/type/TestTypeEncoder.java similarity index 94% rename from tajo-common/src/test/java/org/apache/tajo/type/TestTypeSerializer.java rename to tajo-common/src/test/java/org/apache/tajo/type/TestTypeEncoder.java index fb32df81b5..4a0eea98ea 100644 --- a/tajo-common/src/test/java/org/apache/tajo/type/TestTypeSerializer.java +++ b/tajo-common/src/test/java/org/apache/tajo/type/TestTypeEncoder.java @@ -44,10 +44,10 @@ import static org.apache.tajo.type.Type.Record; import static org.apache.tajo.type.Type.Text; import static org.apache.tajo.type.Type.Varchar; -import static org.apache.tajo.type.TypeSerializer.*; +import static org.apache.tajo.type.TypeStringEncoder.*; import static org.junit.Assert.assertEquals; -public class TestTypeSerializer { +public abstract class TestTypeEncoder { public static final Record RECORD_POINT = Record(Field($("x"), Float8), Field($("y"), Float8)); public static final Record RECORD_PERSON = @@ -73,7 +73,7 @@ public final void testTypesWithoutParams() { @Test public final void testValueParams() { - assertSerialize(Numeric()); + //assertSerialize(Numeric()); assertSerialize(Numeric(10)); assertSerialize(Numeric(10, 12)); @@ -120,7 +120,5 @@ public Type apply(@Nullable String s) { assertEquals(Array(Float8), types.get(1)); } - public final void assertSerialize(Type type) { - assertEquals(type, deserialize(serialize(type))); - } + abstract void assertSerialize(Type type); } \ No newline at end of file diff --git a/tajo-common/src/test/java/org/apache/tajo/type/TestTypeProtobufEncoder.java b/tajo-common/src/test/java/org/apache/tajo/type/TestTypeProtobufEncoder.java new file mode 100644 index 0000000000..6eda557c3d --- /dev/null +++ b/tajo-common/src/test/java/org/apache/tajo/type/TestTypeProtobufEncoder.java @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.type; + +import static org.apache.tajo.type.TypeProtobufEncoder.deserialize; +import static org.apache.tajo.type.TypeProtobufEncoder.serialize; +import static org.junit.Assert.assertEquals; + +public class TestTypeProtobufEncoder extends TestTypeEncoder { + + public void assertSerialize(Type type) { + assertEquals(type, deserialize(serialize(type))); + } +} \ No newline at end of file diff --git a/tajo-common/src/test/java/org/apache/tajo/type/TestTypeStringEncoder.java b/tajo-common/src/test/java/org/apache/tajo/type/TestTypeStringEncoder.java new file mode 100644 index 0000000000..a7a57894bd --- /dev/null +++ b/tajo-common/src/test/java/org/apache/tajo/type/TestTypeStringEncoder.java @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.type; + +import static org.apache.tajo.type.TypeStringEncoder.deserialize; +import static org.apache.tajo.type.TypeStringEncoder.serialize; +import static org.junit.Assert.assertEquals; + +public class TestTypeStringEncoder extends TestTypeEncoder { + public void assertSerialize(Type type) { + assertEquals(type, deserialize(serialize(type))); + } +} \ No newline at end of file diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/TestEvalTree.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/TestEvalTree.java index 22e8e56127..b0fb8005ed 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/TestEvalTree.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/eval/TestEvalTree.java @@ -26,6 +26,7 @@ import org.apache.tajo.storage.Tuple; import org.apache.tajo.storage.VTuple; import org.apache.tajo.type.Type; +import org.apache.tajo.type.TypeFactory; import org.junit.Test; import static org.apache.tajo.common.TajoDataTypes.Type.INT4; @@ -387,26 +388,26 @@ private void assertCloneEqual(EvalNode eval) throws CloneNotSupportedException { @Test public final void testHashAndEqual() { EvalNode e1 = new BinaryEval(EvalType.EQUAL, - new FieldEval("default.n1.n_nationkey", TypeConverter.convert(INT4)), - new FieldEval("default.n2.n_nationkey", TypeConverter.convert(INT4)) + new FieldEval("default.n1.n_nationkey", TypeFactory.create(INT4)), + new FieldEval("default.n2.n_nationkey", TypeFactory.create(INT4)) ); EvalNode e2 = new BinaryEval(EvalType.EQUAL, - new FieldEval("default.n1.n_name", TypeConverter.convert(INT4)), - new FieldEval("default.n2.n_name", TypeConverter.convert(INT4)) + new FieldEval("default.n1.n_name", TypeFactory.create(INT4)), + new FieldEval("default.n2.n_name", TypeFactory.create(INT4)) ); EvalNode e3 = new BinaryEval(EvalType.EQUAL, - new FieldEval("default.n1.n_regionkey", TypeConverter.convert(INT4)), - new FieldEval("default.n2.n_regionkey", TypeConverter.convert(INT4)) + new FieldEval("default.n1.n_regionkey", TypeFactory.create(INT4)), + new FieldEval("default.n2.n_regionkey", TypeFactory.create(INT4)) ); EvalNode e4 = new BinaryEval(EvalType.EQUAL, - new FieldEval("default.n1.n_comment", TypeConverter.convert(TEXT)), - new FieldEval("default.n2.n_comment", TypeConverter.convert(TEXT)) + new FieldEval("default.n1.n_comment", TypeFactory.create(TEXT)), + new FieldEval("default.n2.n_comment", TypeFactory.create(TEXT)) ); - FieldEval f1 = new FieldEval("default.n1.n_nationkey", TypeConverter.convert(INT4)); + FieldEval f1 = new FieldEval("default.n1.n_nationkey", TypeFactory.create(INT4)); System.out.println(f1.hashCode()); System.out.println(e1.hashCode()); diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java index 4996bf5fef..fcee1a39ba 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java @@ -51,6 +51,7 @@ import org.apache.tajo.schema.Field; import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.type.Type; +import org.apache.tajo.type.TypeFactory; import org.apache.tajo.util.KeyValueSet; import org.apache.tajo.util.Pair; import org.apache.tajo.util.StringUtils; @@ -60,7 +61,6 @@ import java.util.*; import static org.apache.tajo.algebra.CreateTable.PartitionType; -import static org.apache.tajo.catalog.TypeConverter.convert; import static org.apache.tajo.plan.ExprNormalizer.ExprNormalizedResult; import static org.apache.tajo.plan.LogicalPlan.BlockType; import static org.apache.tajo.plan.verifier.SyntaxErrorUtil.makeSyntaxError; @@ -2133,7 +2133,7 @@ public static Type convertDataType(DataTypeExpr dataType) { return Numeric(); } } else { - return convert(type); + return TypeFactory.create(type); } } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/TypeDeterminant.java b/tajo-plan/src/main/java/org/apache/tajo/plan/TypeDeterminant.java index deb88ea003..f67b50f839 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/TypeDeterminant.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/TypeDeterminant.java @@ -21,7 +21,10 @@ import com.google.common.base.Preconditions; import org.apache.tajo.DataTypeUtil; import org.apache.tajo.algebra.*; -import org.apache.tajo.catalog.*; +import org.apache.tajo.catalog.CatalogService; +import org.apache.tajo.catalog.CatalogUtil; +import org.apache.tajo.catalog.Column; +import org.apache.tajo.catalog.FunctionDesc; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.exception.TajoException; diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/ConstEval.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/ConstEval.java index 25c12d4fd6..4f1ece6791 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/ConstEval.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/ConstEval.java @@ -20,10 +20,10 @@ import com.google.common.base.Objects; import com.google.gson.annotations.Expose; -import org.apache.tajo.catalog.TypeConverter; import org.apache.tajo.datum.Datum; import org.apache.tajo.storage.Tuple; import org.apache.tajo.type.Type; +import org.apache.tajo.type.TypeFactory; public class ConstEval extends EvalNode implements Comparable, Cloneable { @Expose Datum datum = null; @@ -51,7 +51,7 @@ public Datum eval(Tuple tuple) { @Override public Type getValueType() { - return TypeConverter.convert(datum.type()); + return TypeFactory.create(datum.type()); } @Override diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/RowConstantEval.java b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/RowConstantEval.java index aaaba22ee3..21d3517b9f 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/expr/RowConstantEval.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/expr/RowConstantEval.java @@ -19,11 +19,11 @@ package org.apache.tajo.plan.expr; import com.google.gson.annotations.Expose; -import org.apache.tajo.catalog.TypeConverter; import org.apache.tajo.datum.Datum; import org.apache.tajo.datum.NullDatum; import org.apache.tajo.storage.Tuple; import org.apache.tajo.type.Type; +import org.apache.tajo.type.TypeFactory; import org.apache.tajo.util.StringUtils; import org.apache.tajo.util.TUtil; @@ -39,7 +39,7 @@ public RowConstantEval(Datum [] values) { @Override public Type getValueType() { - return TypeConverter.convert(values[0].type()); + return TypeFactory.create(values[0].type()); } @Override From 8f4914f2fad9196183aba3d10024c5238d1f9745 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Tue, 10 May 2016 10:40:24 -0700 Subject: [PATCH 18/23] Applied new Type to catalog. --- .../java/org/apache/tajo/DataTypeUtil.java | 4 +- .../java/org/apache/tajo/catalog/Column.java | 12 ++- .../org/apache/tajo/catalog/SchemaLegacy.java | 79 ++++--------------- .../apache/tajo/catalog/TypeConverter.java | 8 +- .../tajo/catalog/statistics/TableStats.java | 9 ++- .../src/main/proto/CatalogProtos.proto | 2 +- .../tajo/catalog/store/HiveCatalogStore.java | 7 +- .../tajo/catalog/store/HiveCatalogUtil.java | 10 ++- .../dictionary/AbstractTableDescriptor.java | 9 +-- .../tajo/catalog/store/AbstractDBStore.java | 78 ++++++------------ .../main/resources/schemas/derby/derby.xml | 4 +- .../resources/schemas/mariadb/mariadb.xml | 4 +- .../main/resources/schemas/mysql/mysql.xml | 4 +- .../main/resources/schemas/oracle/oracle.xml | 4 +- .../schemas/postgresql/postgresql.xml | 4 +- .../TestCatalogAgainstCaseSensitivity.java | 2 +- .../InvalidValueForCastException.java | 2 +- .../java/org/apache/tajo/schema/Field.java | 2 +- .../main/java/org/apache/tajo/type/Any.java | 2 +- .../main/java/org/apache/tajo/type/Blob.java | 2 +- .../main/java/org/apache/tajo/type/Bool.java | 2 +- .../main/java/org/apache/tajo/type/Date.java | 2 +- .../java/org/apache/tajo/type/Float4.java | 2 +- .../java/org/apache/tajo/type/Float8.java | 2 +- .../main/java/org/apache/tajo/type/Inet4.java | 2 +- .../main/java/org/apache/tajo/type/Int1.java | 2 +- .../main/java/org/apache/tajo/type/Int2.java | 2 +- .../main/java/org/apache/tajo/type/Int4.java | 2 +- .../main/java/org/apache/tajo/type/Int8.java | 2 +- .../java/org/apache/tajo/type/Interval.java | 2 +- .../main/java/org/apache/tajo/type/Null.java | 2 +- .../org/apache/tajo/type/PrimitiveType.java | 27 ------- .../java/org/apache/tajo/type/Protobuf.java | 2 +- .../java/org/apache/tajo/type/Record.java | 8 +- .../main/java/org/apache/tajo/type/Text.java | 2 +- .../main/java/org/apache/tajo/type/Time.java | 2 +- .../java/org/apache/tajo/type/Timestamp.java | 2 +- .../main/java/org/apache/tajo/type/Type.java | 22 +++--- .../tajo/type/TypeParamterizedType.java | 6 +- .../apache/tajo/type/TypeProtobufEncoder.java | 23 +++--- .../apache/tajo/type/TypeStringEncoder.java | 2 +- .../org/apache/tajo/type/TypeVisitor.java | 2 +- .../tajo/type/ValueParamterizedType.java | 6 +- tajo-common/src/main/proto/DataTypes.proto | 5 +- .../java/org/apache/tajo/type/TestType.java | 40 +++++----- .../tajo/type/TestTypeProtobufEncoder.java | 6 +- .../engine/codegen/EvalCodeGenContext.java | 2 +- .../engine/codegen/EvalCodeGenerator.java | 8 +- .../engine/codegen/TajoGeneratorAdapter.java | 28 +++---- .../engine/codegen/VariablesPreBuilder.java | 4 +- .../NonForwardQueryResultSystemScanner.java | 13 +-- .../org/apache/tajo/plan/ExprAnnotator.java | 8 +- .../plan/util/EvalNodeToExprConverter.java | 2 +- .../tajo/plan/verifier/ExprsVerifier.java | 14 ++-- .../storage/jdbc/SQLExpressionGenerator.java | 4 +- 55 files changed, 201 insertions(+), 307 deletions(-) delete mode 100644 tajo-common/src/main/java/org/apache/tajo/type/PrimitiveType.java diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/DataTypeUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/DataTypeUtil.java index 9cbdbe0391..69206aa996 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/DataTypeUtil.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/DataTypeUtil.java @@ -87,8 +87,8 @@ public static boolean isUpperCastable(TajoDataTypes.Type define, TajoDataTypes.T */ public static org.apache.tajo.type.Type determineType(org.apache.tajo.type.Type left, org.apache.tajo.type.Type right) { - TajoDataTypes.Type rhsBaseType = right.baseType(); - switch (left.baseType()) { + TajoDataTypes.Type rhsBaseType = right.kind(); + switch (left.kind()) { case INT1: case INT2: diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java index a79008394a..39fb278a89 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/Column.java @@ -28,6 +28,7 @@ import org.apache.tajo.json.GsonObject; import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.type.Type; +import org.apache.tajo.type.TypeProtobufEncoder; /** * Describes a column. It is an immutable object. @@ -77,7 +78,7 @@ public Column(String name, Type type) { public Column(ColumnProto proto) { name = proto.getName(); - type = TypeConverter.convert(new TypeDesc(proto.getDataType())); + type = TypeProtobufEncoder.decode(proto.getType()); } /** @@ -121,6 +122,13 @@ public TypeDesc getTypeDesc() { return TypeConverter.convert(this.type); } + /** + * @return Type which includes domain type and scale. + */ + public Type getType() { + return this.type; + } + /** * * @return DataType which includes domain type and scale. @@ -151,7 +159,7 @@ public ColumnProto getProto() { ColumnProto.Builder builder = ColumnProto.newBuilder(); builder .setName(this.name) - .setDataType(TypeConverter.convert(this.type).dataType); + .setType(this.type.getProto()); return builder.build(); } diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaLegacy.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaLegacy.java index 8cda938706..0acdd72f82 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaLegacy.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaLegacy.java @@ -18,23 +18,26 @@ package org.apache.tajo.catalog; +import com.google.common.base.Function; import com.google.common.base.Objects; import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; import com.google.gson.annotations.Expose; import org.apache.tajo.catalog.SchemaUtil.ColumnVisitor; import org.apache.tajo.catalog.json.CatalogGsonHelper; import org.apache.tajo.catalog.proto.CatalogProtos.ColumnProto; import org.apache.tajo.catalog.proto.CatalogProtos.SchemaProto; import org.apache.tajo.common.ProtoObject; -import org.apache.tajo.common.TajoDataTypes.DataType; -import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.exception.DuplicateColumnException; import org.apache.tajo.exception.TajoRuntimeException; import org.apache.tajo.json.GsonObject; import org.apache.tajo.util.StringUtils; +import javax.annotation.Nullable; import java.util.*; +import static com.google.common.collect.Collections2.transform; + public class SchemaLegacy implements Schema, ProtoObject, Cloneable, GsonObject { @Expose protected List fields = null; @@ -55,47 +58,18 @@ public SchemaLegacy() { public SchemaLegacy(SchemaProto proto) { init(); - List toBeAdded = new ArrayList<>(); - for (int i = 0; i < proto.getFieldsCount(); i++) { - deserializeColumn(toBeAdded, proto.getFieldsList(), i); - } + Collection toBeAdded = transform(proto.getFieldsList(), new Function() { + @Override + public Column apply(@Nullable ColumnProto proto) { + return new Column(proto); + } + }); for (Column c : toBeAdded) { addColumn(c); } } - /** - * This method transforms a list of ColumnProtos into a schema tree. - * It assumes that protos contains a list of ColumnProtos in the depth-first order. - * - * @param tobeAdded - * @param protos - * @param serializedColumnIndex - */ - private static void deserializeColumn(List tobeAdded, List protos, int serializedColumnIndex) { - ColumnProto columnProto = protos.get(serializedColumnIndex); - if (columnProto.getDataType().getType() == Type.RECORD) { - - // Get the number of child fields - int childNum = columnProto.getDataType().getNumNestedFields(); - // where is start index of nested fields? - int childStartIndex = tobeAdded.size() - childNum; - // Extract nested fields - List nestedColumns = new ArrayList<>(tobeAdded.subList(childStartIndex, childStartIndex + childNum)); - - // Remove nested fields from the the current level - for (int i = 0; i < childNum; i++) { - tobeAdded.remove(tobeAdded.size() - 1); - } - - // Add the nested fields to the list as a single record column - tobeAdded.add(new Column(columnProto.getName(), new TypeDesc(new SchemaLegacy(nestedColumns)))); - } else { - tobeAdded.add(new Column(protos.get(serializedColumnIndex))); - } - } - public SchemaLegacy(Schema schema) { new SchemaLegacy(schema.getRootColumns()); } @@ -472,34 +446,15 @@ public Object clone() throws CloneNotSupportedException { @Override public SchemaProto getProto() { SchemaProto.Builder builder = SchemaProto.newBuilder(); - SchemaProtoBuilder recursiveBuilder = new SchemaProtoBuilder(builder); - SchemaUtil.visitSchema(this, recursiveBuilder); + builder.addAllFields(Iterables.transform(getRootColumns(), new Function() { + @Override + public ColumnProto apply(@Nullable Column column) { + return column.getProto(); + } + })); return builder.build(); } - private static class SchemaProtoBuilder implements ColumnVisitor { - private SchemaProto.Builder builder; - public SchemaProtoBuilder(SchemaProto.Builder builder) { - this.builder = builder; - } - - @Override - public void visit(int depth, List path, Column column) { - - if (column.getDataType().getType() == Type.RECORD) { - DataType.Builder updatedType = DataType.newBuilder(column.getDataType()); - updatedType.setNumNestedFields(TypeConverter.convert(column.type).nestedRecordSchema.size()); - - ColumnProto.Builder updatedColumn = ColumnProto.newBuilder(column.getProto()); - updatedColumn.setDataType(updatedType); - - builder.addFields(updatedColumn.build()); - } else { - builder.addFields(column.getProto()); - } - } - } - @Override public String toString() { StringBuilder sb = new StringBuilder(); diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TypeConverter.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TypeConverter.java index a926d05a30..387cf006f5 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TypeConverter.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/TypeConverter.java @@ -64,7 +64,7 @@ public static TypeDesc convert(Field src) { } public static TypeDesc convert(Type type) { - switch (type.baseType()) { + switch (type.kind()) { case CHAR: Char charType = (Char) type; return new TypeDesc(newDataTypeWithLen(TajoDataTypes.Type.CHAR, charType.length())); @@ -88,7 +88,7 @@ public static TypeDesc convert(Type type) { case ARRAY: Array array = (Array) type; Type elemType = array.elementType(); - switch (elemType.baseType()) { + switch (elemType.kind()) { case INT1: return new TypeDesc(newSimpleDataType(INT1_ARRAY)); case INT2: @@ -102,11 +102,11 @@ public static TypeDesc convert(Type type) { case FLOAT8: return new TypeDesc(newSimpleDataType(FLOAT8_ARRAY)); default: - return new TypeDesc(newSimpleDataType(type.baseType())); + return new TypeDesc(newSimpleDataType(type.kind())); } default: - return new TypeDesc(newSimpleDataType(type.baseType())); + return new TypeDesc(newSimpleDataType(type.kind())); } } } diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/TableStats.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/TableStats.java index c66500d2f3..47acb50f7c 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/TableStats.java +++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/TableStats.java @@ -26,6 +26,7 @@ import com.google.gson.annotations.Expose; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.common.TajoDataTypes; +import org.apache.tajo.common.TajoDataTypes.TypeProto; import org.apache.tajo.json.GsonObject; import org.apache.tajo.catalog.json.CatalogGsonHelper; import org.apache.tajo.catalog.proto.CatalogProtos.TableStatsProto; @@ -35,6 +36,8 @@ import java.util.ArrayList; import java.util.List; +import static org.apache.tajo.common.TajoDataTypes.Type.PROTOBUF; + public class TableStats implements ProtoObject, Cloneable, GsonObject { @Expose private Long numRows = null; // required @Expose private Long numBytes = null; // required @@ -85,13 +88,17 @@ public TableStats(CatalogProtos.TableStatsProto proto) { this.columnStatses = new ArrayList<>(); for (CatalogProtos.ColumnStatsProto colProto : proto.getColStatList()) { - if (colProto.getColumn().getDataType().getType() == TajoDataTypes.Type.PROTOBUF) { + if (peekType(colProto.getColumn().getType()) == PROTOBUF) { continue; } columnStatses.add(new ColumnStats(colProto)); } } + private static TajoDataTypes.Type peekType(TypeProto proto) { + return proto.getElements(proto.getElementsCount() - 1).getKind(); + } + public Long getNumRows() { return this.numRows; } diff --git a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto index 2b3409570d..7bfb3da910 100644 --- a/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto +++ b/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto @@ -66,7 +66,7 @@ enum AlterTableType { message ColumnProto { required string name = 1; optional int32 tid = 2; - required DataType data_type = 3; + required TypeProto type = 3; } message SchemaProto { diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java index 698ebe9cf3..b855c77733 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java @@ -59,6 +59,7 @@ import org.apache.tajo.plan.util.PartitionFilterAlgebraVisitor; import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.storage.StorageConstants; +import org.apache.tajo.type.TypeProtobufEncoder; import org.apache.tajo.util.KeyValueSet; import org.apache.thrift.TException; @@ -461,7 +462,7 @@ public final void createTable(final CatalogProtos.TableDescProto tableDescProto) for (Column eachField : columns) { cols.add(new FieldSchema(eachField.getSimpleName(), - HiveCatalogUtil.getHiveFieldType(eachField.getDataType()), "")); + HiveCatalogUtil.getHiveFieldType(eachField.getType()), "")); } sd.setCols(cols); @@ -470,7 +471,7 @@ public final void createTable(final CatalogProtos.TableDescProto tableDescProto) List partitionKeys = new ArrayList<>(); for (Column eachPartitionKey : tableDesc.getPartitionMethod().getExpressionSchema().getRootColumns()) { partitionKeys.add(new FieldSchema(eachPartitionKey.getSimpleName(), - HiveCatalogUtil.getHiveFieldType(eachPartitionKey.getDataType()), "")); + HiveCatalogUtil.getHiveFieldType(eachPartitionKey.getType()), "")); } table.setPartitionKeys(partitionKeys); } @@ -714,7 +715,7 @@ private void addNewColumn(String databaseName, String tableName, CatalogProtos.C Table table = client.getHiveClient().getTable(databaseName, tableName); List columns = table.getSd().getCols(); columns.add(new FieldSchema(columnProto.getName(), - HiveCatalogUtil.getHiveFieldType(columnProto.getDataType()), "")); + HiveCatalogUtil.getHiveFieldType(TypeProtobufEncoder.decode(columnProto.getType())), "")); client.getHiveClient().alter_table(databaseName, tableName, table); diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogUtil.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogUtil.java index 87b391ea60..687437e1a5 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogUtil.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogUtil.java @@ -39,6 +39,8 @@ import org.apache.tajo.exception.TajoRuntimeException; import org.apache.tajo.exception.UnknownDataFormatException; import org.apache.tajo.exception.UnsupportedException; +import org.apache.tajo.type.Type; +import org.apache.tajo.type.TypeStringEncoder; import org.apache.thrift.TException; public class HiveCatalogUtil { @@ -82,10 +84,10 @@ public static TajoDataTypes.Type getTajoFieldType(String dataType) throws LMDNoM } } - public static String getHiveFieldType(TajoDataTypes.DataType dataType) throws LMDNoMatchedDatatypeException { - Preconditions.checkNotNull(dataType); + public static String getHiveFieldType(Type type) throws LMDNoMatchedDatatypeException { + Preconditions.checkNotNull(type); - switch (dataType.getType()) { + switch (type.kind()) { case CHAR: return serdeConstants.CHAR_TYPE_NAME; case BOOLEAN: return serdeConstants.BOOLEAN_TYPE_NAME; case INT1: return serdeConstants.TINYINT_TYPE_NAME; @@ -104,7 +106,7 @@ public static String getHiveFieldType(TajoDataTypes.DataType dataType) throws LM case DATE: return serdeConstants.DATE_TYPE_NAME; case TIMESTAMP: return serdeConstants.TIMESTAMP_TYPE_NAME; default: - throw new LMDNoMatchedDatatypeException(dataType.getType().name()); + throw new LMDNoMatchedDatatypeException(TypeStringEncoder.serialize(type)); } } diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/AbstractTableDescriptor.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/AbstractTableDescriptor.java index 473e3c69b2..1cf3e64c09 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/AbstractTableDescriptor.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/dictionary/AbstractTableDescriptor.java @@ -22,6 +22,7 @@ import org.apache.tajo.catalog.proto.CatalogProtos.*; import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.KeyValueSetProto; import org.apache.tajo.schema.IdentifierUtil; +import org.apache.tajo.type.TypeFactory; abstract class AbstractTableDescriptor implements TableDescriptor { @@ -41,13 +42,7 @@ protected SchemaProto getSchemaProto() { columnBuilder = ColumnProto.newBuilder(); columnBuilder.setName(columnDescriptor.getName().toLowerCase()); - if (columnDescriptor.getLength() > 0) { - columnBuilder.setDataType(CatalogUtil.newDataTypeWithLen(columnDescriptor.getType(), - columnDescriptor.getLength())); - } else { - columnBuilder.setDataType(CatalogUtil.newSimpleDataType(columnDescriptor.getType())); - } - + columnBuilder.setType(TypeFactory.create(columnDescriptor.getType()).getProto()); schemaBuilder.addFields(columnBuilder.build()); } diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java index 972a85b6c4..5317c74940 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java @@ -31,13 +31,14 @@ import org.apache.tajo.catalog.*; import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.catalog.proto.CatalogProtos.*; -import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.common.TajoDataTypes.Type; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.exception.*; import org.apache.tajo.plan.expr.AlgebraicUtil; import org.apache.tajo.plan.util.PartitionFilterAlgebraVisitor; import org.apache.tajo.schema.IdentifierUtil; +import org.apache.tajo.type.TypeProtobufEncoder; +import org.apache.tajo.type.TypeStringEncoder; import org.apache.tajo.util.JavaResourceUtil; import org.apache.tajo.util.Pair; @@ -812,9 +813,9 @@ public void createTable(final CatalogProtos.TableDescProto table) String colSql = "INSERT INTO " + TB_COLUMNS + - // 1 2 3 4 5 6 - " (TID, COLUMN_NAME, ORDINAL_POSITION, NESTED_FIELD_NUM, DATA_TYPE, TYPE_LENGTH)" + - " VALUES(?, ?, ?, ?, ?, ?) "; + // 1 2 3 4 + " (TID, COLUMN_NAME, ORDINAL_POSITION, DATA_TYPE)" + + " VALUES(?, ?, ?, ?) "; if (LOG.isDebugEnabled()) { LOG.debug(colSql); @@ -823,15 +824,12 @@ public void createTable(final CatalogProtos.TableDescProto table) pstmt = conn.prepareStatement(colSql); for (int i = 0; i < table.getSchema().getFieldsCount(); i++) { ColumnProto col = table.getSchema().getFields(i); - TajoDataTypes.DataType dataType = col.getDataType(); + org.apache.tajo.type.Type type = TypeProtobufEncoder.decode(col.getType()); pstmt.setInt(1, tableId); pstmt.setString(2, IdentifierUtil.extractSimpleName(col.getName())); pstmt.setInt(3, i); - // the default number of nested fields is 0. - pstmt.setInt(4, dataType.hasNumNestedFields() ? dataType.getNumNestedFields() : 0); - pstmt.setString(5, dataType.getType().name()); - pstmt.setInt(6, (col.getDataType().hasLength() ? col.getDataType().getLength() : 0)); + pstmt.setString(4, TypeStringEncoder.serialize(type)); pstmt.addBatch(); pstmt.clearParameters(); } @@ -1150,13 +1148,13 @@ private void renameColumn(final int tableId, final CatalogProtos.AlterColumnProt throws UndefinedColumnException, AmbiguousTableException { final String selectColumnSql = - "SELECT COLUMN_NAME, DATA_TYPE, TYPE_LENGTH, ORDINAL_POSITION, NESTED_FIELD_NUM from " + TB_COLUMNS + + "SELECT COLUMN_NAME, DATA_TYPE, ORDINAL_POSITION from " + TB_COLUMNS + " WHERE " + COL_TABLES_PK + " = ?" + " AND COLUMN_NAME = ?" ; final String deleteColumnNameSql = "DELETE FROM " + TB_COLUMNS + " WHERE TID = ? AND COLUMN_NAME = ?"; final String insertNewColumnSql = "INSERT INTO " + TB_COLUMNS + - " (TID, COLUMN_NAME, ORDINAL_POSITION, NESTED_FIELD_NUM, DATA_TYPE, TYPE_LENGTH) VALUES(?, ?, ?, ?, ?, ?) "; + " (TID, COLUMN_NAME, ORDINAL_POSITION, DATA_TYPE) VALUES(?, ?, ?, ?) "; if (LOG.isDebugEnabled()) { LOG.debug(selectColumnSql); @@ -1189,15 +1187,15 @@ private void renameColumn(final int tableId, final CatalogProtos.AlterColumnProt resultSet = pstmt.executeQuery(); CatalogProtos.ColumnProto columnProto = null; + String typeStr = null; int ordinalPosition = 0; - int nestedFieldNum = 0; if (resultSet.next()) { columnProto = resultToColumnProto(resultSet); //NOTE ==> Setting new column Name columnProto = columnProto.toBuilder().setName(alterColumnProto.getNewColumnName()).build(); ordinalPosition = resultSet.getInt("ORDINAL_POSITION"); - nestedFieldNum = resultSet.getInt("NESTED_FIELD_NUM"); + typeStr = TypeStringEncoder.serialize(TypeProtobufEncoder.decode(columnProto.getType())); } else { throw new UndefinedColumnException(alterColumnProto.getOldColumnName()); } @@ -1218,9 +1216,7 @@ private void renameColumn(final int tableId, final CatalogProtos.AlterColumnProt pstmt.setInt(1, tableId); pstmt.setString(2, simpleNewColumnName); pstmt.setInt(3, ordinalPosition); - pstmt.setInt(4, nestedFieldNum); - pstmt.setString(5, columnProto.getDataType().getType().name()); - pstmt.setInt(6, (columnProto.getDataType().hasLength() ? columnProto.getDataType().getLength() : 0)); + pstmt.setString(4, typeStr); pstmt.executeUpdate(); conn.commit(); @@ -1244,7 +1240,7 @@ private void addNewColumn(int tableId, CatalogProtos.ColumnProto columnProto) th final String insertNewColumnSql = "INSERT INTO " + TB_COLUMNS + - " (TID, COLUMN_NAME, ORDINAL_POSITION, NESTED_FIELD_NUM, DATA_TYPE, TYPE_LENGTH) VALUES(?, ?, ?, ?, ?, ?) "; + " (TID, COLUMN_NAME, ORDINAL_POSITION, DATA_TYPE) VALUES(?, ?, ?, ?) "; final String columnCountSql = "SELECT MAX(ORDINAL_POSITION) AS POSITION FROM " + TB_COLUMNS + " WHERE TID = ?"; @@ -1272,15 +1268,13 @@ private void addNewColumn(int tableId, CatalogProtos.ColumnProto columnProto) th pstmt.close(); resultSet = null; - TajoDataTypes.DataType dataType = columnProto.getDataType(); + org.apache.tajo.type.Type type = TypeProtobufEncoder.decode(columnProto.getType()); pstmt = conn.prepareStatement(insertNewColumnSql); pstmt.setInt(1, tableId); pstmt.setString(2, IdentifierUtil.extractSimpleName(columnProto.getName())); pstmt.setInt(3, position + 1); - pstmt.setInt(4, dataType.hasNumNestedFields() ? dataType.getNumNestedFields() : 0); - pstmt.setString(5, dataType.getType().name()); - pstmt.setInt(6, (columnProto.getDataType().hasLength() ? columnProto.getDataType().getLength() : 0)); + pstmt.setString(4, TypeStringEncoder.serialize(type)); pstmt.executeUpdate(); } catch (SQLException sqlException) { @@ -1619,7 +1613,7 @@ public CatalogProtos.TableDescProto getTable(String databaseName, String tableNa // Geting Column Descriptions ////////////////////////////////////////// CatalogProtos.SchemaProto.Builder schemaBuilder = CatalogProtos.SchemaProto.newBuilder(); - sql = "SELECT COLUMN_NAME, NESTED_FIELD_NUM, DATA_TYPE, TYPE_LENGTH from " + TB_COLUMNS + + sql = "SELECT COLUMN_NAME, DATA_TYPE from " + TB_COLUMNS + " WHERE " + COL_TABLES_PK + " = ? ORDER BY ORDINAL_POSITION ASC"; if (LOG.isDebugEnabled()) { @@ -1848,7 +1842,7 @@ public List getAllColumns() { try { String sql = - "SELECT TID, COLUMN_NAME, ORDINAL_POSITION, NESTED_FIELD_NUM, DATA_TYPE, TYPE_LENGTH FROM " + TB_COLUMNS + + "SELECT TID, COLUMN_NAME, ORDINAL_POSITION, DATA_TYPE FROM " + TB_COLUMNS + " ORDER BY TID ASC, ORDINAL_POSITION ASC"; conn = getConnection(); @@ -1862,20 +1856,8 @@ public List getAllColumns() { String tableName = getTableName(conn, tid); builder.setTid(tid); builder.setName(IdentifierUtil.buildFQName(databaseName, tableName, resultSet.getString("COLUMN_NAME"))); - - int nestedFieldNum = resultSet.getInt("NESTED_FIELD_NUM"); - - Type type = getDataType(resultSet.getString("DATA_TYPE").trim()); - int typeLength = resultSet.getInt("TYPE_LENGTH"); - - if (nestedFieldNum > 0) { - builder.setDataType(CatalogUtil.newRecordType(nestedFieldNum)); - } else if (typeLength > 0) { - builder.setDataType(CatalogUtil.newDataTypeWithLen(type, typeLength)); - } else { - builder.setDataType(CatalogUtil.newSimpleDataType(type)); - } - + org.apache.tajo.type.Type type = TypeStringEncoder.deserialize(resultSet.getString("DATA_TYPE").trim()); + builder.setType(type.getProto()); columns.add(builder.build()); } } catch (SQLException se) { @@ -2530,7 +2512,7 @@ public void createIndex(final IndexDescProto proto) // Since the key columns are always sorted in order of their occurrence position in the relation schema, // the concatenated name can be uniquely identified. columnNamesBuilder.append(columnSpec.getSortKey().getSimpleName()).append(","); - dataTypesBuilder.append(columnSpec.getSortKey().getDataType().getType().name()).append(","); + dataTypesBuilder.append(columnSpec.getSortKey().getDataType().getType().name()).append("|"); ordersBuilder.append(columnSpec.isAscending()).append(","); nullOrdersBuilder.append(columnSpec.isNullsFirst()).append(","); } @@ -2898,14 +2880,14 @@ private void resultToIndexDescProtoBuilder(final String qualifier, builder.setIndexPath(res.getString("path")); String[] columnNames, dataTypes, orders, nullOrders; columnNames = res.getString("column_names").trim().split(","); - dataTypes = res.getString("data_types").trim().split(","); + dataTypes = res.getString("data_types").trim().split("\\|"); orders = res.getString("orders").trim().split(","); nullOrders = res.getString("null_orders").trim().split(","); int columnNum = columnNames.length; for (int i = 0; i < columnNum; i++) { SortSpecProto.Builder colSpecBuilder = SortSpecProto.newBuilder(); colSpecBuilder.setColumn(ColumnProto.newBuilder().setName(IdentifierUtil.buildFQName(qualifier, columnNames[i])) - .setDataType(CatalogUtil.newSimpleDataType(getDataType(dataTypes[i]))).build()); + .setType(TypeStringEncoder.deserialize(dataTypes[i]).getProto()).build()); colSpecBuilder.setAscending(orders[i].equals("true")); colSpecBuilder.setNullFirst(nullOrders[i].equals("true")); builder.addKeySortSpecs(colSpecBuilder.build()); @@ -2917,20 +2899,8 @@ private void resultToIndexDescProtoBuilder(final String qualifier, private ColumnProto resultToColumnProto(final ResultSet res) throws SQLException { ColumnProto.Builder builder = ColumnProto.newBuilder(); builder.setName(res.getString("column_name").trim()); - - int nestedFieldNum = res.getInt("NESTED_FIELD_NUM"); - - Type type = getDataType(res.getString("data_type").trim()); - int typeLength = res.getInt("type_length"); - - if (nestedFieldNum > 0) { - builder.setDataType(CatalogUtil.newRecordType(nestedFieldNum)); - } else if (typeLength > 0) { - builder.setDataType(CatalogUtil.newDataTypeWithLen(type, typeLength)); - } else { - builder.setDataType(CatalogUtil.newSimpleDataType(type)); - } - + org.apache.tajo.type.Type type = TypeStringEncoder.deserialize(res.getString("data_type").trim()); + builder.setType(type.getProto()); return builder.build(); } diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml index 96100e8430..d7ef9e9cc8 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml +++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml @@ -88,9 +88,7 @@ TID INT NOT NULL REFERENCES TABLES (TID) ON DELETE CASCADE, COLUMN_NAME VARCHAR(128) NOT NULL, ORDINAL_POSITION INTEGER NOT NULL, - NESTED_FIELD_NUM INTEGER NOT NULL, - DATA_TYPE CHAR(16), - TYPE_LENGTH INTEGER, + DATA_TYPE VARCHAR(8192), CONSTRAINT COLUMNS_PK PRIMARY KEY (TID, ORDINAL_POSITION) )]]> diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mariadb/mariadb.xml b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mariadb/mariadb.xml index a94489d73c..f67851d1c4 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mariadb/mariadb.xml +++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mariadb/mariadb.xml @@ -81,9 +81,7 @@ TID INT NOT NULL, COLUMN_NAME VARCHAR(255) BINARY NOT NULL, ORDINAL_POSITION INT NOT NULL, - NESTED_FIELD_NUM INT NOT NULL, - DATA_TYPE CHAR(16), - TYPE_LENGTH INTEGER, + DATA_TYPE VARCHAR(8192), PRIMARY KEY (TID, ORDINAL_POSITION), FOREIGN KEY (TID) REFERENCES TABLES (TID) ON DELETE CASCADE )]]> diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mysql/mysql.xml b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mysql/mysql.xml index c0dadaa3da..cf465870b9 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mysql/mysql.xml +++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/mysql/mysql.xml @@ -81,9 +81,7 @@ TID INT NOT NULL, COLUMN_NAME VARCHAR(255) BINARY NOT NULL, ORDINAL_POSITION INT NOT NULL, - NESTED_FIELD_NUM INT NOT NULL, - DATA_TYPE CHAR(16), - TYPE_LENGTH INTEGER, + DATA_TYPE VARCHAR(8192), PRIMARY KEY (TID, ORDINAL_POSITION), FOREIGN KEY (TID) REFERENCES TABLES (TID) ON DELETE CASCADE )]]> diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/oracle.xml b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/oracle.xml index 190270c87d..033f16faae 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/oracle.xml +++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/oracle/oracle.xml @@ -134,9 +134,7 @@ TID INT NOT NULL, COLUMN_NAME VARCHAR2(255) NOT NULL, ORDINAL_POSITION INT NOT NULL, - NESTED_FIELD_NUM INT NOT NULL, - DATA_TYPE CHAR(16), - TYPE_LENGTH INTEGER, + DATA_TYPE VARCHAR(8192), CONSTRAINT COLUMNS_PKEY PRIMARY KEY (TID, ORDINAL_POSITION), FOREIGN KEY (TID) REFERENCES TABLES (TID) ON DELETE CASCADE )]]> diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/postgresql.xml b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/postgresql.xml index 33a1fd24c4..c901e2dd74 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/postgresql.xml +++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/postgresql/postgresql.xml @@ -94,9 +94,7 @@ xsi:schemaLocation="http://tajo.apache.org/catalogstore ../DBMSSchemaDefinition. TID INT NOT NULL, COLUMN_NAME VARCHAR(255) NOT NULL, ORDINAL_POSITION INT NOT NULL, - NESTED_FIELD_NUM INT NOT NULL, - DATA_TYPE CHAR(16), - TYPE_LENGTH INTEGER, + DATA_TYPE VARCHAR(8192), CONSTRAINT COLUMNS_PKEY PRIMARY KEY (TID, ORDINAL_POSITION), FOREIGN KEY (TID) REFERENCES TABLES (TID) ON DELETE CASCADE )]]> diff --git a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java index 873c938c29..8df15b8347 100644 --- a/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java +++ b/tajo-catalog/tajo-catalog-server/src/test/java/org/apache/tajo/catalog/TestCatalogAgainstCaseSensitivity.java @@ -409,7 +409,7 @@ public void testTableColumnWithNestedField() throws Exception { int expected = 0; for (String eachTableName : catalog.getAllTableNames(databaseName)) { - expected += catalog.getTableDesc(databaseName, eachTableName).getSchema().getAllColumns().size(); + expected += catalog.getTableDesc(databaseName, eachTableName).getSchema().getRootColumns().size(); } assertEquals(expected, columnCount); } diff --git a/tajo-common/src/main/java/org/apache/tajo/exception/InvalidValueForCastException.java b/tajo-common/src/main/java/org/apache/tajo/exception/InvalidValueForCastException.java index 6af8d1a1b3..63ee354dfc 100644 --- a/tajo-common/src/main/java/org/apache/tajo/exception/InvalidValueForCastException.java +++ b/tajo-common/src/main/java/org/apache/tajo/exception/InvalidValueForCastException.java @@ -39,6 +39,6 @@ public InvalidValueForCastException(TajoDataTypes.Type src, TajoDataTypes.Type t } public InvalidValueForCastException(Type src, Type target) { - super(Errors.ResultCode.INVALID_VALUE_FOR_CAST, src.baseType().name(), target.baseType().name()); + super(Errors.ResultCode.INVALID_VALUE_FOR_CAST, src.kind().name(), target.kind().name()); } } diff --git a/tajo-common/src/main/java/org/apache/tajo/schema/Field.java b/tajo-common/src/main/java/org/apache/tajo/schema/Field.java index ae0a3792cd..90193acae5 100644 --- a/tajo-common/src/main/java/org/apache/tajo/schema/Field.java +++ b/tajo-common/src/main/java/org/apache/tajo/schema/Field.java @@ -60,7 +60,7 @@ public QualifiedIdentifier name() { } public TajoDataTypes.Type baseType() { - return this.type.baseType(); + return this.type.kind(); } public T type() { diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Any.java b/tajo-common/src/main/java/org/apache/tajo/type/Any.java index 04dfcebc0e..342fdb68f7 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Any.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Any.java @@ -20,7 +20,7 @@ import static org.apache.tajo.common.TajoDataTypes.Type.ANY; -public class Any extends PrimitiveType { +public class Any extends Type { public Any() { super(ANY); } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Blob.java b/tajo-common/src/main/java/org/apache/tajo/type/Blob.java index 5ae9a83edc..360ea13fba 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Blob.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Blob.java @@ -20,7 +20,7 @@ import static org.apache.tajo.common.TajoDataTypes.Type.BLOB; -public class Blob extends PrimitiveType { +public class Blob extends Type { public Blob() { super(BLOB); } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Bool.java b/tajo-common/src/main/java/org/apache/tajo/type/Bool.java index b32752592a..923e8e90ec 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Bool.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Bool.java @@ -20,7 +20,7 @@ import static org.apache.tajo.common.TajoDataTypes.Type.BOOLEAN; -public class Bool extends PrimitiveType { +public class Bool extends Type { public Bool() { super(BOOLEAN); } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Date.java b/tajo-common/src/main/java/org/apache/tajo/type/Date.java index 1ff38cd5de..a68ab97a49 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Date.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Date.java @@ -20,7 +20,7 @@ import static org.apache.tajo.common.TajoDataTypes.Type.DATE; -public class Date extends PrimitiveType { +public class Date extends Type { public Date() { super(DATE); } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Float4.java b/tajo-common/src/main/java/org/apache/tajo/type/Float4.java index a6dc610b79..b8c82181ee 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Float4.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Float4.java @@ -20,7 +20,7 @@ import static org.apache.tajo.common.TajoDataTypes.Type.FLOAT4; -public class Float4 extends PrimitiveType { +public class Float4 extends Type { public Float4() { super(FLOAT4); } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Float8.java b/tajo-common/src/main/java/org/apache/tajo/type/Float8.java index a804654b66..f36fac87c8 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Float8.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Float8.java @@ -20,7 +20,7 @@ import static org.apache.tajo.common.TajoDataTypes.Type.FLOAT8; -public class Float8 extends PrimitiveType { +public class Float8 extends Type { public Float8() { super(FLOAT8); } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Inet4.java b/tajo-common/src/main/java/org/apache/tajo/type/Inet4.java index 020509be1b..541841ec07 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Inet4.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Inet4.java @@ -20,7 +20,7 @@ import static org.apache.tajo.common.TajoDataTypes.Type.INET4; -public class Inet4 extends PrimitiveType { +public class Inet4 extends Type { public Inet4() { super(INET4); } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Int1.java b/tajo-common/src/main/java/org/apache/tajo/type/Int1.java index 2a4e3996ae..d86d36e196 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Int1.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Int1.java @@ -20,7 +20,7 @@ import static org.apache.tajo.common.TajoDataTypes.Type.INT1; -public class Int1 extends PrimitiveType { +public class Int1 extends Type { public Int1() { super(INT1); } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Int2.java b/tajo-common/src/main/java/org/apache/tajo/type/Int2.java index a8f4eec79a..e7c221541d 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Int2.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Int2.java @@ -20,7 +20,7 @@ import static org.apache.tajo.common.TajoDataTypes.Type.INT2; -public class Int2 extends PrimitiveType { +public class Int2 extends Type { public Int2() { super(INT2); } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Int4.java b/tajo-common/src/main/java/org/apache/tajo/type/Int4.java index 61c673f5b4..e8db7841a0 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Int4.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Int4.java @@ -20,7 +20,7 @@ import static org.apache.tajo.common.TajoDataTypes.Type.INT4; -public class Int4 extends PrimitiveType { +public class Int4 extends Type { public Int4() { super(INT4); } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Int8.java b/tajo-common/src/main/java/org/apache/tajo/type/Int8.java index 2e257d349c..999e9acd97 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Int8.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Int8.java @@ -20,7 +20,7 @@ import static org.apache.tajo.common.TajoDataTypes.Type.INT8; -public class Int8 extends PrimitiveType { +public class Int8 extends Type { public Int8() { super(INT8); } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Interval.java b/tajo-common/src/main/java/org/apache/tajo/type/Interval.java index 5bc0c69641..d6dc17eda8 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Interval.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Interval.java @@ -20,7 +20,7 @@ import static org.apache.tajo.common.TajoDataTypes.Type.INTERVAL; -public class Interval extends PrimitiveType { +public class Interval extends Type { public Interval() { super(INTERVAL); } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Null.java b/tajo-common/src/main/java/org/apache/tajo/type/Null.java index 0896e8274b..74cf052a0b 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Null.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Null.java @@ -20,7 +20,7 @@ import static org.apache.tajo.common.TajoDataTypes.Type.NULL_TYPE; -public class Null extends PrimitiveType { +public class Null extends Type { public Null() { super(NULL_TYPE); } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/PrimitiveType.java b/tajo-common/src/main/java/org/apache/tajo/type/PrimitiveType.java deleted file mode 100644 index bda946718f..0000000000 --- a/tajo-common/src/main/java/org/apache/tajo/type/PrimitiveType.java +++ /dev/null @@ -1,27 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.type; - -import org.apache.tajo.common.TajoDataTypes; - -public abstract class PrimitiveType extends Type { - public PrimitiveType(TajoDataTypes.Type baseType) { - super(baseType); - } -} diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Protobuf.java b/tajo-common/src/main/java/org/apache/tajo/type/Protobuf.java index 2cc08f2f9e..e286724644 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Protobuf.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Protobuf.java @@ -20,7 +20,7 @@ import static org.apache.tajo.common.TajoDataTypes.Type.PROTOBUF; -public class Protobuf extends PrimitiveType { +public class Protobuf extends Type { private String msgName; public Protobuf(String msgName) { diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Record.java b/tajo-common/src/main/java/org/apache/tajo/type/Record.java index c6526dcaac..46d7e1902c 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Record.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Record.java @@ -19,7 +19,6 @@ package org.apache.tajo.type; import com.google.common.collect.ImmutableList; -import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.schema.Field; import org.apache.tajo.util.StringUtils; @@ -60,7 +59,7 @@ public String toString() { @Override public int hashCode() { - return Objects.hash(baseType(), Objects.hash(fields)); + return Objects.hash(kind(), Objects.hash(fields)); } @Override @@ -73,11 +72,6 @@ public boolean equals(Object object) { return false; } - @Override - public TajoDataTypes.TypeProto getProto() { - return null; - } - @Override public Iterator iterator() { return fields.iterator(); diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Text.java b/tajo-common/src/main/java/org/apache/tajo/type/Text.java index 066c839e33..aab49a497d 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Text.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Text.java @@ -20,7 +20,7 @@ import static org.apache.tajo.common.TajoDataTypes.Type.TEXT; -public class Text extends PrimitiveType { +public class Text extends Type { public Text() { super(TEXT); } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Time.java b/tajo-common/src/main/java/org/apache/tajo/type/Time.java index 6d624d7086..12fe5f82db 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Time.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Time.java @@ -20,7 +20,7 @@ import static org.apache.tajo.common.TajoDataTypes.Type.TIME; -public class Time extends PrimitiveType { +public class Time extends Type { public Time() { super(TIME); } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Timestamp.java b/tajo-common/src/main/java/org/apache/tajo/type/Timestamp.java index 17a0cb3384..577f877419 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Timestamp.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Timestamp.java @@ -20,7 +20,7 @@ import static org.apache.tajo.common.TajoDataTypes.Type.TIMESTAMP; -public class Timestamp extends PrimitiveType { +public class Timestamp extends Type { public Timestamp() { super(TIMESTAMP); } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Type.java b/tajo-common/src/main/java/org/apache/tajo/type/Type.java index 33334c2493..35d666c712 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Type.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Type.java @@ -52,14 +52,14 @@ public abstract class Type implements Cloneable, ProtoObject { public static final Blob Blob = new Blob(); public static final Inet4 Inet4 = new Inet4(); - protected TajoDataTypes.Type baseType; + protected TajoDataTypes.Type kind; - public Type(TajoDataTypes.Type baseType) { - this.baseType = baseType; + public Type(TajoDataTypes.Type kind) { + this.kind = kind; } - public TajoDataTypes.Type baseType() { - return baseType; + public TajoDataTypes.Type kind() { + return kind; } public boolean isTypeParameterized() { @@ -84,24 +84,24 @@ protected static String typeName(TajoDataTypes.Type type) { @Override public int hashCode() { - return baseType().hashCode(); + return kind().hashCode(); } @Override public boolean equals(Object t) { - return t instanceof Type && ((Type)t).baseType() == baseType(); + return t instanceof Type && ((Type)t).kind() == kind(); } @Override public String toString() { - return typeName(baseType()); + return typeName(kind()); } public boolean isStruct() { - return this.baseType() == TajoDataTypes.Type.RECORD; + return this.kind() == TajoDataTypes.Type.RECORD; } - public boolean isNull() { return this.baseType() == TajoDataTypes.Type.NULL_TYPE; } + public boolean isNull() { return this.kind() == TajoDataTypes.Type.NULL_TYPE; } public static int DEFAULT_PRECISION = 0; public static int DEFAULT_SCALE = 0; @@ -148,6 +148,6 @@ public static Null Null() { @Override public TypeProto getProto() { - return TypeProtobufEncoder.serialize(this); + return TypeProtobufEncoder.encode(this); } } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/TypeParamterizedType.java b/tajo-common/src/main/java/org/apache/tajo/type/TypeParamterizedType.java index 1abd4616fb..bacf6c7a3e 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/TypeParamterizedType.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/TypeParamterizedType.java @@ -54,7 +54,7 @@ public boolean equals(Object object) { if (object instanceof TypeParamterizedType) { TypeParamterizedType other = (TypeParamterizedType) object; - return this.baseType.equals(other.baseType) && params.equals(other.params); + return this.kind.equals(other.kind) && params.equals(other.params); } return false; @@ -62,13 +62,13 @@ public boolean equals(Object object) { @Override public int hashCode() { - return Objects.hash(baseType(), params); + return Objects.hash(kind(), params); } @Override public String toString() { StringBuilder sb = new StringBuilder(); - sb.append(typeName(this.baseType)); + sb.append(typeName(this.kind)); sb.append("<"); sb.append(StringUtils.join(params, ",")); sb.append(">"); diff --git a/tajo-common/src/main/java/org/apache/tajo/type/TypeProtobufEncoder.java b/tajo-common/src/main/java/org/apache/tajo/type/TypeProtobufEncoder.java index 5711a26f50..4fd2897701 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/TypeProtobufEncoder.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/TypeProtobufEncoder.java @@ -40,14 +40,14 @@ import static org.apache.tajo.type.Type.Record; public class TypeProtobufEncoder { - public static TypeProto serialize(Type type) { + public static TypeProto encode(Type type) { final TypeProto.Builder builder = TypeProto.newBuilder(); new Visitor(builder).visit(type); return builder.build(); } - public static Type deserialize(TypeProto proto) { + public static Type decode(TypeProto proto) { Stack stack = new Stack<>(); for (int curIdx = 0; curIdx < proto.getElementsCount(); curIdx++) { @@ -90,14 +90,7 @@ static List popMultiItems(Stack stack, int num) { } static boolean isValueParameterized(TajoDataTypes.Type baseType) { - switch (baseType) { - case CHAR: - case VARCHAR: - case NUMERIC: - return true; - default: - return false; - } + return baseType == CHAR || baseType == VARCHAR || baseType == NUMERIC; } static Type createPrimitiveType(TypeElement element) { @@ -105,6 +98,8 @@ static Type createPrimitiveType(TypeElement element) { if (isValueParameterized(element.getKind())) { return TypeFactory.create(element.getKind(), EMPTY_LIST, element.getValueParamsList(), EMPTY_LIST); + } else if (element.getKind() == PROTOBUF) { // TODO - PROTOBUF type should be removed later + return new Protobuf(element.getStringParams(0)); } else { return TypeFactory.create(element.getKind()); } @@ -138,10 +133,12 @@ static class Visitor extends TypeVisitor { @Override public void visitPrimitive(Type type) { TypeElement.Builder typeElemBuilder = TypeElement.newBuilder() - .setKind(type.baseType); + .setKind(type.kind); if (type.isValueParameterized()) { typeElemBuilder.addAllValueParams(type.getValueParameters()); + } else if (type.kind == PROTOBUF) { + typeElemBuilder.addStringParams(((Protobuf)type).getMessageName()); } builder.addElements(typeElemBuilder); @@ -151,7 +148,7 @@ public void visitPrimitive(Type type) { public void visitMap(Map map) { super.visitMap(map); builder.addElements(TypeElement.newBuilder() - .setKind(map.baseType) + .setKind(map.kind) .setChildNum(2) ); } @@ -161,7 +158,7 @@ public void visitArray(Array array) { super.visitArray(array); builder .addElements(TypeElement.newBuilder() - .setKind(array.baseType) + .setKind(array.kind) .setChildNum(1) ); } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/TypeStringEncoder.java b/tajo-common/src/main/java/org/apache/tajo/type/TypeStringEncoder.java index acda5a7dfb..16dcd7ef71 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/TypeStringEncoder.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/TypeStringEncoder.java @@ -46,7 +46,7 @@ public class TypeStringEncoder { * @return A type string representation */ public static String serialize(Type type) { - StringBuilder sb = new StringBuilder(type.baseType().name()); + StringBuilder sb = new StringBuilder(type.kind().name()); if (type.isTypeParameterized()) { sb.append("<"); diff --git a/tajo-common/src/main/java/org/apache/tajo/type/TypeVisitor.java b/tajo-common/src/main/java/org/apache/tajo/type/TypeVisitor.java index ee067d1c48..8e437f4a2e 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/TypeVisitor.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/TypeVisitor.java @@ -22,7 +22,7 @@ public abstract class TypeVisitor { public void visit(Type type) { - switch (type.baseType) { + switch (type.kind) { case ARRAY: visitArray((Array) type); break; diff --git a/tajo-common/src/main/java/org/apache/tajo/type/ValueParamterizedType.java b/tajo-common/src/main/java/org/apache/tajo/type/ValueParamterizedType.java index f5300b6c51..30986d3dba 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/ValueParamterizedType.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/ValueParamterizedType.java @@ -57,7 +57,7 @@ public boolean equals(Object object) { if (object instanceof ValueParamterizedType) { ValueParamterizedType other = (ValueParamterizedType) object; - return this.baseType.equals(other.baseType) && params.equals(other.params); + return this.kind.equals(other.kind) && params.equals(other.params); } return false; @@ -65,13 +65,13 @@ public boolean equals(Object object) { @Override public int hashCode() { - return Objects.hash(baseType(), params); + return Objects.hash(kind(), params); } @Override public String toString() { StringBuilder sb = new StringBuilder(); - sb.append(typeName(this.baseType)); + sb.append(typeName(this.kind)); sb.append("("); sb.append(StringUtils.join(params, ",")); sb.append(")"); diff --git a/tajo-common/src/main/proto/DataTypes.proto b/tajo-common/src/main/proto/DataTypes.proto index 11f1582c49..360f59516b 100644 --- a/tajo-common/src/main/proto/DataTypes.proto +++ b/tajo-common/src/main/proto/DataTypes.proto @@ -123,8 +123,9 @@ message TypeProto { message TypeElement { required Type kind = 1; repeated int32 value_params = 2; - repeated QualifiedIdentifierProto field_names = 3; - optional int32 child_num = 4; + repeated string string_params = 3; + repeated QualifiedIdentifierProto field_names = 4; + optional int32 child_num = 5; } message QualifiedIdentifierProto { diff --git a/tajo-common/src/test/java/org/apache/tajo/type/TestType.java b/tajo-common/src/test/java/org/apache/tajo/type/TestType.java index e73ca864bc..1da0db7681 100644 --- a/tajo-common/src/test/java/org/apache/tajo/type/TestType.java +++ b/tajo-common/src/test/java/org/apache/tajo/type/TestType.java @@ -44,47 +44,47 @@ public class TestType { @Test public final void testPrimitiveTypes() { - assertEquals(Bool.baseType(), BOOLEAN); - assertEquals(Int1.baseType(), INT1); - assertEquals(Int2.baseType(), INT2); - assertEquals(Int4.baseType(), INT4); - assertEquals(Int8.baseType(), INT8); - assertEquals(Float4.baseType(), FLOAT4); - assertEquals(Float8.baseType(), FLOAT8); - assertEquals(Date.baseType(), DATE); - assertEquals(Time.baseType(), TIME); - assertEquals(Timestamp.baseType(), TIMESTAMP); + assertEquals(Bool.kind(), BOOLEAN); + assertEquals(Int1.kind(), INT1); + assertEquals(Int2.kind(), INT2); + assertEquals(Int4.kind(), INT4); + assertEquals(Int8.kind(), INT8); + assertEquals(Float4.kind(), FLOAT4); + assertEquals(Float8.kind(), FLOAT8); + assertEquals(Date.kind(), DATE); + assertEquals(Time.kind(), TIME); + assertEquals(Timestamp.kind(), TIMESTAMP); Numeric n = Numeric(4, 2); - assertEquals(n.baseType(), NUMERIC); + assertEquals(n.kind(), NUMERIC); assertEquals(n.precision(), 4); assertEquals(n.scale(), 2); - assertEquals(Blob.baseType(), BLOB); + assertEquals(Blob.kind(), BLOB); Char c = Char(2); - assertEquals(c.baseType(), CHAR); + assertEquals(c.kind(), CHAR); assertEquals(c.length(), 2); Varchar varchar = Varchar(2); - assertEquals(varchar.baseType(), VARCHAR); + assertEquals(varchar.kind(), VARCHAR); assertEquals(varchar.length(), 2); Record record = Type.Record(Field("x", Int8), Field("y", Array(Float8))); - assertEquals(record.baseType(), RECORD); + assertEquals(record.kind(), RECORD); assertEquals(record.field(0).baseType(), INT8); assertEquals(record.field(0).name(), $("x")); assertEquals(record.field(1).baseType(), ARRAY); assertEquals(record.field(1).name(), $("y")); Map map = Map(Int8, Array(Timestamp)); - assertEquals(map.baseType(), MAP); - assertEquals(map.keyType().baseType(), INT8); - assertEquals(map.valueType().baseType(), ARRAY); + assertEquals(map.kind(), MAP); + assertEquals(map.keyType().kind(), INT8); + assertEquals(map.valueType().kind(), ARRAY); Array array = Array(Int8); - assertEquals(array.baseType(), ARRAY); - assertEquals(array.elementType().baseType(), INT8); + assertEquals(array.kind(), ARRAY); + assertEquals(array.elementType().kind(), INT8); } @Test diff --git a/tajo-common/src/test/java/org/apache/tajo/type/TestTypeProtobufEncoder.java b/tajo-common/src/test/java/org/apache/tajo/type/TestTypeProtobufEncoder.java index 6eda557c3d..c2f6aa7a51 100644 --- a/tajo-common/src/test/java/org/apache/tajo/type/TestTypeProtobufEncoder.java +++ b/tajo-common/src/test/java/org/apache/tajo/type/TestTypeProtobufEncoder.java @@ -18,13 +18,13 @@ package org.apache.tajo.type; -import static org.apache.tajo.type.TypeProtobufEncoder.deserialize; -import static org.apache.tajo.type.TypeProtobufEncoder.serialize; +import static org.apache.tajo.type.TypeProtobufEncoder.decode; +import static org.apache.tajo.type.TypeProtobufEncoder.encode; import static org.junit.Assert.assertEquals; public class TestTypeProtobufEncoder extends TestTypeEncoder { public void assertSerialize(Type type) { - assertEquals(type, deserialize(serialize(type))); + assertEquals(type, decode(encode(type))); } } \ No newline at end of file diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/codegen/EvalCodeGenContext.java b/tajo-core/src/main/java/org/apache/tajo/engine/codegen/EvalCodeGenContext.java index 0dea1044cc..c485a941f5 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/codegen/EvalCodeGenContext.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/codegen/EvalCodeGenContext.java @@ -116,7 +116,7 @@ public void emitConstructor() { if (entry.getKey().getType() == EvalType.CONST) { ConstEval constEval = (ConstEval) entry.getKey(); - if (constEval.getValueType().baseType() == TajoDataTypes.Type.INTERVAL) { + if (constEval.getValueType().kind() == TajoDataTypes.Type.INTERVAL) { IntervalDatum datum = (IntervalDatum) constEval.getValue(); final String internalName = TajoGeneratorAdapter.getInternalName(IntervalDatum.class); diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/codegen/EvalCodeGenerator.java b/tajo-core/src/main/java/org/apache/tajo/engine/codegen/EvalCodeGenerator.java index 5ee6d82efa..981370b83b 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/codegen/EvalCodeGenerator.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/codegen/EvalCodeGenerator.java @@ -170,7 +170,7 @@ public EvalNode visitUnaryEval(EvalCodeGenContext context, UnaryEval unary, Stac context.emitNullityCheck(ifNull); SignedEval signed = (SignedEval) unary; - switch (signed.getValueType().baseType()) { + switch (signed.getValueType().kind()) { case BOOLEAN: case CHAR: case INT1: @@ -375,7 +375,7 @@ public EvalNode visitField(EvalCodeGenContext context, FieldEval field, Stack stack) { - switch (constEval.getValueType().baseType()) { + switch (constEval.getValueType().kind()) { case NULL_TYPE: if (stack.isEmpty()) { diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/codegen/TajoGeneratorAdapter.java b/tajo-core/src/main/java/org/apache/tajo/engine/codegen/TajoGeneratorAdapter.java index 8a958bbed7..d11701b144 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/codegen/TajoGeneratorAdapter.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/codegen/TajoGeneratorAdapter.java @@ -162,12 +162,12 @@ public TajoGeneratorAdapter(int access, MethodVisitor methodVisitor, String name } public static boolean isJVMInternalInt(org.apache.tajo.type.Type type) { - final TajoDataTypes.Type baseType = type.baseType(); + final TajoDataTypes.Type baseType = type.kind(); return baseType == BOOLEAN || baseType == INT1 || baseType == INT2 || baseType == INT4 || baseType== INET4; } public static int getWordSize(org.apache.tajo.type.Type type) { - final TajoDataTypes.Type baseType = type.baseType(); + final TajoDataTypes.Type baseType = type.kind(); if (baseType == INT8 || baseType == FLOAT8 || baseType == TIMESTAMP || baseType == TIME) { return 2; } else { @@ -249,7 +249,7 @@ public void ifCmp(org.apache.tajo.type.Type type, EvalType evalType, Label elseL } } else { - if (type.baseType() == TEXT) { + if (type.kind() == TEXT) { invokeVirtual(String.class, "compareTo", int.class, new Class[]{String.class}); } else { int opCode = TajoGeneratorAdapter.getOpCode(evalType, type); @@ -282,7 +282,7 @@ public void ifCmp(org.apache.tajo.type.Type type, EvalType evalType, Label elseL } public void load(org.apache.tajo.type.Type type, int idx) { - switch (type.baseType()) { + switch (type.kind()) { case NULL_TYPE: case BOOLEAN: case CHAR: @@ -401,7 +401,7 @@ public void emitNullityCheck(Label ifNull, int ... varIds) { } public void pushDummyValue(org.apache.tajo.type.Type type) { - TajoDataTypes.Type baseType = type.baseType(); + TajoDataTypes.Type baseType = type.kind(); if (type.isNull()) { pushNullOfThreeValuedLogic(); @@ -448,19 +448,19 @@ public void invokeInterface(Class owner, String methodName, Class returnType, Cl } public static boolean isPrimitiveOpCode(EvalType evalType, org.apache.tajo.type.Type returnType) { - return TUtil.containsInNestedMap(OpCodesMap, evalType, returnType.baseType()); + return TUtil.containsInNestedMap(OpCodesMap, evalType, returnType.kind()); } public static int getOpCode(EvalType evalType, org.apache.tajo.type.Type returnType) { if (!isPrimitiveOpCode(evalType, returnType)) { throw new CompilationError("No Such OpCode for " + evalType + " returning " + returnType); } - return TUtil.getFromNestedMap(OpCodesMap, evalType, returnType.baseType()); + return TUtil.getFromNestedMap(OpCodesMap, evalType, returnType.kind()); } public void castInsn(org.apache.tajo.type.Type srcType, org.apache.tajo.type.Type targetType) { - TajoDataTypes.Type srcBaseType = srcType.baseType(); - TajoDataTypes.Type targetBaseType = targetType.baseType(); + TajoDataTypes.Type srcBaseType = srcType.kind(); + TajoDataTypes.Type targetBaseType = targetType.kind(); switch(srcBaseType) { case BOOLEAN: case CHAR: { @@ -600,7 +600,7 @@ public void convertToPrimitive(org.apache.tajo.type.Type type) { methodvisitor.visitJumpInsn(Opcodes.IFEQ, ifNull); // datum aload(datum); - switch (type.baseType()) { + switch (type.kind()) { case BOOLEAN: case INT1: case INT2: @@ -643,7 +643,7 @@ public void convertToDatum(org.apache.tajo.type.Type type, boolean castToDatum) String convertMethod; Class returnType; Class [] paramTypes; - switch (type.baseType()) { + switch (type.kind()) { case NULL_TYPE: pop(); // pop null flag pop(type); // pop null datum @@ -895,7 +895,7 @@ public int store(org.apache.tajo.type.Type type) { int varId = nextVarId; nextVarId += TajoGeneratorAdapter.getWordSize(type); - switch (type.baseType()) { + switch (type.kind()) { case NULL_TYPE: case BOOLEAN: case CHAR: @@ -928,7 +928,7 @@ public int store(org.apache.tajo.type.Type type) { } public void emitBoxing(EvalCodeGenContext context, org.apache.tajo.type.Type type) { - switch (type.baseType()) { + switch (type.kind()) { case CHAR: case TEXT: @@ -954,7 +954,7 @@ public void emitBoxing(EvalCodeGenContext context, org.apache.tajo.type.Type typ } public void emitUnboxing(EvalCodeGenContext context, org.apache.tajo.type.Type type) { - switch (type.baseType()) { + switch (type.kind()) { case CHAR: case TEXT: diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/codegen/VariablesPreBuilder.java b/tajo-core/src/main/java/org/apache/tajo/engine/codegen/VariablesPreBuilder.java index 9d0f9d0026..bc2ccfcd18 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/codegen/VariablesPreBuilder.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/codegen/VariablesPreBuilder.java @@ -55,9 +55,9 @@ public EvalNode visitBinaryEval(EvalCodeGenContext context, Stack stac @Override public EvalNode visitConst(EvalCodeGenContext context, ConstEval constEval, Stack stack) { - if (constEval.getValueType().baseType() == TajoDataTypes.Type.INTERVAL) { + if (constEval.getValueType().kind() == TajoDataTypes.Type.INTERVAL) { if (!context.symbols.containsKey(constEval)) { - String fieldName = constEval.getValueType().baseType().name() + "_" + context.seqId++; + String fieldName = constEval.getValueType().kind().name() + "_" + context.seqId++; context.symbols.put(constEval, fieldName); context.classWriter.visitField(Opcodes.ACC_PRIVATE, fieldName, diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java index b064952df9..1ae71bd2d5 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/NonForwardQueryResultSystemScanner.java @@ -28,7 +28,6 @@ import org.apache.tajo.catalog.*; import org.apache.tajo.catalog.proto.CatalogProtos.*; import org.apache.tajo.catalog.statistics.TableStats; -import org.apache.tajo.common.TajoDataTypes.DataType; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.datum.DatumFactory; import org.apache.tajo.engine.codegen.CompilationError; @@ -40,6 +39,7 @@ import org.apache.tajo.engine.planner.global.MasterPlan; import org.apache.tajo.engine.planner.physical.PhysicalExec; import org.apache.tajo.engine.query.QueryContext; +import org.apache.tajo.exception.InvalidSessionException; import org.apache.tajo.exception.TajoException; import org.apache.tajo.exception.TajoInternalError; import org.apache.tajo.ipc.ClientProtos.SerializedResultSet; @@ -52,7 +52,6 @@ import org.apache.tajo.plan.logical.ScanNode; import org.apache.tajo.resource.NodeResource; import org.apache.tajo.resource.NodeResources; -import org.apache.tajo.exception.InvalidSessionException; import org.apache.tajo.schema.IdentifierUtil; import org.apache.tajo.storage.RowStoreUtil; import org.apache.tajo.storage.RowStoreUtil.RowStoreEncoder; @@ -60,6 +59,8 @@ import org.apache.tajo.storage.VTuple; import org.apache.tajo.tuple.memory.MemoryBlock; import org.apache.tajo.tuple.memory.MemoryRowBlock; +import org.apache.tajo.type.Type; +import org.apache.tajo.type.TypeProtobufEncoder; import org.apache.tajo.util.KeyValueSet; import org.apache.tajo.worker.TaskAttemptContext; @@ -286,11 +287,11 @@ private List getColumns(Schema outSchema) { } else if ("ordinal_position".equalsIgnoreCase(colObj.getSimpleName())) { aTuple.put(fieldId, DatumFactory.createInt4(columnId)); } else if ("data_type".equalsIgnoreCase(colObj.getSimpleName())) { - aTuple.put(fieldId, DatumFactory.createText(column.getDataType().getType().toString())); + aTuple.put(fieldId, DatumFactory.createText(column.getType().toString())); } else if ("type_length".equalsIgnoreCase(colObj.getSimpleName())) { - DataType dataType = column.getDataType(); - if (dataType.hasLength()) { - aTuple.put(fieldId, DatumFactory.createInt4(dataType.getLength())); + Type type = TypeProtobufEncoder.decode(column.getType()); + if (type.isValueParameterized()) { + aTuple.put(fieldId, DatumFactory.createInt4(type.getValueParameters().get(0))); } else { aTuple.put(fieldId, DatumFactory.createNullDatum()); } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/ExprAnnotator.java b/tajo-plan/src/main/java/org/apache/tajo/plan/ExprAnnotator.java index 0ba54dca24..c5ad51e259 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/ExprAnnotator.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/ExprAnnotator.java @@ -112,8 +112,8 @@ public static void assertEval(boolean condition, String message) throws TajoExce * @return a pair including left/right hand side terms */ private static Pair convertTypesIfNecessary(Context ctx, EvalNode lhs, EvalNode rhs) { - Type lhsType = lhs.getValueType().baseType(); - Type rhsType = rhs.getValueType().baseType(); + Type lhsType = lhs.getValueType().kind(); + Type rhsType = rhs.getValueType().kind(); // If one of both is NULL, it just returns the original types without casting. if (lhsType == Type.NULL_TYPE || rhsType == Type.NULL_TYPE) { @@ -439,10 +439,10 @@ public EvalNode visitConcatenate(Context ctx, Stack stack, BinaryOperator EvalNode rhs = visit(ctx, stack, expr.getRight()); stack.pop(); - if (lhs.getValueType().baseType() != Type.TEXT) { + if (lhs.getValueType().kind() != Type.TEXT) { lhs = convertType(ctx, lhs, CatalogUtil.newSimpleDataType(Type.TEXT)); } - if (rhs.getValueType().baseType() != Type.TEXT) { + if (rhs.getValueType().kind() != Type.TEXT) { rhs = convertType(ctx, rhs, CatalogUtil.newSimpleDataType(Type.TEXT)); } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/util/EvalNodeToExprConverter.java b/tajo-plan/src/main/java/org/apache/tajo/plan/util/EvalNodeToExprConverter.java index 86f82860e6..2ad6b6f103 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/util/EvalNodeToExprConverter.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/util/EvalNodeToExprConverter.java @@ -145,7 +145,7 @@ protected EvalNode visitBinaryEval(Object o, Stack stack, BinaryEval b @Override protected EvalNode visitConst(Object o, ConstEval evalNode, Stack stack) { - exprs.push(convertDatumToExpr(evalNode.getValueType().baseType(), evalNode.getValue())); + exprs.push(convertDatumToExpr(evalNode.getValueType().kind(), evalNode.getValue())); return super.visitConst(o, evalNode, stack); } diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/ExprsVerifier.java b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/ExprsVerifier.java index 2040ae3632..a0fecf2c44 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/ExprsVerifier.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/ExprsVerifier.java @@ -146,8 +146,8 @@ private static void checkArithmeticOperand(VerificationState state, BinaryEval e org.apache.tajo.type.Type leftDataType = leftExpr.getValueType(); org.apache.tajo.type.Type rightDataType = rightExpr.getValueType(); - TajoDataTypes.Type leftType = leftDataType.baseType(); - TajoDataTypes.Type rightType = rightDataType.baseType(); + TajoDataTypes.Type leftType = leftDataType.kind(); + TajoDataTypes.Type rightType = rightDataType.kind(); if (leftType == DATE && (checkIntType(rightDataType) || @@ -178,26 +178,26 @@ private static void checkArithmeticOperand(VerificationState state, BinaryEval e } private static boolean checkNetworkType(org.apache.tajo.type.Type dataType) { - return dataType.baseType() == INET4 || dataType.baseType() == INET6; + return dataType.kind() == INET4 || dataType.kind() == INET6; } private static boolean checkIntType(org.apache.tajo.type.Type dataType) { - int typeNumber = dataType.baseType().getNumber(); + int typeNumber = dataType.kind().getNumber(); return INT1.getNumber() < typeNumber && typeNumber <= INT8.getNumber(); } private static boolean checkNumericType(org.apache.tajo.type.Type dataType) { - int typeNumber = dataType.baseType().getNumber(); + int typeNumber = dataType.kind().getNumber(); return INT1.getNumber() <= typeNumber && typeNumber <= NUMERIC.getNumber(); } private static boolean checkTextData(org.apache.tajo.type.Type dataType) { - int typeNumber = dataType.baseType().getNumber(); + int typeNumber = dataType.kind().getNumber(); return CHAR.getNumber() <= typeNumber && typeNumber <= TEXT.getNumber(); } private static boolean checkDateTime(org.apache.tajo.type.Type dataType) { - int typeNumber = dataType.baseType().getNumber(); + int typeNumber = dataType.kind().getNumber(); return (DATE.getNumber() <= typeNumber && typeNumber <= INTERVAL.getNumber()) || (TIMEZ.getNumber() <= typeNumber && typeNumber <= TIMESTAMPZ.getNumber()); } diff --git a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/SQLExpressionGenerator.java b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/SQLExpressionGenerator.java index cd5f2926fc..da97e32e92 100644 --- a/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/SQLExpressionGenerator.java +++ b/tajo-storage/tajo-storage-jdbc/src/main/java/org/apache/tajo/storage/jdbc/SQLExpressionGenerator.java @@ -271,7 +271,7 @@ public String convertDatumToSQLLiteral(Datum d) { * @return SQL DataType */ public String convertTajoTypeToSQLType(Type dataType) { - switch (dataType.baseType()) { + switch (dataType.kind()) { case INT1: return "TINYINT"; case INT2: @@ -285,7 +285,7 @@ public String convertTajoTypeToSQLType(Type dataType) { case FLOAT8: return "DOUBLE"; default: - return dataType.baseType().name(); + return dataType.kind().name(); } } From 1dec9b839a95fae8d9d764c9f220d2a9f1d51244 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Tue, 10 May 2016 16:35:19 -0700 Subject: [PATCH 19/23] Fixed all unit tests. --- .../java/org/apache/tajo/algebra/DataTypeExpr.java | 10 ++++++++++ .../main/java/org/apache/tajo/type/TypeFactory.java | 13 ++++++++----- .../org/apache/tajo/type/ValueParamterizedType.java | 5 ----- .../apache/tajo/engine/query/TestCreateTable.java | 8 ++++---- .../queries/TestCreateTable/testArrayType1.sql | 1 - .../queries/TestCreateTable/testComplexType1.sql | 6 ++++++ .../java/org/apache/tajo/plan/LogicalPlanner.java | 2 ++ 7 files changed, 30 insertions(+), 15 deletions(-) delete mode 100644 tajo-core-tests/src/test/resources/queries/TestCreateTable/testArrayType1.sql create mode 100644 tajo-core-tests/src/test/resources/queries/TestCreateTable/testComplexType1.sql diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/DataTypeExpr.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/DataTypeExpr.java index 0c98640d2a..9b14278a3f 100644 --- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/DataTypeExpr.java +++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/DataTypeExpr.java @@ -88,6 +88,16 @@ public DataTypeExpr getElementType() { return arrayType.type; } + public DataTypeExpr getKeyType() { + Preconditions.checkState(isMapType()); + return mapType.keyType; + } + + public DataTypeExpr getValueType() { + Preconditions.checkState(isMapType()); + return mapType.valueType; + } + public ColumnDefinition [] getNestedRecordTypes() { return recordType.schema; } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/TypeFactory.java b/tajo-common/src/main/java/org/apache/tajo/type/TypeFactory.java index b9c04bef6c..e2cf45b00f 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/TypeFactory.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/TypeFactory.java @@ -30,6 +30,7 @@ import java.util.List; import static org.apache.tajo.type.Type.Char; +import static org.apache.tajo.type.Type.Null; import static org.apache.tajo.type.Type.Varchar; public class TypeFactory { @@ -162,17 +163,19 @@ public static Type create(TajoDataTypes.Type baseType, "Array Type requires 1 type parameters, but it takes (%s).", StringUtils.join(typeParams)); return Type.Array(typeParams.get(0)); } + case RECORD: { + Assert.assertCondition(fieldParams.size() >= 1, + "Record Type requires at least 1 field parameters, but it takes (%s).", StringUtils.join(fieldParams)); + return Type.Record(fieldParams); + } case MAP: { Assert.assertCondition(typeParams.size() == 2, "Map Type requires 2 type parameters, but it takes (%s).", StringUtils.join(typeParams)); return Type.Map(typeParams.get(0), typeParams.get(1)); } - case RECORD: { - Assert.assertCondition(fieldParams.size() >= 1, - "Record Type requires at least 1 field parameters, but it takes (%s).", StringUtils.join(fieldParams)); - return Type.Record(fieldParams); - } + case NULL_TYPE: + return Null(); default: throw new TajoInternalError(new UnsupportedException(baseType.name())); diff --git a/tajo-common/src/main/java/org/apache/tajo/type/ValueParamterizedType.java b/tajo-common/src/main/java/org/apache/tajo/type/ValueParamterizedType.java index 30986d3dba..09bfc94214 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/ValueParamterizedType.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/ValueParamterizedType.java @@ -77,9 +77,4 @@ public String toString() { sb.append(")"); return sb.toString(); } - - @Override - public TypeProto getProto() { - return TypeProto.newBuilder().addElements(TypeElement.newBuilder().addAllValueParams(params)).build(); - } } diff --git a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java index d733abbafe..3aee9bff8e 100644 --- a/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java +++ b/tajo-core-tests/src/test/java/org/apache/tajo/engine/query/TestCreateTable.java @@ -681,15 +681,15 @@ public final void testSelfDescTable2() throws Exception { } @Test - public final void testArrayType1() throws Exception { + public final void testComplexType1() throws Exception { try { executeString("CREATE DATABASE D11;").close(); - assertTableNotExists("d11.array_table1"); + assertTableNotExists("d11.complex_type1"); executeQuery().close(); - assertTableExists("d11.array_table1"); + assertTableExists("d11.complex_type1"); } finally { - executeString("DROP TABLE IF EXISTS D11.array_table1"); + executeString("DROP TABLE IF EXISTS D11.complex_type1"); executeString("DROP DATABASE IF EXISTS D11").close(); } } diff --git a/tajo-core-tests/src/test/resources/queries/TestCreateTable/testArrayType1.sql b/tajo-core-tests/src/test/resources/queries/TestCreateTable/testArrayType1.sql deleted file mode 100644 index 30ee94c01e..0000000000 --- a/tajo-core-tests/src/test/resources/queries/TestCreateTable/testArrayType1.sql +++ /dev/null @@ -1 +0,0 @@ -CREATE TABLE d11.array_table1 (vec array); \ No newline at end of file diff --git a/tajo-core-tests/src/test/resources/queries/TestCreateTable/testComplexType1.sql b/tajo-core-tests/src/test/resources/queries/TestCreateTable/testComplexType1.sql new file mode 100644 index 0000000000..0bba96e253 --- /dev/null +++ b/tajo-core-tests/src/test/resources/queries/TestCreateTable/testComplexType1.sql @@ -0,0 +1,6 @@ +CREATE TABLE d11.complex_type1 ( + col1 array, + col2 array, + col3 array>, + col4 array> +); \ No newline at end of file diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java index fcee1a39ba..69ae667209 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalPlanner.java @@ -2124,6 +2124,8 @@ public static Type convertDataType(DataTypeExpr dataType) { return Array(convertDataType(dataType.getElementType())); } else if (type == TajoDataTypes.Type.RECORD) { return Record(convertTableElementsSchema(dataType.getNestedRecordTypes())); + } else if (type == TajoDataTypes.Type.MAP) { + return Map(convertDataType(dataType.getKeyType()), convertDataType(dataType.getValueType())); } else if (type == TajoDataTypes.Type.NUMERIC) { if (dataType.hasLengthOrPrecision() && dataType.hasScale()) { return Numeric(dataType.getLengthOrPrecision(), dataType.getScale()); From 7091b40e8589515d6f0a0453a250ee5a3303832d Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Tue, 10 May 2016 18:13:28 -0700 Subject: [PATCH 20/23] Removed SchemaConverter. --- .../org/apache/tajo/algebra/DataTypeExpr.java | 4 +- .../apache/tajo/catalog/SchemaConverter.java | 41 ------------------- 2 files changed, 2 insertions(+), 43 deletions(-) delete mode 100644 tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaConverter.java diff --git a/tajo-algebra/src/main/java/org/apache/tajo/algebra/DataTypeExpr.java b/tajo-algebra/src/main/java/org/apache/tajo/algebra/DataTypeExpr.java index 9b14278a3f..f5eb8bd948 100644 --- a/tajo-algebra/src/main/java/org/apache/tajo/algebra/DataTypeExpr.java +++ b/tajo-algebra/src/main/java/org/apache/tajo/algebra/DataTypeExpr.java @@ -33,7 +33,7 @@ public class DataTypeExpr extends Expr { @Expose @SerializedName("Scale") Integer scale; @Expose @SerializedName("Array") - ArrayType arrayType; // not null if the type is RECORD + ArrayType arrayType; // not null if the type is ARRAY @Expose @SerializedName("Record") RecordType recordType; // not null if the type is RECORD @Expose @SerializedName("Map") @@ -68,7 +68,7 @@ public String getTypeName() { } public boolean isPrimitiveType() { - return !this.isRecordType() && !isMapType(); + return !isArrayType()&& !isRecordType() && !isMapType(); } public boolean isArrayType() { diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaConverter.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaConverter.java deleted file mode 100644 index 9fd7c0ea08..0000000000 --- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/SchemaConverter.java +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.catalog; - -import com.google.common.base.Function; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Iterables; -import org.apache.tajo.schema.Field; -import org.apache.tajo.schema.Schema; - -import javax.annotation.Nullable; - -public class SchemaConverter { - public static org.apache.tajo.catalog.Schema convert(Schema schema) { - ImmutableList columns = ImmutableList.copyOf( - Iterables.transform(schema, new Function() { - @Override - public Column apply(@Nullable Field namedType) { - return FieldConverter.convert(namedType); - } - }) - ); - return new SchemaLegacy(columns); - } -} From 8c948462655233891895dacbc9f20bbd2fac92a3 Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Tue, 10 May 2016 19:13:06 -0700 Subject: [PATCH 21/23] Renames *Encoder and clean up. --- .../tajo/catalog/store/HiveCatalogUtil.java | 2 +- .../tajo/catalog/store/AbstractDBStore.java | 36 ++++++++++--------- .../org/apache/tajo/TajoTestingCluster.java | 2 -- .../org/apache/tajo/json/TypeAdapter.java | 4 +-- .../main/java/org/apache/tajo/type/Map.java | 3 -- .../apache/tajo/type/TypeStringEncoder.java | 16 ++++----- .../tajo/type/ValueParamterizedType.java | 1 - .../org/apache/tajo/type/TestTypeEncoder.java | 2 +- .../tajo/type/TestTypeStringEncoder.java | 6 ++-- .../engine/codegen/EvalCodeGenContext.java | 2 +- .../master/exec/prehook/CreateTableHook.java | 1 - .../org/apache/tajo/plan/ExprAnnotator.java | 1 - 12 files changed, 35 insertions(+), 41 deletions(-) diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogUtil.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogUtil.java index 687437e1a5..faefd2808e 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogUtil.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogUtil.java @@ -106,7 +106,7 @@ public static String getHiveFieldType(Type type) throws LMDNoMatchedDatatypeExce case DATE: return serdeConstants.DATE_TYPE_NAME; case TIMESTAMP: return serdeConstants.TIMESTAMP_TYPE_NAME; default: - throw new LMDNoMatchedDatatypeException(TypeStringEncoder.serialize(type)); + throw new LMDNoMatchedDatatypeException(TypeStringEncoder.encode(type)); } } diff --git a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java index 5317c74940..2e2db6f9f9 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java +++ b/tajo-catalog/tajo-catalog-server/src/main/java/org/apache/tajo/catalog/store/AbstractDBStore.java @@ -51,6 +51,8 @@ import static org.apache.tajo.catalog.proto.CatalogProtos.AlterTablespaceProto.AlterTablespaceCommand; import static org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.KeyValueProto; import static org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.KeyValueSetProto; +import static org.apache.tajo.schema.IdentifierUtil.extractQualifier; +import static org.apache.tajo.schema.IdentifierUtil.extractSimpleName; public abstract class AbstractDBStore extends CatalogConstants implements CatalogStore { protected final Log LOG = LogFactory.getLog(getClass()); @@ -827,9 +829,9 @@ public void createTable(final CatalogProtos.TableDescProto table) org.apache.tajo.type.Type type = TypeProtobufEncoder.decode(col.getType()); pstmt.setInt(1, tableId); - pstmt.setString(2, IdentifierUtil.extractSimpleName(col.getName())); + pstmt.setString(2, extractSimpleName(col.getName())); pstmt.setInt(3, i); - pstmt.setString(4, TypeStringEncoder.serialize(type)); + pstmt.setString(4, TypeStringEncoder.encode(type)); pstmt.addBatch(); pstmt.clearParameters(); } @@ -992,7 +994,7 @@ public void alterTable(CatalogProtos.AlterTableDescProto alterTableDescProto) switch (alterTableDescProto.getAlterTableType()) { case RENAME_TABLE: - String simpleNewTableName = IdentifierUtil.extractSimpleName(alterTableDescProto.getNewTableName()); + String simpleNewTableName = extractSimpleName(alterTableDescProto.getNewTableName()); if (existTable(databaseName, simpleNewTableName)) { throw new DuplicateTableException(alterTableDescProto.getNewTableName()); } @@ -1171,13 +1173,13 @@ private void renameColumn(final int tableId, final CatalogProtos.AlterColumnProt conn = getConnection(); conn.setAutoCommit(false); - String tableName = IdentifierUtil.extractQualifier(alterColumnProto.getOldColumnName()); - String simpleOldColumnName = IdentifierUtil.extractSimpleName(alterColumnProto.getOldColumnName()); - String simpleNewColumnName = IdentifierUtil.extractSimpleName(alterColumnProto.getNewColumnName()); + String tableName = extractQualifier(alterColumnProto.getOldColumnName()); + String simpleOldColumnName = extractSimpleName(alterColumnProto.getOldColumnName()); + String simpleNewColumnName = extractSimpleName(alterColumnProto.getNewColumnName()); - if (!tableName.equals(IdentifierUtil.extractQualifier(alterColumnProto.getNewColumnName()))) { + if (!tableName.equals(extractQualifier(alterColumnProto.getNewColumnName()))) { throw new AmbiguousTableException( - tableName + ", " + IdentifierUtil.extractQualifier(alterColumnProto.getNewColumnName())); + tableName + ", " + extractQualifier(alterColumnProto.getNewColumnName())); } //SELECT COLUMN @@ -1187,7 +1189,7 @@ private void renameColumn(final int tableId, final CatalogProtos.AlterColumnProt resultSet = pstmt.executeQuery(); CatalogProtos.ColumnProto columnProto = null; - String typeStr = null; + String typeStr; int ordinalPosition = 0; if (resultSet.next()) { @@ -1195,7 +1197,7 @@ private void renameColumn(final int tableId, final CatalogProtos.AlterColumnProt //NOTE ==> Setting new column Name columnProto = columnProto.toBuilder().setName(alterColumnProto.getNewColumnName()).build(); ordinalPosition = resultSet.getInt("ORDINAL_POSITION"); - typeStr = TypeStringEncoder.serialize(TypeProtobufEncoder.decode(columnProto.getType())); + typeStr = TypeStringEncoder.encode(TypeProtobufEncoder.decode(columnProto.getType())); } else { throw new UndefinedColumnException(alterColumnProto.getOldColumnName()); } @@ -1248,7 +1250,7 @@ private void addNewColumn(int tableId, CatalogProtos.ColumnProto columnProto) th conn = getConnection(); pstmt = conn.prepareStatement(existColumnSql); pstmt.setInt(1, tableId); - pstmt.setString(2, IdentifierUtil.extractSimpleName(columnProto.getName())); + pstmt.setString(2, extractSimpleName(columnProto.getName())); resultSet = pstmt.executeQuery(); if (resultSet.next()) { @@ -1272,9 +1274,9 @@ private void addNewColumn(int tableId, CatalogProtos.ColumnProto columnProto) th pstmt = conn.prepareStatement(insertNewColumnSql); pstmt.setInt(1, tableId); - pstmt.setString(2, IdentifierUtil.extractSimpleName(columnProto.getName())); + pstmt.setString(2, extractSimpleName(columnProto.getName())); pstmt.setInt(3, position + 1); - pstmt.setString(4, TypeStringEncoder.serialize(type)); + pstmt.setString(4, TypeStringEncoder.encode(type)); pstmt.executeUpdate(); } catch (SQLException sqlException) { @@ -1856,7 +1858,7 @@ public List getAllColumns() { String tableName = getTableName(conn, tid); builder.setTid(tid); builder.setName(IdentifierUtil.buildFQName(databaseName, tableName, resultSet.getString("COLUMN_NAME"))); - org.apache.tajo.type.Type type = TypeStringEncoder.deserialize(resultSet.getString("DATA_TYPE").trim()); + org.apache.tajo.type.Type type = TypeStringEncoder.decode(resultSet.getString("DATA_TYPE").trim()); builder.setType(type.getProto()); columns.add(builder.build()); } @@ -2467,7 +2469,7 @@ public void createIndex(final IndexDescProto proto) PreparedStatement pstmt = null; final String databaseName = proto.getTableIdentifier().getDatabaseName(); - final String tableName = IdentifierUtil.extractSimpleName(proto.getTableIdentifier().getTableName()); + final String tableName = extractSimpleName(proto.getTableIdentifier().getTableName()); try { @@ -2887,7 +2889,7 @@ private void resultToIndexDescProtoBuilder(final String qualifier, for (int i = 0; i < columnNum; i++) { SortSpecProto.Builder colSpecBuilder = SortSpecProto.newBuilder(); colSpecBuilder.setColumn(ColumnProto.newBuilder().setName(IdentifierUtil.buildFQName(qualifier, columnNames[i])) - .setType(TypeStringEncoder.deserialize(dataTypes[i]).getProto()).build()); + .setType(TypeStringEncoder.decode(dataTypes[i]).getProto()).build()); colSpecBuilder.setAscending(orders[i].equals("true")); colSpecBuilder.setNullFirst(nullOrders[i].equals("true")); builder.addKeySortSpecs(colSpecBuilder.build()); @@ -2899,7 +2901,7 @@ private void resultToIndexDescProtoBuilder(final String qualifier, private ColumnProto resultToColumnProto(final ResultSet res) throws SQLException { ColumnProto.Builder builder = ColumnProto.newBuilder(); builder.setName(res.getString("column_name").trim()); - org.apache.tajo.type.Type type = TypeStringEncoder.deserialize(res.getString("data_type").trim()); + org.apache.tajo.type.Type type = TypeStringEncoder.decode(res.getString("data_type").trim()); builder.setType(type.getProto()); return builder.build(); } diff --git a/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java b/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java index 6c526c5019..4b89c8e274 100644 --- a/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java +++ b/tajo-cluster-tests/src/test/java/org/apache/tajo/TajoTestingCluster.java @@ -49,7 +49,6 @@ import org.apache.tajo.service.ServiceTrackerFactory; import org.apache.tajo.storage.FileTablespace; import org.apache.tajo.storage.TablespaceManager; -import org.apache.tajo.util.KeyValueSet; import org.apache.tajo.util.NetUtils; import org.apache.tajo.util.Pair; import org.apache.tajo.util.history.QueryHistory; @@ -64,7 +63,6 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.List; -import java.util.TimeZone; import java.util.UUID; public class TajoTestingCluster { diff --git a/tajo-common/src/main/java/org/apache/tajo/json/TypeAdapter.java b/tajo-common/src/main/java/org/apache/tajo/json/TypeAdapter.java index ddd88a9742..5c3c1e4be3 100644 --- a/tajo-common/src/main/java/org/apache/tajo/json/TypeAdapter.java +++ b/tajo-common/src/main/java/org/apache/tajo/json/TypeAdapter.java @@ -30,13 +30,13 @@ public class TypeAdapter implements GsonSerDerAdapter public org.apache.tajo.type.Type deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context) throws JsonParseException { JsonObject obj = (JsonObject) json; - return TypeStringEncoder.deserialize(CommonGsonHelper.getOrDie(obj, "type").getAsString()); + return TypeStringEncoder.decode(CommonGsonHelper.getOrDie(obj, "type").getAsString()); } @Override public JsonElement serialize(org.apache.tajo.type.Type src, Type typeOfSrc, JsonSerializationContext context) { JsonObject json = new JsonObject(); - json.addProperty("type", TypeStringEncoder.serialize(src)); + json.addProperty("type", TypeStringEncoder.encode(src)); return json; } } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/Map.java b/tajo-common/src/main/java/org/apache/tajo/type/Map.java index 07e6c7da27..786d6af4ff 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/Map.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/Map.java @@ -19,9 +19,6 @@ package org.apache.tajo.type; import com.google.common.collect.ImmutableList; -import org.apache.tajo.common.TajoDataTypes; - -import java.util.Objects; import static org.apache.tajo.common.TajoDataTypes.Type.MAP; diff --git a/tajo-common/src/main/java/org/apache/tajo/type/TypeStringEncoder.java b/tajo-common/src/main/java/org/apache/tajo/type/TypeStringEncoder.java index 16dcd7ef71..b6e1b18950 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/TypeStringEncoder.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/TypeStringEncoder.java @@ -41,11 +41,11 @@ public class TypeStringEncoder { /** - * Serialize a type into a string representation + * Encode a type into a string representation * @param type A type * @return A type string representation */ - public static String serialize(Type type) { + public static String encode(Type type) { StringBuilder sb = new StringBuilder(type.kind().name()); if (type.isTypeParameterized()) { @@ -53,7 +53,7 @@ public static String serialize(Type type) { sb.append(StringUtils.join(type.getTypeParameters(), ",", new Function() { @Override public String apply(@Nullable Type type) { - return TypeStringEncoder.serialize(type); + return TypeStringEncoder.encode(type); } })); sb.append(">"); @@ -87,15 +87,15 @@ public String apply(@Nullable Field field) { * @return String representation for a field */ static String serializeField(Field field) { - return field.name().raw(DefaultPolicy()) + " " + serialize(field.type()); + return field.name().raw(DefaultPolicy()) + " " + encode(field.type()); } /** - * Parse and transform a type string representation to a Type. + * Decode a string representation to a Type. * @param signature Type string representation * @return Type */ - public static Type deserialize(String signature) { + public static Type decode(String signature) { // termination condition in this recursion if (!(signature.contains("<") || signature.contains("(") || signature.contains("["))) { @@ -128,7 +128,7 @@ public static Type deserialize(String signature) { parseList(signature.substring(paramStartIdx, i), new Function() { @Override public Type apply(@Nullable String s) { - return deserialize(s); + return decode(s); } }), ImmutableList.of(), @@ -270,7 +270,7 @@ static Field parseField(String str) { QualifiedIdentifier identifier = IdentifierUtil.makeIdentifier(str.substring(paramStartIdx, i), DefaultPolicy()); String typePart = str.substring(i + 1, str.length()); - return new Field(identifier, deserialize(typePart)); + return new Field(identifier, decode(typePart)); } } } diff --git a/tajo-common/src/main/java/org/apache/tajo/type/ValueParamterizedType.java b/tajo-common/src/main/java/org/apache/tajo/type/ValueParamterizedType.java index 09bfc94214..476629ee0c 100644 --- a/tajo-common/src/main/java/org/apache/tajo/type/ValueParamterizedType.java +++ b/tajo-common/src/main/java/org/apache/tajo/type/ValueParamterizedType.java @@ -21,7 +21,6 @@ import com.google.common.collect.ImmutableList; import org.apache.tajo.common.ProtoObject; import org.apache.tajo.common.TajoDataTypes; -import org.apache.tajo.common.TajoDataTypes.TypeElement; import org.apache.tajo.common.TajoDataTypes.TypeProto; import org.apache.tajo.util.StringUtils; diff --git a/tajo-common/src/test/java/org/apache/tajo/type/TestTypeEncoder.java b/tajo-common/src/test/java/org/apache/tajo/type/TestTypeEncoder.java index 4a0eea98ea..abd427a3ff 100644 --- a/tajo-common/src/test/java/org/apache/tajo/type/TestTypeEncoder.java +++ b/tajo-common/src/test/java/org/apache/tajo/type/TestTypeEncoder.java @@ -112,7 +112,7 @@ public final void testParseTypeList() { List types = parseList("TEXT,ARRAY", new Function() { @Override public Type apply(@Nullable String s) { - return deserialize(s); + return decode(s); } }); assertEquals(2, types.size()); diff --git a/tajo-common/src/test/java/org/apache/tajo/type/TestTypeStringEncoder.java b/tajo-common/src/test/java/org/apache/tajo/type/TestTypeStringEncoder.java index a7a57894bd..56db01be40 100644 --- a/tajo-common/src/test/java/org/apache/tajo/type/TestTypeStringEncoder.java +++ b/tajo-common/src/test/java/org/apache/tajo/type/TestTypeStringEncoder.java @@ -18,12 +18,12 @@ package org.apache.tajo.type; -import static org.apache.tajo.type.TypeStringEncoder.deserialize; -import static org.apache.tajo.type.TypeStringEncoder.serialize; +import static org.apache.tajo.type.TypeStringEncoder.decode; +import static org.apache.tajo.type.TypeStringEncoder.encode; import static org.junit.Assert.assertEquals; public class TestTypeStringEncoder extends TestTypeEncoder { public void assertSerialize(Type type) { - assertEquals(type, deserialize(serialize(type))); + assertEquals(type, decode(encode(type))); } } \ No newline at end of file diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/codegen/EvalCodeGenContext.java b/tajo-core/src/main/java/org/apache/tajo/engine/codegen/EvalCodeGenContext.java index c485a941f5..c2840c5ec1 100644 --- a/tajo-core/src/main/java/org/apache/tajo/engine/codegen/EvalCodeGenContext.java +++ b/tajo-core/src/main/java/org/apache/tajo/engine/codegen/EvalCodeGenContext.java @@ -141,7 +141,7 @@ public void emitConstructor() { initMethod.visitTypeInsn(Opcodes.NEW, internalName); consAdapter.dup(); emitCreateEval(consAdapter, initMethod, inEval.getLeftExpr()); - emitRowConstantEval(consAdapter, initMethod, (RowConstantEval) inEval.getRightExpr()); + emitRowConstantEval(consAdapter, initMethod, inEval.getRightExpr()); consAdapter.push(inEval.isNot()); consAdapter.invokeSpecial(InEval.class, "", void.class, new Class [] {EvalNode.class, RowConstantEval.class, boolean.class}); diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/prehook/CreateTableHook.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/prehook/CreateTableHook.java index fa2b5dfd36..e6fee07536 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/prehook/CreateTableHook.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/prehook/CreateTableHook.java @@ -38,7 +38,6 @@ public void hook(QueryContext queryContext, LogicalPlan plan) throws Exception { LogicalRootNode rootNode = plan.getRootBlock().getRoot(); CreateTableNode createTableNode = rootNode.getChild(); String [] splitted = IdentifierUtil.splitFQTableName(createTableNode.getTableName()); - String databaseName = splitted[0]; String tableName = splitted[1]; queryContext.setOutputTable(tableName); diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/ExprAnnotator.java b/tajo-plan/src/main/java/org/apache/tajo/plan/ExprAnnotator.java index cd38a04297..92ba368cc6 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/ExprAnnotator.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/ExprAnnotator.java @@ -26,7 +26,6 @@ import org.apache.tajo.catalog.CatalogUtil; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.FunctionDesc; -import org.apache.tajo.catalog.*; import org.apache.tajo.catalog.CatalogUtil.Direction; import org.apache.tajo.common.TajoDataTypes; import org.apache.tajo.datum.*; From 54b639a010079c5a3aa6bcb85f4e24c062d3b75d Mon Sep 17 00:00:00 2001 From: Hyunsik Choi Date: Tue, 10 May 2016 21:17:27 -0700 Subject: [PATCH 22/23] Updated schema history. --- .../src/main/resources/schemas/derby/derby.xml | 1 + .../src/main/resources/schemas/mariadb/mariadb.xml | 1 + .../src/main/resources/schemas/mysql/mysql.xml | 1 + .../src/main/resources/schemas/oracle/oracle.xml | 1 + .../src/main/resources/schemas/postgresql/postgresql.xml | 1 + 5 files changed, 5 insertions(+) diff --git a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml index d7ef9e9cc8..70ef436bec 100644 --- a/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml +++ b/tajo-catalog/tajo-catalog-server/src/main/resources/schemas/derby/derby.xml @@ -19,6 +19,7 @@