diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java index 308cda53d7..81fbd3f775 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java @@ -564,6 +564,16 @@ public final void createTable(final CatalogProtos.TableDescProto tableDescProto) table.putToParameters(ParquetOutputFormat.COMPRESSION, tableDesc.getMeta().getProperty(ParquetOutputFormat.COMPRESSION)); } + } else if (tableDesc.getMeta().getDataFormat().equalsIgnoreCase(BuiltinStorages.ORC)) { + StorageFormatDescriptor descriptor = storageFormatFactory.get(IOConstants.ORC); + sd.setInputFormat(descriptor.getInputFormat()); + sd.setOutputFormat(descriptor.getOutputFormat()); + sd.getSerdeInfo().setSerializationLib(descriptor.getSerde()); + + if (tableDesc.getMeta().containsProperty(StorageConstants.ORC_COMPRESSION)) { + table.putToParameters(StorageConstants.ORC_COMPRESSION, + tableDesc.getMeta().getProperty(StorageConstants.ORC_COMPRESSION)); + } } else { throw new UnsupportedException(tableDesc.getMeta().getDataFormat() + " in HivecatalogStore"); } diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogUtil.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogUtil.java index bbb7adeee3..c15fa9a11c 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogUtil.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogUtil.java @@ -22,6 +22,7 @@ import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.ql.io.RCFileInputFormat; +import org.apache.hadoop.hive.ql.io.orc.OrcSerde; import org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.serde.serdeConstants; @@ -135,6 +136,8 @@ public static String getDataFormat(StorageDescriptor descriptor) { } } else if (ParquetHiveSerDe.class.getName().equals(serde)) { return BuiltinStorages.PARQUET; + } else if (OrcSerde.class.getName().equals(serde)) { + return BuiltinStorages.ORC; } else if (AvroSerDe.class.getName().equals(serde)) { return BuiltinStorages.AVRO; } else { diff --git a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java index 7e1a3a4ff6..aed9525279 100644 --- a/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java +++ b/tajo-catalog/tajo-catalog-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java @@ -649,6 +649,41 @@ public void testTableUsingParquet() throws Exception { store.dropTable(DB_NAME, CUSTOMER); } + @Test + public void testTableUsingORC() throws Exception { + TableMeta meta = new TableMeta("ORC", new KeyValueSet()); + + org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema(); + schema.addColumn("c_custkey", TajoDataTypes.Type.INT4); + schema.addColumn("c_name", TajoDataTypes.Type.TEXT); + schema.addColumn("c_address", TajoDataTypes.Type.TEXT); + schema.addColumn("c_nationkey", TajoDataTypes.Type.INT4); + schema.addColumn("c_phone", TajoDataTypes.Type.TEXT); + schema.addColumn("c_acctbal", TajoDataTypes.Type.FLOAT8); + schema.addColumn("c_mktsegment", TajoDataTypes.Type.TEXT); + schema.addColumn("c_comment", TajoDataTypes.Type.TEXT); + + TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, CUSTOMER), schema, meta, + new Path(warehousePath, new Path(DB_NAME, CUSTOMER)).toUri()); + store.createTable(table.getProto()); + assertTrue(store.existTable(DB_NAME, CUSTOMER)); + + StorageFormatDescriptor descriptor = formatFactory.get(IOConstants.ORC); + org.apache.hadoop.hive.ql.metadata.Table hiveTable = store.getHiveTable(DB_NAME, CUSTOMER); + assertEquals(descriptor.getInputFormat(), hiveTable.getSd().getInputFormat()); + assertEquals(descriptor.getOutputFormat(), hiveTable.getSd().getOutputFormat()); + + TableDesc table1 = new TableDesc(store.getTable(DB_NAME, CUSTOMER)); + assertEquals(table.getName(), table1.getName()); + assertEquals(table.getUri(), table1.getUri()); + assertEquals(table.getSchema().size(), table1.getSchema().size()); + for (int i = 0; i < table.getSchema().size(); i++) { + assertEquals(table.getSchema().getColumn(i).getSimpleName(), table1.getSchema().getColumn(i).getSimpleName()); + } + + store.dropTable(DB_NAME, CUSTOMER); + } + @Test public void testDataTypeCompatibility() throws Exception { String tableName = CatalogUtil.normalizeIdentifier("testDataTypeCompatibility"); diff --git a/tajo-dist/pom.xml b/tajo-dist/pom.xml index 095f128809..121622d277 100644 --- a/tajo-dist/pom.xml +++ b/tajo-dist/pom.xml @@ -154,9 +154,6 @@ run cp -r ${project.basedir}/src/main/conf . run rm -rf lib/tajo-*-${project.version}.jar - run mkdir hive - run mv lib/hive-*.jar hive/ - run mkdir -p share/jdbc-dist run cp -r $ROOT/tajo-jdbc/target/tajo-jdbc-${project.version}-jar-with-dependencies.jar ./share/jdbc-dist/tajo-jdbc-${project.version}.jar diff --git a/tajo-dist/src/main/bin/tajo b/tajo-dist/src/main/bin/tajo index c08c538201..744d66814e 100755 --- a/tajo-dist/src/main/bin/tajo +++ b/tajo-dist/src/main/bin/tajo @@ -303,10 +303,6 @@ if [ ! -z ${HIVE_HOME} ] && [ -d ${HIVE_HOME} ] && [ -d ${HIVE_LIB} ]; then for f in ${HIVE_LIB}/datanucleus-*.jar; do CLASSPATH=${CLASSPATH}:$f; done -else - for f in $TAJO_HOME/hive/*.jar; do - CLASSPATH=${CLASSPATH}:$f; - done fi if [ "${HIVE_JDBC_DRIVER_DIR}" != "" ]; then