Skip to content
This repository was archived by the owner on May 12, 2021. It is now read-only.

TAJO-1929: Can't read ORC file on hive-meta #974

Closed
eminency wants to merge 5 commits into from
Closed

TAJO-1929: Can't read ORC file on hive-meta #974

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
3938810
test is added
eminency Jan 8, 2016
715576d
Missing parts
eminency Jan 8, 2016
11aaf44
Merge branch 'master' into hive_catalog_orc
eminency Mar 9, 2016
5db03c3
remove useless code
eminency Mar 9, 2016
f12616a
Merge branch 'master' into hive_catalog_orc
eminency Mar 9, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions ...talog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogStore.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -564,6 +564,16 @@ public final void createTable(final CatalogProtos.TableDescProto tableDescProto)
table.putToParameters(ParquetOutputFormat.COMPRESSION,
tableDesc.getMeta().getProperty(ParquetOutputFormat.COMPRESSION));
}
} else if (tableDesc.getMeta().getDataFormat().equalsIgnoreCase(BuiltinStorages.ORC)) {
StorageFormatDescriptor descriptor = storageFormatFactory.get(IOConstants.ORC);
sd.setInputFormat(descriptor.getInputFormat());
sd.setOutputFormat(descriptor.getOutputFormat());
sd.getSerdeInfo().setSerializationLib(descriptor.getSerde());

if (tableDesc.getMeta().containsProperty(StorageConstants.ORC_COMPRESSION)) {
table.putToParameters(StorageConstants.ORC_COMPRESSION,
tableDesc.getMeta().getProperty(StorageConstants.ORC_COMPRESSION));
}
} else {
throw new UnsupportedException(tableDesc.getMeta().getDataFormat() + " in HivecatalogStore");
}
Expand Down
3 changes: 3 additions & 0 deletions ...atalog-drivers/tajo-hive/src/main/java/org/apache/tajo/catalog/store/HiveCatalogUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.ql.io.RCFileInputFormat;
import org.apache.hadoop.hive.ql.io.orc.OrcSerde;
import org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.serde.serdeConstants;
Expand Down Expand Up @@ -135,6 +136,8 @@ public static String getDataFormat(StorageDescriptor descriptor) {
}
} else if (ParquetHiveSerDe.class.getName().equals(serde)) {
return BuiltinStorages.PARQUET;
} else if (OrcSerde.class.getName().equals(serde)) {
return BuiltinStorages.ORC;
} else if (AvroSerDe.class.getName().equals(serde)) {
return BuiltinStorages.AVRO;
} else {
Expand Down
35 changes: 35 additions & 0 deletions ...g-drivers/tajo-hive/src/test/java/org/apache/tajo/catalog/store/TestHiveCatalogStore.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -649,6 +649,41 @@ public void testTableUsingParquet() throws Exception {
store.dropTable(DB_NAME, CUSTOMER);
}

@Test
public void testTableUsingORC() throws Exception {
TableMeta meta = new TableMeta("ORC", new KeyValueSet());

org.apache.tajo.catalog.Schema schema = new org.apache.tajo.catalog.Schema();
schema.addColumn("c_custkey", TajoDataTypes.Type.INT4);
schema.addColumn("c_name", TajoDataTypes.Type.TEXT);
schema.addColumn("c_address", TajoDataTypes.Type.TEXT);
schema.addColumn("c_nationkey", TajoDataTypes.Type.INT4);
schema.addColumn("c_phone", TajoDataTypes.Type.TEXT);
schema.addColumn("c_acctbal", TajoDataTypes.Type.FLOAT8);
schema.addColumn("c_mktsegment", TajoDataTypes.Type.TEXT);
schema.addColumn("c_comment", TajoDataTypes.Type.TEXT);

TableDesc table = new TableDesc(CatalogUtil.buildFQName(DB_NAME, CUSTOMER), schema, meta,
new Path(warehousePath, new Path(DB_NAME, CUSTOMER)).toUri());
store.createTable(table.getProto());
assertTrue(store.existTable(DB_NAME, CUSTOMER));

StorageFormatDescriptor descriptor = formatFactory.get(IOConstants.ORC);
org.apache.hadoop.hive.ql.metadata.Table hiveTable = store.getHiveTable(DB_NAME, CUSTOMER);
assertEquals(descriptor.getInputFormat(), hiveTable.getSd().getInputFormat());
assertEquals(descriptor.getOutputFormat(), hiveTable.getSd().getOutputFormat());

TableDesc table1 = new TableDesc(store.getTable(DB_NAME, CUSTOMER));
assertEquals(table.getName(), table1.getName());
assertEquals(table.getUri(), table1.getUri());
assertEquals(table.getSchema().size(), table1.getSchema().size());
for (int i = 0; i < table.getSchema().size(); i++) {
assertEquals(table.getSchema().getColumn(i).getSimpleName(), table1.getSchema().getColumn(i).getSimpleName());
}

store.dropTable(DB_NAME, CUSTOMER);
}

@Test
public void testDataTypeCompatibility() throws Exception {
String tableName = CatalogUtil.normalizeIdentifier("testDataTypeCompatibility");
Expand Down
3 changes: 0 additions & 3 deletions tajo-dist/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -154,9 +154,6 @@
run cp -r ${project.basedir}/src/main/conf .
run rm -rf lib/tajo-*-${project.version}.jar

run mkdir hive
run mv lib/hive-*.jar hive/

run mkdir -p share/jdbc-dist
Copy link
Member

@hyunsik hyunsik Mar 14, 2016

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It should be kept because org.apache.hadoop.hive.ql.metadata.Table::getSd() included in hive-apache-0.9 causes NoSuchMethod exception.

run cp -r $ROOT/tajo-jdbc/target/tajo-jdbc-${project.version}-jar-with-dependencies.jar ./share/jdbc-dist/tajo-jdbc-${project.version}.jar

Expand Down
4 changes: 0 additions & 4 deletions tajo-dist/src/main/bin/tajo
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -303,10 +303,6 @@ if [ ! -z ${HIVE_HOME} ] && [ -d ${HIVE_HOME} ] && [ -d ${HIVE_LIB} ]; then
for f in ${HIVE_LIB}/datanucleus-*.jar; do
CLASSPATH=${CLASSPATH}:$f;
done
else
for f in $TAJO_HOME/hive/*.jar; do
CLASSPATH=${CLASSPATH}:$f;
done
fi

if [ "${HIVE_JDBC_DRIVER_DIR}" != "" ]; then
Expand Down