From 3869b2815ce58dda99666bbeb8a7795a8f2b6237 Mon Sep 17 00:00:00 2001 From: Sourabh Badhya Date: Mon, 31 Oct 2022 11:03:45 +0530 Subject: [PATCH 1/6] HIVE-26680: Make CMV use Direct Insert Semantics --- .../create/CreateMaterializedViewDesc.java | 10 +- .../hive/ql/parse/SemanticAnalyzer.java | 72 ++-- .../hadoop/hive/ql/parse/TaskCompiler.java | 6 +- .../test/queries/clientpositive/cmv_direct.q | 52 +++ .../cmv_direct_with_specified_locations.q | 52 +++ .../cmv_direct_with_suffixed_locations.q | 53 +++ .../clientpositive/llap/cmv_direct.q.out | 307 +++++++++++++++++ .../cmv_direct_with_specified_locations.q.out | 317 ++++++++++++++++++ .../cmv_direct_with_suffixed_locations.q.out | 307 +++++++++++++++++ 9 files changed, 1141 insertions(+), 35 deletions(-) create mode 100644 ql/src/test/queries/clientpositive/cmv_direct.q create mode 100644 ql/src/test/queries/clientpositive/cmv_direct_with_specified_locations.q create mode 100644 ql/src/test/queries/clientpositive/cmv_direct_with_suffixed_locations.q create mode 100644 ql/src/test/results/clientpositive/llap/cmv_direct.q.out create mode 100644 ql/src/test/results/clientpositive/llap/cmv_direct_with_specified_locations.q.out create mode 100644 ql/src/test/results/clientpositive/llap/cmv_direct_with_suffixed_locations.q.out diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateMaterializedViewDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateMaterializedViewDesc.java index 5f34b8bb7c3e..7ab0589b71fc 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateMaterializedViewDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateMaterializedViewDesc.java @@ -75,7 +75,7 @@ public class CreateMaterializedViewDesc implements DDLDesc, Serializable { private List sortCols; private List distributeColNames; private List distributeCols; - private Long initialMmWriteId; // Initial MM write ID for CMV and import. + private Long initialWriteId; // Initial write ID for CMV and import. // The FSOP configuration for the FSOP that is going to write initial data during cmv. // This is not needed beyond compilation, so it is transient. private transient FileSinkDesc writer; @@ -389,12 +389,12 @@ public Table toTable(HiveConf conf) throws HiveException { return tbl; } - public void setInitialMmWriteId(Long mmWriteId) { - this.initialMmWriteId = mmWriteId; + public void setInitialWriteId(Long writeId) { + this.initialWriteId = writeId; } - public Long getInitialMmWriteId() { - return initialMmWriteId; + public Long getInitialWriteId() { + return initialWriteId; } public FileSinkDesc getAndUnsetWriter() { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 0f07fa4dbd4b..66eed9a4808c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -7265,7 +7265,7 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) LoadTableDesc ltd = null; ListBucketingCtx lbCtx = null; Map partSpec = null; - boolean isMmTable = false, isMmCreate = false, isNonNativeTable = false; + boolean isMmTable = false, isMmCreate = false, isDirectInsertCreate = false, isNonNativeTable = false; Long writeId = null; HiveTxnManager txnMgr = getTxnMgr(); @@ -7610,30 +7610,44 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) destTableIsMaterialization = false; tableName = HiveTableName.ofNullableWithNoDefault(viewDesc.getViewName()); tblProps = viewDesc.getTblProps(); + // Add suffix only when required confs are present + // and user has not specified a location to the table. + createTableUseSuffix = (HiveConf.getBoolVar(conf, ConfVars.HIVE_ACID_CREATE_TABLE_USE_SUFFIX) + || HiveConf.getBoolVar(conf, ConfVars.HIVE_ACID_LOCKLESS_READS_ENABLED)) + && viewDesc.getLocation() == null; } destTableIsTransactional = tblProps != null && AcidUtils.isTablePropertyTransactional(tblProps); if (destTableIsTransactional) { isNonNativeTable = MetaStoreUtils.isNonNativeTable(tblProps); boolean isCtas = tblDesc != null && tblDesc.isCTAS(); + boolean isCMV = viewDesc != null && qb.isMaterializedView(); isMmTable = isMmCreate = AcidUtils.isInsertOnlyTable(tblProps); - if (!isNonNativeTable && !destTableIsTemporary && isCtas) { + if (!isNonNativeTable && !destTableIsTemporary && (isCtas || isCMV)) { destTableIsFullAcid = AcidUtils.isFullAcidTable(tblProps); acidOperation = getAcidType(dest); - isDirectInsert = isDirectInsert(destTableIsFullAcid, acidOperation); + isDirectInsert = isDirectInsertCreate = isDirectInsert(destTableIsFullAcid, acidOperation); if (isDirectInsert || isMmTable) { - destinationPath = getCtasLocation(tblDesc, createTableUseSuffix); + destinationPath = getCtasOrCMVLocation(tblDesc, viewDesc, createTableUseSuffix); if (createTableUseSuffix) { - tblDesc.getTblProps().put(SOFT_DELETE_TABLE, Boolean.TRUE.toString()); + if (tblDesc != null) { + tblDesc.getTblProps().put(SOFT_DELETE_TABLE, Boolean.TRUE.toString()); + } else { + viewDesc.getTblProps().put(SOFT_DELETE_TABLE, Boolean.TRUE.toString()); + } } // Set the location in context for possible rollback. ctx.setLocation(destinationPath); // Setting the location so that metadata transformers // does not change the location later while creating the table. - tblDesc.setLocation(destinationPath.toString()); + if (tblDesc != null) { + tblDesc.setLocation(destinationPath.toString()); + } else { + viewDesc.setLocation(destinationPath.toString()); + } } else { // Set the location in context for possible rollback. - ctx.setLocation(getCtasLocation(tblDesc, createTableUseSuffix)); + ctx.setLocation(getCtasOrCMVLocation(tblDesc, viewDesc, createTableUseSuffix)); } } try { @@ -7645,14 +7659,12 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) } catch (LockException ex) { throw new SemanticException("Failed to allocate write Id", ex); } - if (isMmTable) { + if (isMmTable || isDirectInsert) { if (tblDesc != null) { tblDesc.setInitialWriteId(writeId); } else { - viewDesc.setInitialMmWriteId(writeId); + viewDesc.setInitialWriteId(writeId); } - } else if (isDirectInsert) { - tblDesc.setInitialWriteId(writeId); } } @@ -7917,17 +7929,13 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) destinationPath, currentTableId, destTableIsFullAcid, destTableIsTemporary,//this was 1/4 acid destTableIsMaterialization, queryTmpdir, rsCtx, dpCtx, lbCtx, fsRS, canBeMerged, destinationTable, writeId, isMmCreate, destType, qb, isDirectInsert, acidOperation, moveTaskId); - if (isMmCreate) { + if (isMmCreate || isDirectInsertCreate) { // Add FSD so that the LoadTask compilation could fix up its path to avoid the move. if (tableDesc != null) { tableDesc.setWriter(fileSinkDesc); } else { createVwDesc.setWriter(fileSinkDesc); } - } else if (qb.isCTAS() && isDirectInsert) { - if (tableDesc != null) { - tableDesc.setWriter(fileSinkDesc); - } } if (fileSinkDesc.getInsertOverwrite()) { @@ -7990,19 +7998,29 @@ protected boolean enableColumnStatsCollecting() { return true; } - private Path getCtasLocation(CreateTableDesc tblDesc, boolean createTableWithSuffix) throws SemanticException { + private Path getCtasOrCMVLocation(CreateTableDesc tblDesc, CreateMaterializedViewDesc viewDesc, + boolean createTableWithSuffix) throws SemanticException { Path location; + String protoName; + String[] names; + Table tbl; try { - String protoName = tblDesc.getDbTableName(); - String[] names = Utilities.getDbTableName(protoName); - - // Handle table translation initially and if not present - // use default table path. - // Property modifications of the table is handled later. - // We are interested in the location if it has changed - // due to table translation. - Table tbl = tblDesc.toTable(conf); - tbl = db.getTranslateTableDryrun(tbl.getTTable()); + if (tblDesc != null) { + protoName = tblDesc.getDbTableName(); + names = Utilities.getDbTableName(protoName); + + // Handle table translation initially and if not present + // use default table path. + // Property modifications of the table is handled later. + // We are interested in the location if it has changed + // due to table translation. + tbl = tblDesc.toTable(conf); + tbl = db.getTranslateTableDryrun(tbl.getTTable()); + } else { + protoName = viewDesc.getViewName(); + names = Utilities.getDbTableName(protoName); + tbl = viewDesc.toTable(conf); + } Warehouse wh = new Warehouse(conf); if (tbl.getSd() == null diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java index 8cc755e7d13c..79728007c75b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java @@ -473,10 +473,10 @@ private void setLoadFileLocation( } else { CreateMaterializedViewDesc cmv = pCtx.getCreateViewDesc(); dataSink = cmv.getAndUnsetWriter(); - txnId = cmv.getInitialMmWriteId(); + txnId = cmv.getInitialWriteId(); loc = cmv.getLocation(); } - Path location = (loc == null) ? getDefaultCtasLocation(pCtx) : new Path(loc); + Path location = (loc == null) ? getDefaultCtasOrCMVLocation(pCtx) : new Path(loc); if (pCtx.getQueryProperties().isCTAS()) { CreateTableDesc ctd = pCtx.getCreateTable(); if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.CREATE_TABLE_AS_EXTERNAL)) { @@ -511,7 +511,7 @@ private void setLoadFileLocation( lfd.setTargetDir(location); } - private Path getDefaultCtasLocation(final ParseContext pCtx) throws SemanticException { + private Path getDefaultCtasOrCMVLocation(final ParseContext pCtx) throws SemanticException { try { String protoName = null, suffix = ""; boolean isExternal = false; diff --git a/ql/src/test/queries/clientpositive/cmv_direct.q b/ql/src/test/queries/clientpositive/cmv_direct.q new file mode 100644 index 000000000000..a443149a5290 --- /dev/null +++ b/ql/src/test/queries/clientpositive/cmv_direct.q @@ -0,0 +1,52 @@ +-- SORT_QUERY_RESULTS +--! qt:dataset:alltypesorc + +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.acid.direct.insert.enabled=true; + +DROP TABLE IF EXISTS source; + +DROP MATERIALIZED VIEW IF EXISTS test_orc_cmv; + +DROP MATERIALIZED VIEW IF EXISTS test_orc_mmcmv; + +DROP MATERIALIZED VIEW IF EXISTS test_parquet_mmcmv; + +DROP MATERIALIZED VIEW IF EXISTS test_avro_mmcmv; + +DROP MATERIALIZED VIEW IF EXISTS test_textfile_mmcmv; + +CREATE TABLE source STORED AS ORC TBLPROPERTIES('transactional'='true') AS (SELECT cint, cfloat, cdouble, cstring1, ctimestamp1 FROM alltypesorc); + +CREATE MATERIALIZED VIEW test_orc_cmv STORED AS ORC TBLPROPERTIES('transactional'='true') AS ((SELECT * FROM source WHERE cint > 200 LIMIT 10) UNION (SELECT * FROM source WHERE cint < -100 LIMIT 10)); + +CREATE MATERIALIZED VIEW test_orc_mmcmv STORED AS ORC TBLPROPERTIES('transactional'='true', 'transactional_properties'='insert_only') AS ((SELECT * FROM source WHERE cint > 200 LIMIT 10) UNION (SELECT * FROM source WHERE cint < -100 LIMIT 10)); + +CREATE MATERIALIZED VIEW test_parquet_mmcmv STORED AS PARQUET TBLPROPERTIES('transactional'='true', 'transactional_properties'='insert_only') AS ((SELECT * FROM source WHERE cint > 200 LIMIT 10) UNION (SELECT * FROM source WHERE cint < -100 LIMIT 10)); + +CREATE MATERIALIZED VIEW test_avro_mmcmv STORED AS AVRO TBLPROPERTIES('transactional'='true', 'transactional_properties'='insert_only') AS ((SELECT * FROM source WHERE cint > 200 LIMIT 10) UNION (SELECT * FROM source WHERE cint < -100 LIMIT 10)); + +CREATE MATERIALIZED VIEW test_textfile_mmcmv STORED AS TEXTFILE TBLPROPERTIES('transactional'='true', 'transactional_properties'='insert_only') AS ((SELECT * FROM source WHERE cint > 200 LIMIT 10) UNION (SELECT * FROM source WHERE cint < -100 LIMIT 10)); + +SELECT * FROM test_orc_cmv ORDER BY cint; + +SELECT * FROM test_orc_mmcmv ORDER BY cint; + +SELECT * FROM test_parquet_mmcmv ORDER BY cint; + +SELECT * FROM test_avro_mmcmv ORDER BY cint; + +SELECT * FROM test_textfile_mmcmv ORDER BY cint; + +DROP MATERIALIZED VIEW IF EXISTS test_orc_cmv; + +DROP MATERIALIZED VIEW IF EXISTS test_orc_mmcmv; + +DROP MATERIALIZED VIEW IF EXISTS test_parquet_mmcmv; + +DROP MATERIALIZED VIEW IF EXISTS test_avro_mmcmv; + +DROP MATERIALIZED VIEW IF EXISTS test_textfile_mmcmv; + +DROP TABLE IF EXISTS source; \ No newline at end of file diff --git a/ql/src/test/queries/clientpositive/cmv_direct_with_specified_locations.q b/ql/src/test/queries/clientpositive/cmv_direct_with_specified_locations.q new file mode 100644 index 000000000000..e8df67ebdc25 --- /dev/null +++ b/ql/src/test/queries/clientpositive/cmv_direct_with_specified_locations.q @@ -0,0 +1,52 @@ +-- SORT_QUERY_RESULTS +--! qt:dataset:alltypesorc + +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.acid.direct.insert.enabled=true; + +DROP TABLE IF EXISTS source; + +DROP MATERIALIZED VIEW IF EXISTS test_orc_cmv; + +DROP MATERIALIZED VIEW IF EXISTS test_orc_mmcmv; + +DROP MATERIALIZED VIEW IF EXISTS test_parquet_mmcmv; + +DROP MATERIALIZED VIEW IF EXISTS test_avro_mmcmv; + +DROP MATERIALIZED VIEW IF EXISTS test_textfile_mmcmv; + +CREATE TABLE source STORED AS ORC TBLPROPERTIES('transactional'='true') AS (SELECT cint, cfloat, cdouble, cstring1, ctimestamp1 FROM alltypesorc); + +CREATE MATERIALIZED VIEW test_orc_cmv STORED AS ORC LOCATION '/build/ql/test/data/warehouse/test_cmv_orc' TBLPROPERTIES('transactional'='true') AS ((SELECT * FROM source WHERE cint > 200 LIMIT 10) UNION (SELECT * FROM source WHERE cint < -100 LIMIT 10)); + +CREATE MATERIALIZED VIEW test_orc_mmcmv STORED AS ORC LOCATION '/build/ql/test/data/warehouse/test_mmcmv_orc' TBLPROPERTIES('transactional'='true', 'transactional_properties'='insert_only') AS ((SELECT * FROM source WHERE cint > 200 LIMIT 10) UNION (SELECT * FROM source WHERE cint < -100 LIMIT 10)); + +CREATE MATERIALIZED VIEW test_parquet_mmcmv STORED AS PARQUET LOCATION '/build/ql/test/data/warehouse/test_mmcmv_parquet' TBLPROPERTIES('transactional'='true', 'transactional_properties'='insert_only') AS ((SELECT * FROM source WHERE cint > 200 LIMIT 10) UNION (SELECT * FROM source WHERE cint < -100 LIMIT 10)); + +CREATE MATERIALIZED VIEW test_avro_mmcmv STORED AS AVRO LOCATION '/build/ql/test/data/warehouse/test_mmcmv_avro' TBLPROPERTIES('transactional'='true', 'transactional_properties'='insert_only') AS ((SELECT * FROM source WHERE cint > 200 LIMIT 10) UNION (SELECT * FROM source WHERE cint < -100 LIMIT 10)); + +CREATE MATERIALIZED VIEW test_textfile_mmcmv STORED AS TEXTFILE LOCATION '/build/ql/test/data/warehouse/test_mmcmv_textfile' TBLPROPERTIES('transactional'='true', 'transactional_properties'='insert_only') AS ((SELECT * FROM source WHERE cint > 200 LIMIT 10) UNION (SELECT * FROM source WHERE cint < -100 LIMIT 10)); + +SELECT * FROM test_orc_cmv ORDER BY cint; + +SELECT * FROM test_orc_mmcmv ORDER BY cint; + +SELECT * FROM test_parquet_mmcmv ORDER BY cint; + +SELECT * FROM test_avro_mmcmv ORDER BY cint; + +SELECT * FROM test_textfile_mmcmv ORDER BY cint; + +DROP MATERIALIZED VIEW IF EXISTS test_orc_cmv; + +DROP MATERIALIZED VIEW IF EXISTS test_orc_mmcmv; + +DROP MATERIALIZED VIEW IF EXISTS test_parquet_mmcmv; + +DROP MATERIALIZED VIEW IF EXISTS test_avro_mmcmv; + +DROP MATERIALIZED VIEW IF EXISTS test_textfile_mmcmv; + +DROP TABLE IF EXISTS source; \ No newline at end of file diff --git a/ql/src/test/queries/clientpositive/cmv_direct_with_suffixed_locations.q b/ql/src/test/queries/clientpositive/cmv_direct_with_suffixed_locations.q new file mode 100644 index 000000000000..7c52a5e702cb --- /dev/null +++ b/ql/src/test/queries/clientpositive/cmv_direct_with_suffixed_locations.q @@ -0,0 +1,53 @@ +-- SORT_QUERY_RESULTS +--! qt:dataset:alltypesorc + +set hive.support.concurrency=true; +set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; +set hive.acid.direct.insert.enabled=true; +set hive.acid.createtable.softdelete=true; + +DROP TABLE IF EXISTS source; + +DROP MATERIALIZED VIEW IF EXISTS test_orc_cmv; + +DROP MATERIALIZED VIEW IF EXISTS test_orc_mmcmv; + +DROP MATERIALIZED VIEW IF EXISTS test_parquet_mmcmv; + +DROP MATERIALIZED VIEW IF EXISTS test_avro_mmcmv; + +DROP MATERIALIZED VIEW IF EXISTS test_textfile_mmcmv; + +CREATE TABLE source STORED AS ORC TBLPROPERTIES('transactional'='true') AS (SELECT cint, cfloat, cdouble, cstring1, ctimestamp1 FROM alltypesorc); + +CREATE MATERIALIZED VIEW test_orc_cmv STORED AS ORC TBLPROPERTIES('transactional'='true') AS ((SELECT * FROM source WHERE cint > 200 LIMIT 10) UNION (SELECT * FROM source WHERE cint < -100 LIMIT 10)); + +CREATE MATERIALIZED VIEW test_orc_mmcmv STORED AS ORC TBLPROPERTIES('transactional'='true', 'transactional_properties'='insert_only') AS ((SELECT * FROM source WHERE cint > 200 LIMIT 10) UNION (SELECT * FROM source WHERE cint < -100 LIMIT 10)); + +CREATE MATERIALIZED VIEW test_parquet_mmcmv STORED AS PARQUET TBLPROPERTIES('transactional'='true', 'transactional_properties'='insert_only') AS ((SELECT * FROM source WHERE cint > 200 LIMIT 10) UNION (SELECT * FROM source WHERE cint < -100 LIMIT 10)); + +CREATE MATERIALIZED VIEW test_avro_mmcmv STORED AS AVRO TBLPROPERTIES('transactional'='true', 'transactional_properties'='insert_only') AS ((SELECT * FROM source WHERE cint > 200 LIMIT 10) UNION (SELECT * FROM source WHERE cint < -100 LIMIT 10)); + +CREATE MATERIALIZED VIEW test_textfile_mmcmv STORED AS TEXTFILE TBLPROPERTIES('transactional'='true', 'transactional_properties'='insert_only') AS ((SELECT * FROM source WHERE cint > 200 LIMIT 10) UNION (SELECT * FROM source WHERE cint < -100 LIMIT 10)); + +SELECT * FROM test_orc_cmv ORDER BY cint; + +SELECT * FROM test_orc_mmcmv ORDER BY cint; + +SELECT * FROM test_parquet_mmcmv ORDER BY cint; + +SELECT * FROM test_avro_mmcmv ORDER BY cint; + +SELECT * FROM test_textfile_mmcmv ORDER BY cint; + +DROP MATERIALIZED VIEW IF EXISTS test_orc_cmv; + +DROP MATERIALIZED VIEW IF EXISTS test_orc_mmcmv; + +DROP MATERIALIZED VIEW IF EXISTS test_parquet_mmcmv; + +DROP MATERIALIZED VIEW IF EXISTS test_avro_mmcmv; + +DROP MATERIALIZED VIEW IF EXISTS test_textfile_mmcmv; + +DROP TABLE IF EXISTS source; \ No newline at end of file diff --git a/ql/src/test/results/clientpositive/llap/cmv_direct.q.out b/ql/src/test/results/clientpositive/llap/cmv_direct.q.out new file mode 100644 index 000000000000..9baf3f7045be --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/cmv_direct.q.out @@ -0,0 +1,307 @@ +PREHOOK: query: DROP TABLE IF EXISTS source +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS source +POSTHOOK: type: DROPTABLE +PREHOOK: query: DROP MATERIALIZED VIEW IF EXISTS test_orc_cmv +PREHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: query: DROP MATERIALIZED VIEW IF EXISTS test_orc_cmv +POSTHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: query: DROP MATERIALIZED VIEW IF EXISTS test_orc_mmcmv +PREHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: query: DROP MATERIALIZED VIEW IF EXISTS test_orc_mmcmv +POSTHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: query: DROP MATERIALIZED VIEW IF EXISTS test_parquet_mmcmv +PREHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: query: DROP MATERIALIZED VIEW IF EXISTS test_parquet_mmcmv +POSTHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: query: DROP MATERIALIZED VIEW IF EXISTS test_avro_mmcmv +PREHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: query: DROP MATERIALIZED VIEW IF EXISTS test_avro_mmcmv +POSTHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: query: DROP MATERIALIZED VIEW IF EXISTS test_textfile_mmcmv +PREHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: query: DROP MATERIALIZED VIEW IF EXISTS test_textfile_mmcmv +POSTHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: query: CREATE TABLE source STORED AS ORC TBLPROPERTIES('transactional'='true') AS (SELECT cint, cfloat, cdouble, cstring1, ctimestamp1 FROM alltypesorc) +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: database:default +PREHOOK: Output: default@source +POSTHOOK: query: CREATE TABLE source STORED AS ORC TBLPROPERTIES('transactional'='true') AS (SELECT cint, cfloat, cdouble, cstring1, ctimestamp1 FROM alltypesorc) +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: database:default +POSTHOOK: Output: default@source +POSTHOOK: Lineage: source.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] +POSTHOOK: Lineage: source.cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] +POSTHOOK: Lineage: source.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: source.cstring1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +POSTHOOK: Lineage: source.ctimestamp1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] +Only query text based automatic rewriting is available for materialized view. Statement has unsupported operator: union. +PREHOOK: query: CREATE MATERIALIZED VIEW test_orc_cmv STORED AS ORC TBLPROPERTIES('transactional'='true') AS ((SELECT * FROM source WHERE cint > 200 LIMIT 10) UNION (SELECT * FROM source WHERE cint < -100 LIMIT 10)) +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@source +PREHOOK: Output: database:default +PREHOOK: Output: default@test_orc_cmv +POSTHOOK: query: CREATE MATERIALIZED VIEW test_orc_cmv STORED AS ORC TBLPROPERTIES('transactional'='true') AS ((SELECT * FROM source WHERE cint > 200 LIMIT 10) UNION (SELECT * FROM source WHERE cint < -100 LIMIT 10)) +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@source +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test_orc_cmv +POSTHOOK: Lineage: test_orc_cmv.cdouble EXPRESSION [(source)source.FieldSchema(name:cdouble, type:double, comment:null), ] +POSTHOOK: Lineage: test_orc_cmv.cfloat EXPRESSION [(source)source.FieldSchema(name:cfloat, type:float, comment:null), ] +POSTHOOK: Lineage: test_orc_cmv.cint EXPRESSION [(source)source.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: test_orc_cmv.cstring1 EXPRESSION [(source)source.FieldSchema(name:cstring1, type:string, comment:null), ] +POSTHOOK: Lineage: test_orc_cmv.ctimestamp1 EXPRESSION [(source)source.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] +Only query text based automatic rewriting is available for materialized view. Statement has unsupported operator: union. +PREHOOK: query: CREATE MATERIALIZED VIEW test_orc_mmcmv STORED AS ORC TBLPROPERTIES('transactional'='true', 'transactional_properties'='insert_only') AS ((SELECT * FROM source WHERE cint > 200 LIMIT 10) UNION (SELECT * FROM source WHERE cint < -100 LIMIT 10)) +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@source +PREHOOK: Output: database:default +PREHOOK: Output: default@test_orc_mmcmv +POSTHOOK: query: CREATE MATERIALIZED VIEW test_orc_mmcmv STORED AS ORC TBLPROPERTIES('transactional'='true', 'transactional_properties'='insert_only') AS ((SELECT * FROM source WHERE cint > 200 LIMIT 10) UNION (SELECT * FROM source WHERE cint < -100 LIMIT 10)) +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@source +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test_orc_mmcmv +POSTHOOK: Lineage: test_orc_mmcmv.cdouble EXPRESSION [(source)source.FieldSchema(name:cdouble, type:double, comment:null), ] +POSTHOOK: Lineage: test_orc_mmcmv.cfloat EXPRESSION [(source)source.FieldSchema(name:cfloat, type:float, comment:null), ] +POSTHOOK: Lineage: test_orc_mmcmv.cint EXPRESSION [(source)source.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: test_orc_mmcmv.cstring1 EXPRESSION [(source)source.FieldSchema(name:cstring1, type:string, comment:null), ] +POSTHOOK: Lineage: test_orc_mmcmv.ctimestamp1 EXPRESSION [(source)source.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] +Only query text based automatic rewriting is available for materialized view. Statement has unsupported operator: union. +PREHOOK: query: CREATE MATERIALIZED VIEW test_parquet_mmcmv STORED AS PARQUET TBLPROPERTIES('transactional'='true', 'transactional_properties'='insert_only') AS ((SELECT * FROM source WHERE cint > 200 LIMIT 10) UNION (SELECT * FROM source WHERE cint < -100 LIMIT 10)) +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@source +PREHOOK: Output: database:default +PREHOOK: Output: default@test_parquet_mmcmv +POSTHOOK: query: CREATE MATERIALIZED VIEW test_parquet_mmcmv STORED AS PARQUET TBLPROPERTIES('transactional'='true', 'transactional_properties'='insert_only') AS ((SELECT * FROM source WHERE cint > 200 LIMIT 10) UNION (SELECT * FROM source WHERE cint < -100 LIMIT 10)) +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@source +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test_parquet_mmcmv +POSTHOOK: Lineage: test_parquet_mmcmv.cdouble EXPRESSION [(source)source.FieldSchema(name:cdouble, type:double, comment:null), ] +POSTHOOK: Lineage: test_parquet_mmcmv.cfloat EXPRESSION [(source)source.FieldSchema(name:cfloat, type:float, comment:null), ] +POSTHOOK: Lineage: test_parquet_mmcmv.cint EXPRESSION [(source)source.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: test_parquet_mmcmv.cstring1 EXPRESSION [(source)source.FieldSchema(name:cstring1, type:string, comment:null), ] +POSTHOOK: Lineage: test_parquet_mmcmv.ctimestamp1 EXPRESSION [(source)source.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] +Only query text based automatic rewriting is available for materialized view. Statement has unsupported operator: union. +PREHOOK: query: CREATE MATERIALIZED VIEW test_avro_mmcmv STORED AS AVRO TBLPROPERTIES('transactional'='true', 'transactional_properties'='insert_only') AS ((SELECT * FROM source WHERE cint > 200 LIMIT 10) UNION (SELECT * FROM source WHERE cint < -100 LIMIT 10)) +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@source +PREHOOK: Output: database:default +PREHOOK: Output: default@test_avro_mmcmv +POSTHOOK: query: CREATE MATERIALIZED VIEW test_avro_mmcmv STORED AS AVRO TBLPROPERTIES('transactional'='true', 'transactional_properties'='insert_only') AS ((SELECT * FROM source WHERE cint > 200 LIMIT 10) UNION (SELECT * FROM source WHERE cint < -100 LIMIT 10)) +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@source +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test_avro_mmcmv +POSTHOOK: Lineage: test_avro_mmcmv.cdouble EXPRESSION [(source)source.FieldSchema(name:cdouble, type:double, comment:null), ] +POSTHOOK: Lineage: test_avro_mmcmv.cfloat EXPRESSION [(source)source.FieldSchema(name:cfloat, type:float, comment:null), ] +POSTHOOK: Lineage: test_avro_mmcmv.cint EXPRESSION [(source)source.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: test_avro_mmcmv.cstring1 EXPRESSION [(source)source.FieldSchema(name:cstring1, type:string, comment:null), ] +POSTHOOK: Lineage: test_avro_mmcmv.ctimestamp1 EXPRESSION [(source)source.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] +Only query text based automatic rewriting is available for materialized view. Statement has unsupported operator: union. +PREHOOK: query: CREATE MATERIALIZED VIEW test_textfile_mmcmv STORED AS TEXTFILE TBLPROPERTIES('transactional'='true', 'transactional_properties'='insert_only') AS ((SELECT * FROM source WHERE cint > 200 LIMIT 10) UNION (SELECT * FROM source WHERE cint < -100 LIMIT 10)) +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@source +PREHOOK: Output: database:default +PREHOOK: Output: default@test_textfile_mmcmv +POSTHOOK: query: CREATE MATERIALIZED VIEW test_textfile_mmcmv STORED AS TEXTFILE TBLPROPERTIES('transactional'='true', 'transactional_properties'='insert_only') AS ((SELECT * FROM source WHERE cint > 200 LIMIT 10) UNION (SELECT * FROM source WHERE cint < -100 LIMIT 10)) +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@source +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test_textfile_mmcmv +POSTHOOK: Lineage: test_textfile_mmcmv.cdouble EXPRESSION [(source)source.FieldSchema(name:cdouble, type:double, comment:null), ] +POSTHOOK: Lineage: test_textfile_mmcmv.cfloat EXPRESSION [(source)source.FieldSchema(name:cfloat, type:float, comment:null), ] +POSTHOOK: Lineage: test_textfile_mmcmv.cint EXPRESSION [(source)source.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: test_textfile_mmcmv.cstring1 EXPRESSION [(source)source.FieldSchema(name:cstring1, type:string, comment:null), ] +POSTHOOK: Lineage: test_textfile_mmcmv.ctimestamp1 EXPRESSION [(source)source.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] +PREHOOK: query: SELECT * FROM test_orc_cmv ORDER BY cint +PREHOOK: type: QUERY +PREHOOK: Input: default@test_orc_cmv +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT * FROM test_orc_cmv ORDER BY cint +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_orc_cmv +POSTHOOK: Output: hdfs://### HDFS PATH ### +-109813638 -51.0 NULL t32s57Cjt4a250qQgVNAB5T 1969-12-31 16:00:08.451 +-164031131 -51.0 NULL AwVW3sV2gsM 1969-12-31 16:00:08.451 +-204359131 -51.0 NULL 21UE6fJyy 1969-12-31 16:00:08.451 +-285355633 -51.0 NULL LFgU5WT87C2yJ4W4YU0r8Pp 1969-12-31 16:00:08.451 +-468112720 -51.0 NULL XWIExC7NI3bqu6VhR14g2 1969-12-31 16:00:08.451 +-497211600 -51.0 NULL m4eSLx4qihVg1e32 1969-12-31 16:00:08.451 +-552134813 -51.0 NULL 7342q5oFQL8QIl7cO 1969-12-31 16:00:08.451 +-601502867 -51.0 NULL M152O 1969-12-31 16:00:08.451 +-660174857 -51.0 NULL VkXY4IOSO 1969-12-31 16:00:08.451 +-881630661 -51.0 NULL 3e27C1jTdTQPdvCWi4if 1969-12-31 16:00:08.451 +528534767 -11.0 -15431.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:52.176 +528534767 -28.0 -15813.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:55.787 +528534767 -34.0 15007.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:50.434 +528534767 -50.0 -13326.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:46.674 +528534767 27.0 -7824.0 cvLH6Eat2yFsyy7p 1969-12-31 16:00:04.963 +528534767 29.0 7021.0 cvLH6Eat2yFsyy7p 1969-12-31 16:00:15.007 +528534767 31.0 -9566.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:44.187 +528534767 31.0 4963.0 cvLH6Eat2yFsyy7p 1969-12-31 16:00:07.021 +528534767 61.0 -15549.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:44.569 +528534767 NULL -4213.0 cvLH6Eat2yFsyy7p NULL +PREHOOK: query: SELECT * FROM test_orc_mmcmv ORDER BY cint +PREHOOK: type: QUERY +PREHOOK: Input: default@test_orc_mmcmv +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT * FROM test_orc_mmcmv ORDER BY cint +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_orc_mmcmv +POSTHOOK: Output: hdfs://### HDFS PATH ### +-109813638 -51.0 NULL t32s57Cjt4a250qQgVNAB5T 1969-12-31 16:00:08.451 +-164031131 -51.0 NULL AwVW3sV2gsM 1969-12-31 16:00:08.451 +-204359131 -51.0 NULL 21UE6fJyy 1969-12-31 16:00:08.451 +-285355633 -51.0 NULL LFgU5WT87C2yJ4W4YU0r8Pp 1969-12-31 16:00:08.451 +-468112720 -51.0 NULL XWIExC7NI3bqu6VhR14g2 1969-12-31 16:00:08.451 +-497211600 -51.0 NULL m4eSLx4qihVg1e32 1969-12-31 16:00:08.451 +-552134813 -51.0 NULL 7342q5oFQL8QIl7cO 1969-12-31 16:00:08.451 +-601502867 -51.0 NULL M152O 1969-12-31 16:00:08.451 +-660174857 -51.0 NULL VkXY4IOSO 1969-12-31 16:00:08.451 +-881630661 -51.0 NULL 3e27C1jTdTQPdvCWi4if 1969-12-31 16:00:08.451 +528534767 -11.0 -15431.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:52.176 +528534767 -28.0 -15813.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:55.787 +528534767 -34.0 15007.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:50.434 +528534767 -50.0 -13326.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:46.674 +528534767 27.0 -7824.0 cvLH6Eat2yFsyy7p 1969-12-31 16:00:04.963 +528534767 29.0 7021.0 cvLH6Eat2yFsyy7p 1969-12-31 16:00:15.007 +528534767 31.0 -9566.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:44.187 +528534767 31.0 4963.0 cvLH6Eat2yFsyy7p 1969-12-31 16:00:07.021 +528534767 61.0 -15549.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:44.569 +528534767 NULL -4213.0 cvLH6Eat2yFsyy7p NULL +PREHOOK: query: SELECT * FROM test_parquet_mmcmv ORDER BY cint +PREHOOK: type: QUERY +PREHOOK: Input: default@test_parquet_mmcmv +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT * FROM test_parquet_mmcmv ORDER BY cint +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_parquet_mmcmv +POSTHOOK: Output: hdfs://### HDFS PATH ### +-109813638 -51.0 NULL t32s57Cjt4a250qQgVNAB5T 1969-12-31 16:00:08.451 +-164031131 -51.0 NULL AwVW3sV2gsM 1969-12-31 16:00:08.451 +-204359131 -51.0 NULL 21UE6fJyy 1969-12-31 16:00:08.451 +-285355633 -51.0 NULL LFgU5WT87C2yJ4W4YU0r8Pp 1969-12-31 16:00:08.451 +-468112720 -51.0 NULL XWIExC7NI3bqu6VhR14g2 1969-12-31 16:00:08.451 +-497211600 -51.0 NULL m4eSLx4qihVg1e32 1969-12-31 16:00:08.451 +-552134813 -51.0 NULL 7342q5oFQL8QIl7cO 1969-12-31 16:00:08.451 +-601502867 -51.0 NULL M152O 1969-12-31 16:00:08.451 +-660174857 -51.0 NULL VkXY4IOSO 1969-12-31 16:00:08.451 +-881630661 -51.0 NULL 3e27C1jTdTQPdvCWi4if 1969-12-31 16:00:08.451 +528534767 -11.0 -15431.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:52.176 +528534767 -28.0 -15813.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:55.787 +528534767 -34.0 15007.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:50.434 +528534767 -50.0 -13326.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:46.674 +528534767 27.0 -7824.0 cvLH6Eat2yFsyy7p 1969-12-31 16:00:04.963 +528534767 29.0 7021.0 cvLH6Eat2yFsyy7p 1969-12-31 16:00:15.007 +528534767 31.0 -9566.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:44.187 +528534767 31.0 4963.0 cvLH6Eat2yFsyy7p 1969-12-31 16:00:07.021 +528534767 61.0 -15549.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:44.569 +528534767 NULL -4213.0 cvLH6Eat2yFsyy7p NULL +PREHOOK: query: SELECT * FROM test_avro_mmcmv ORDER BY cint +PREHOOK: type: QUERY +PREHOOK: Input: default@test_avro_mmcmv +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT * FROM test_avro_mmcmv ORDER BY cint +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_avro_mmcmv +POSTHOOK: Output: hdfs://### HDFS PATH ### +-109813638 -51.0 NULL t32s57Cjt4a250qQgVNAB5T 1969-12-31 16:00:08.451 +-164031131 -51.0 NULL AwVW3sV2gsM 1969-12-31 16:00:08.451 +-204359131 -51.0 NULL 21UE6fJyy 1969-12-31 16:00:08.451 +-285355633 -51.0 NULL LFgU5WT87C2yJ4W4YU0r8Pp 1969-12-31 16:00:08.451 +-468112720 -51.0 NULL XWIExC7NI3bqu6VhR14g2 1969-12-31 16:00:08.451 +-497211600 -51.0 NULL m4eSLx4qihVg1e32 1969-12-31 16:00:08.451 +-552134813 -51.0 NULL 7342q5oFQL8QIl7cO 1969-12-31 16:00:08.451 +-601502867 -51.0 NULL M152O 1969-12-31 16:00:08.451 +-660174857 -51.0 NULL VkXY4IOSO 1969-12-31 16:00:08.451 +-881630661 -51.0 NULL 3e27C1jTdTQPdvCWi4if 1969-12-31 16:00:08.451 +528534767 -11.0 -15431.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:52.176 +528534767 -28.0 -15813.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:55.787 +528534767 -34.0 15007.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:50.434 +528534767 -50.0 -13326.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:46.674 +528534767 27.0 -7824.0 cvLH6Eat2yFsyy7p 1969-12-31 16:00:04.963 +528534767 29.0 7021.0 cvLH6Eat2yFsyy7p 1969-12-31 16:00:15.007 +528534767 31.0 -9566.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:44.187 +528534767 31.0 4963.0 cvLH6Eat2yFsyy7p 1969-12-31 16:00:07.021 +528534767 61.0 -15549.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:44.569 +528534767 NULL -4213.0 cvLH6Eat2yFsyy7p NULL +PREHOOK: query: SELECT * FROM test_textfile_mmcmv ORDER BY cint +PREHOOK: type: QUERY +PREHOOK: Input: default@test_textfile_mmcmv +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT * FROM test_textfile_mmcmv ORDER BY cint +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_textfile_mmcmv +POSTHOOK: Output: hdfs://### HDFS PATH ### +-109813638 -51.0 NULL t32s57Cjt4a250qQgVNAB5T 1969-12-31 16:00:08.451 +-164031131 -51.0 NULL AwVW3sV2gsM 1969-12-31 16:00:08.451 +-204359131 -51.0 NULL 21UE6fJyy 1969-12-31 16:00:08.451 +-285355633 -51.0 NULL LFgU5WT87C2yJ4W4YU0r8Pp 1969-12-31 16:00:08.451 +-468112720 -51.0 NULL XWIExC7NI3bqu6VhR14g2 1969-12-31 16:00:08.451 +-497211600 -51.0 NULL m4eSLx4qihVg1e32 1969-12-31 16:00:08.451 +-552134813 -51.0 NULL 7342q5oFQL8QIl7cO 1969-12-31 16:00:08.451 +-601502867 -51.0 NULL M152O 1969-12-31 16:00:08.451 +-660174857 -51.0 NULL VkXY4IOSO 1969-12-31 16:00:08.451 +-881630661 -51.0 NULL 3e27C1jTdTQPdvCWi4if 1969-12-31 16:00:08.451 +528534767 -11.0 -15431.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:52.176 +528534767 -28.0 -15813.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:55.787 +528534767 -34.0 15007.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:50.434 +528534767 -50.0 -13326.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:46.674 +528534767 27.0 -7824.0 cvLH6Eat2yFsyy7p 1969-12-31 16:00:04.963 +528534767 29.0 7021.0 cvLH6Eat2yFsyy7p 1969-12-31 16:00:15.007 +528534767 31.0 -9566.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:44.187 +528534767 31.0 4963.0 cvLH6Eat2yFsyy7p 1969-12-31 16:00:07.021 +528534767 61.0 -15549.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:44.569 +528534767 NULL -4213.0 cvLH6Eat2yFsyy7p NULL +PREHOOK: query: DROP MATERIALIZED VIEW IF EXISTS test_orc_cmv +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@test_orc_cmv +PREHOOK: Output: default@test_orc_cmv +POSTHOOK: query: DROP MATERIALIZED VIEW IF EXISTS test_orc_cmv +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@test_orc_cmv +POSTHOOK: Output: default@test_orc_cmv +PREHOOK: query: DROP MATERIALIZED VIEW IF EXISTS test_orc_mmcmv +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@test_orc_mmcmv +PREHOOK: Output: default@test_orc_mmcmv +POSTHOOK: query: DROP MATERIALIZED VIEW IF EXISTS test_orc_mmcmv +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@test_orc_mmcmv +POSTHOOK: Output: default@test_orc_mmcmv +PREHOOK: query: DROP MATERIALIZED VIEW IF EXISTS test_parquet_mmcmv +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@test_parquet_mmcmv +PREHOOK: Output: default@test_parquet_mmcmv +POSTHOOK: query: DROP MATERIALIZED VIEW IF EXISTS test_parquet_mmcmv +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@test_parquet_mmcmv +POSTHOOK: Output: default@test_parquet_mmcmv +PREHOOK: query: DROP MATERIALIZED VIEW IF EXISTS test_avro_mmcmv +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@test_avro_mmcmv +PREHOOK: Output: default@test_avro_mmcmv +POSTHOOK: query: DROP MATERIALIZED VIEW IF EXISTS test_avro_mmcmv +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@test_avro_mmcmv +POSTHOOK: Output: default@test_avro_mmcmv +PREHOOK: query: DROP MATERIALIZED VIEW IF EXISTS test_textfile_mmcmv +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@test_textfile_mmcmv +PREHOOK: Output: default@test_textfile_mmcmv +POSTHOOK: query: DROP MATERIALIZED VIEW IF EXISTS test_textfile_mmcmv +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@test_textfile_mmcmv +POSTHOOK: Output: default@test_textfile_mmcmv +PREHOOK: query: DROP TABLE IF EXISTS source +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@source +PREHOOK: Output: default@source +POSTHOOK: query: DROP TABLE IF EXISTS source +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@source +POSTHOOK: Output: default@source diff --git a/ql/src/test/results/clientpositive/llap/cmv_direct_with_specified_locations.q.out b/ql/src/test/results/clientpositive/llap/cmv_direct_with_specified_locations.q.out new file mode 100644 index 000000000000..484ba4979ebf --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/cmv_direct_with_specified_locations.q.out @@ -0,0 +1,317 @@ +PREHOOK: query: DROP TABLE IF EXISTS source +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS source +POSTHOOK: type: DROPTABLE +PREHOOK: query: DROP MATERIALIZED VIEW IF EXISTS test_orc_cmv +PREHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: query: DROP MATERIALIZED VIEW IF EXISTS test_orc_cmv +POSTHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: query: DROP MATERIALIZED VIEW IF EXISTS test_orc_mmcmv +PREHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: query: DROP MATERIALIZED VIEW IF EXISTS test_orc_mmcmv +POSTHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: query: DROP MATERIALIZED VIEW IF EXISTS test_parquet_mmcmv +PREHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: query: DROP MATERIALIZED VIEW IF EXISTS test_parquet_mmcmv +POSTHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: query: DROP MATERIALIZED VIEW IF EXISTS test_avro_mmcmv +PREHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: query: DROP MATERIALIZED VIEW IF EXISTS test_avro_mmcmv +POSTHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: query: DROP MATERIALIZED VIEW IF EXISTS test_textfile_mmcmv +PREHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: query: DROP MATERIALIZED VIEW IF EXISTS test_textfile_mmcmv +POSTHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: query: CREATE TABLE source STORED AS ORC TBLPROPERTIES('transactional'='true') AS (SELECT cint, cfloat, cdouble, cstring1, ctimestamp1 FROM alltypesorc) +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: database:default +PREHOOK: Output: default@source +POSTHOOK: query: CREATE TABLE source STORED AS ORC TBLPROPERTIES('transactional'='true') AS (SELECT cint, cfloat, cdouble, cstring1, ctimestamp1 FROM alltypesorc) +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: database:default +POSTHOOK: Output: default@source +POSTHOOK: Lineage: source.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] +POSTHOOK: Lineage: source.cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] +POSTHOOK: Lineage: source.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: source.cstring1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +POSTHOOK: Lineage: source.ctimestamp1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] +Only query text based automatic rewriting is available for materialized view. Statement has unsupported operator: union. +#### A masked pattern was here #### +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@source +PREHOOK: Input: hdfs://### HDFS PATH ### +PREHOOK: Output: database:default +PREHOOK: Output: default@test_orc_cmv +#### A masked pattern was here #### +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@source +POSTHOOK: Input: hdfs://### HDFS PATH ### +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test_orc_cmv +POSTHOOK: Lineage: test_orc_cmv.cdouble EXPRESSION [(source)source.FieldSchema(name:cdouble, type:double, comment:null), ] +POSTHOOK: Lineage: test_orc_cmv.cfloat EXPRESSION [(source)source.FieldSchema(name:cfloat, type:float, comment:null), ] +POSTHOOK: Lineage: test_orc_cmv.cint EXPRESSION [(source)source.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: test_orc_cmv.cstring1 EXPRESSION [(source)source.FieldSchema(name:cstring1, type:string, comment:null), ] +POSTHOOK: Lineage: test_orc_cmv.ctimestamp1 EXPRESSION [(source)source.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] +Only query text based automatic rewriting is available for materialized view. Statement has unsupported operator: union. +#### A masked pattern was here #### +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@source +PREHOOK: Input: hdfs://### HDFS PATH ### +PREHOOK: Output: database:default +PREHOOK: Output: default@test_orc_mmcmv +#### A masked pattern was here #### +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@source +POSTHOOK: Input: hdfs://### HDFS PATH ### +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test_orc_mmcmv +POSTHOOK: Lineage: test_orc_mmcmv.cdouble EXPRESSION [(source)source.FieldSchema(name:cdouble, type:double, comment:null), ] +POSTHOOK: Lineage: test_orc_mmcmv.cfloat EXPRESSION [(source)source.FieldSchema(name:cfloat, type:float, comment:null), ] +POSTHOOK: Lineage: test_orc_mmcmv.cint EXPRESSION [(source)source.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: test_orc_mmcmv.cstring1 EXPRESSION [(source)source.FieldSchema(name:cstring1, type:string, comment:null), ] +POSTHOOK: Lineage: test_orc_mmcmv.ctimestamp1 EXPRESSION [(source)source.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] +Only query text based automatic rewriting is available for materialized view. Statement has unsupported operator: union. +#### A masked pattern was here #### +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@source +PREHOOK: Input: hdfs://### HDFS PATH ### +PREHOOK: Output: database:default +PREHOOK: Output: default@test_parquet_mmcmv +#### A masked pattern was here #### +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@source +POSTHOOK: Input: hdfs://### HDFS PATH ### +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test_parquet_mmcmv +POSTHOOK: Lineage: test_parquet_mmcmv.cdouble EXPRESSION [(source)source.FieldSchema(name:cdouble, type:double, comment:null), ] +POSTHOOK: Lineage: test_parquet_mmcmv.cfloat EXPRESSION [(source)source.FieldSchema(name:cfloat, type:float, comment:null), ] +POSTHOOK: Lineage: test_parquet_mmcmv.cint EXPRESSION [(source)source.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: test_parquet_mmcmv.cstring1 EXPRESSION [(source)source.FieldSchema(name:cstring1, type:string, comment:null), ] +POSTHOOK: Lineage: test_parquet_mmcmv.ctimestamp1 EXPRESSION [(source)source.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] +Only query text based automatic rewriting is available for materialized view. Statement has unsupported operator: union. +#### A masked pattern was here #### +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@source +PREHOOK: Input: hdfs://### HDFS PATH ### +PREHOOK: Output: database:default +PREHOOK: Output: default@test_avro_mmcmv +#### A masked pattern was here #### +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@source +POSTHOOK: Input: hdfs://### HDFS PATH ### +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test_avro_mmcmv +POSTHOOK: Lineage: test_avro_mmcmv.cdouble EXPRESSION [(source)source.FieldSchema(name:cdouble, type:double, comment:null), ] +POSTHOOK: Lineage: test_avro_mmcmv.cfloat EXPRESSION [(source)source.FieldSchema(name:cfloat, type:float, comment:null), ] +POSTHOOK: Lineage: test_avro_mmcmv.cint EXPRESSION [(source)source.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: test_avro_mmcmv.cstring1 EXPRESSION [(source)source.FieldSchema(name:cstring1, type:string, comment:null), ] +POSTHOOK: Lineage: test_avro_mmcmv.ctimestamp1 EXPRESSION [(source)source.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] +Only query text based automatic rewriting is available for materialized view. Statement has unsupported operator: union. +#### A masked pattern was here #### +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@source +PREHOOK: Input: hdfs://### HDFS PATH ### +PREHOOK: Output: database:default +PREHOOK: Output: default@test_textfile_mmcmv +#### A masked pattern was here #### +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@source +POSTHOOK: Input: hdfs://### HDFS PATH ### +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test_textfile_mmcmv +POSTHOOK: Lineage: test_textfile_mmcmv.cdouble EXPRESSION [(source)source.FieldSchema(name:cdouble, type:double, comment:null), ] +POSTHOOK: Lineage: test_textfile_mmcmv.cfloat EXPRESSION [(source)source.FieldSchema(name:cfloat, type:float, comment:null), ] +POSTHOOK: Lineage: test_textfile_mmcmv.cint EXPRESSION [(source)source.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: test_textfile_mmcmv.cstring1 EXPRESSION [(source)source.FieldSchema(name:cstring1, type:string, comment:null), ] +POSTHOOK: Lineage: test_textfile_mmcmv.ctimestamp1 EXPRESSION [(source)source.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] +PREHOOK: query: SELECT * FROM test_orc_cmv ORDER BY cint +PREHOOK: type: QUERY +PREHOOK: Input: default@test_orc_cmv +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT * FROM test_orc_cmv ORDER BY cint +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_orc_cmv +POSTHOOK: Output: hdfs://### HDFS PATH ### +-109813638 -51.0 NULL t32s57Cjt4a250qQgVNAB5T 1969-12-31 16:00:08.451 +-164031131 -51.0 NULL AwVW3sV2gsM 1969-12-31 16:00:08.451 +-204359131 -51.0 NULL 21UE6fJyy 1969-12-31 16:00:08.451 +-285355633 -51.0 NULL LFgU5WT87C2yJ4W4YU0r8Pp 1969-12-31 16:00:08.451 +-468112720 -51.0 NULL XWIExC7NI3bqu6VhR14g2 1969-12-31 16:00:08.451 +-497211600 -51.0 NULL m4eSLx4qihVg1e32 1969-12-31 16:00:08.451 +-552134813 -51.0 NULL 7342q5oFQL8QIl7cO 1969-12-31 16:00:08.451 +-601502867 -51.0 NULL M152O 1969-12-31 16:00:08.451 +-660174857 -51.0 NULL VkXY4IOSO 1969-12-31 16:00:08.451 +-881630661 -51.0 NULL 3e27C1jTdTQPdvCWi4if 1969-12-31 16:00:08.451 +528534767 -11.0 -15431.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:52.176 +528534767 -28.0 -15813.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:55.787 +528534767 -34.0 15007.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:50.434 +528534767 -50.0 -13326.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:46.674 +528534767 27.0 -7824.0 cvLH6Eat2yFsyy7p 1969-12-31 16:00:04.963 +528534767 29.0 7021.0 cvLH6Eat2yFsyy7p 1969-12-31 16:00:15.007 +528534767 31.0 -9566.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:44.187 +528534767 31.0 4963.0 cvLH6Eat2yFsyy7p 1969-12-31 16:00:07.021 +528534767 61.0 -15549.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:44.569 +528534767 NULL -4213.0 cvLH6Eat2yFsyy7p NULL +PREHOOK: query: SELECT * FROM test_orc_mmcmv ORDER BY cint +PREHOOK: type: QUERY +PREHOOK: Input: default@test_orc_mmcmv +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT * FROM test_orc_mmcmv ORDER BY cint +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_orc_mmcmv +POSTHOOK: Output: hdfs://### HDFS PATH ### +-109813638 -51.0 NULL t32s57Cjt4a250qQgVNAB5T 1969-12-31 16:00:08.451 +-164031131 -51.0 NULL AwVW3sV2gsM 1969-12-31 16:00:08.451 +-204359131 -51.0 NULL 21UE6fJyy 1969-12-31 16:00:08.451 +-285355633 -51.0 NULL LFgU5WT87C2yJ4W4YU0r8Pp 1969-12-31 16:00:08.451 +-468112720 -51.0 NULL XWIExC7NI3bqu6VhR14g2 1969-12-31 16:00:08.451 +-497211600 -51.0 NULL m4eSLx4qihVg1e32 1969-12-31 16:00:08.451 +-552134813 -51.0 NULL 7342q5oFQL8QIl7cO 1969-12-31 16:00:08.451 +-601502867 -51.0 NULL M152O 1969-12-31 16:00:08.451 +-660174857 -51.0 NULL VkXY4IOSO 1969-12-31 16:00:08.451 +-881630661 -51.0 NULL 3e27C1jTdTQPdvCWi4if 1969-12-31 16:00:08.451 +528534767 -11.0 -15431.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:52.176 +528534767 -28.0 -15813.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:55.787 +528534767 -34.0 15007.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:50.434 +528534767 -50.0 -13326.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:46.674 +528534767 27.0 -7824.0 cvLH6Eat2yFsyy7p 1969-12-31 16:00:04.963 +528534767 29.0 7021.0 cvLH6Eat2yFsyy7p 1969-12-31 16:00:15.007 +528534767 31.0 -9566.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:44.187 +528534767 31.0 4963.0 cvLH6Eat2yFsyy7p 1969-12-31 16:00:07.021 +528534767 61.0 -15549.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:44.569 +528534767 NULL -4213.0 cvLH6Eat2yFsyy7p NULL +PREHOOK: query: SELECT * FROM test_parquet_mmcmv ORDER BY cint +PREHOOK: type: QUERY +PREHOOK: Input: default@test_parquet_mmcmv +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT * FROM test_parquet_mmcmv ORDER BY cint +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_parquet_mmcmv +POSTHOOK: Output: hdfs://### HDFS PATH ### +-109813638 -51.0 NULL t32s57Cjt4a250qQgVNAB5T 1969-12-31 16:00:08.451 +-164031131 -51.0 NULL AwVW3sV2gsM 1969-12-31 16:00:08.451 +-204359131 -51.0 NULL 21UE6fJyy 1969-12-31 16:00:08.451 +-285355633 -51.0 NULL LFgU5WT87C2yJ4W4YU0r8Pp 1969-12-31 16:00:08.451 +-468112720 -51.0 NULL XWIExC7NI3bqu6VhR14g2 1969-12-31 16:00:08.451 +-497211600 -51.0 NULL m4eSLx4qihVg1e32 1969-12-31 16:00:08.451 +-552134813 -51.0 NULL 7342q5oFQL8QIl7cO 1969-12-31 16:00:08.451 +-601502867 -51.0 NULL M152O 1969-12-31 16:00:08.451 +-660174857 -51.0 NULL VkXY4IOSO 1969-12-31 16:00:08.451 +-881630661 -51.0 NULL 3e27C1jTdTQPdvCWi4if 1969-12-31 16:00:08.451 +528534767 -11.0 -15431.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:52.176 +528534767 -28.0 -15813.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:55.787 +528534767 -34.0 15007.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:50.434 +528534767 -50.0 -13326.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:46.674 +528534767 27.0 -7824.0 cvLH6Eat2yFsyy7p 1969-12-31 16:00:04.963 +528534767 29.0 7021.0 cvLH6Eat2yFsyy7p 1969-12-31 16:00:15.007 +528534767 31.0 -9566.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:44.187 +528534767 31.0 4963.0 cvLH6Eat2yFsyy7p 1969-12-31 16:00:07.021 +528534767 61.0 -15549.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:44.569 +528534767 NULL -4213.0 cvLH6Eat2yFsyy7p NULL +PREHOOK: query: SELECT * FROM test_avro_mmcmv ORDER BY cint +PREHOOK: type: QUERY +PREHOOK: Input: default@test_avro_mmcmv +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT * FROM test_avro_mmcmv ORDER BY cint +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_avro_mmcmv +POSTHOOK: Output: hdfs://### HDFS PATH ### +-109813638 -51.0 NULL t32s57Cjt4a250qQgVNAB5T 1969-12-31 16:00:08.451 +-164031131 -51.0 NULL AwVW3sV2gsM 1969-12-31 16:00:08.451 +-204359131 -51.0 NULL 21UE6fJyy 1969-12-31 16:00:08.451 +-285355633 -51.0 NULL LFgU5WT87C2yJ4W4YU0r8Pp 1969-12-31 16:00:08.451 +-468112720 -51.0 NULL XWIExC7NI3bqu6VhR14g2 1969-12-31 16:00:08.451 +-497211600 -51.0 NULL m4eSLx4qihVg1e32 1969-12-31 16:00:08.451 +-552134813 -51.0 NULL 7342q5oFQL8QIl7cO 1969-12-31 16:00:08.451 +-601502867 -51.0 NULL M152O 1969-12-31 16:00:08.451 +-660174857 -51.0 NULL VkXY4IOSO 1969-12-31 16:00:08.451 +-881630661 -51.0 NULL 3e27C1jTdTQPdvCWi4if 1969-12-31 16:00:08.451 +528534767 -11.0 -15431.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:52.176 +528534767 -28.0 -15813.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:55.787 +528534767 -34.0 15007.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:50.434 +528534767 -50.0 -13326.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:46.674 +528534767 27.0 -7824.0 cvLH6Eat2yFsyy7p 1969-12-31 16:00:04.963 +528534767 29.0 7021.0 cvLH6Eat2yFsyy7p 1969-12-31 16:00:15.007 +528534767 31.0 -9566.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:44.187 +528534767 31.0 4963.0 cvLH6Eat2yFsyy7p 1969-12-31 16:00:07.021 +528534767 61.0 -15549.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:44.569 +528534767 NULL -4213.0 cvLH6Eat2yFsyy7p NULL +PREHOOK: query: SELECT * FROM test_textfile_mmcmv ORDER BY cint +PREHOOK: type: QUERY +PREHOOK: Input: default@test_textfile_mmcmv +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT * FROM test_textfile_mmcmv ORDER BY cint +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_textfile_mmcmv +POSTHOOK: Output: hdfs://### HDFS PATH ### +-109813638 -51.0 NULL t32s57Cjt4a250qQgVNAB5T 1969-12-31 16:00:08.451 +-164031131 -51.0 NULL AwVW3sV2gsM 1969-12-31 16:00:08.451 +-204359131 -51.0 NULL 21UE6fJyy 1969-12-31 16:00:08.451 +-285355633 -51.0 NULL LFgU5WT87C2yJ4W4YU0r8Pp 1969-12-31 16:00:08.451 +-468112720 -51.0 NULL XWIExC7NI3bqu6VhR14g2 1969-12-31 16:00:08.451 +-497211600 -51.0 NULL m4eSLx4qihVg1e32 1969-12-31 16:00:08.451 +-552134813 -51.0 NULL 7342q5oFQL8QIl7cO 1969-12-31 16:00:08.451 +-601502867 -51.0 NULL M152O 1969-12-31 16:00:08.451 +-660174857 -51.0 NULL VkXY4IOSO 1969-12-31 16:00:08.451 +-881630661 -51.0 NULL 3e27C1jTdTQPdvCWi4if 1969-12-31 16:00:08.451 +528534767 -11.0 -15431.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:52.176 +528534767 -28.0 -15813.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:55.787 +528534767 -34.0 15007.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:50.434 +528534767 -50.0 -13326.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:46.674 +528534767 27.0 -7824.0 cvLH6Eat2yFsyy7p 1969-12-31 16:00:04.963 +528534767 29.0 7021.0 cvLH6Eat2yFsyy7p 1969-12-31 16:00:15.007 +528534767 31.0 -9566.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:44.187 +528534767 31.0 4963.0 cvLH6Eat2yFsyy7p 1969-12-31 16:00:07.021 +528534767 61.0 -15549.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:44.569 +528534767 NULL -4213.0 cvLH6Eat2yFsyy7p NULL +PREHOOK: query: DROP MATERIALIZED VIEW IF EXISTS test_orc_cmv +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@test_orc_cmv +PREHOOK: Output: default@test_orc_cmv +POSTHOOK: query: DROP MATERIALIZED VIEW IF EXISTS test_orc_cmv +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@test_orc_cmv +POSTHOOK: Output: default@test_orc_cmv +PREHOOK: query: DROP MATERIALIZED VIEW IF EXISTS test_orc_mmcmv +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@test_orc_mmcmv +PREHOOK: Output: default@test_orc_mmcmv +POSTHOOK: query: DROP MATERIALIZED VIEW IF EXISTS test_orc_mmcmv +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@test_orc_mmcmv +POSTHOOK: Output: default@test_orc_mmcmv +PREHOOK: query: DROP MATERIALIZED VIEW IF EXISTS test_parquet_mmcmv +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@test_parquet_mmcmv +PREHOOK: Output: default@test_parquet_mmcmv +POSTHOOK: query: DROP MATERIALIZED VIEW IF EXISTS test_parquet_mmcmv +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@test_parquet_mmcmv +POSTHOOK: Output: default@test_parquet_mmcmv +PREHOOK: query: DROP MATERIALIZED VIEW IF EXISTS test_avro_mmcmv +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@test_avro_mmcmv +PREHOOK: Output: default@test_avro_mmcmv +POSTHOOK: query: DROP MATERIALIZED VIEW IF EXISTS test_avro_mmcmv +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@test_avro_mmcmv +POSTHOOK: Output: default@test_avro_mmcmv +PREHOOK: query: DROP MATERIALIZED VIEW IF EXISTS test_textfile_mmcmv +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@test_textfile_mmcmv +PREHOOK: Output: default@test_textfile_mmcmv +POSTHOOK: query: DROP MATERIALIZED VIEW IF EXISTS test_textfile_mmcmv +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@test_textfile_mmcmv +POSTHOOK: Output: default@test_textfile_mmcmv +PREHOOK: query: DROP TABLE IF EXISTS source +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@source +PREHOOK: Output: default@source +POSTHOOK: query: DROP TABLE IF EXISTS source +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@source +POSTHOOK: Output: default@source diff --git a/ql/src/test/results/clientpositive/llap/cmv_direct_with_suffixed_locations.q.out b/ql/src/test/results/clientpositive/llap/cmv_direct_with_suffixed_locations.q.out new file mode 100644 index 000000000000..9baf3f7045be --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/cmv_direct_with_suffixed_locations.q.out @@ -0,0 +1,307 @@ +PREHOOK: query: DROP TABLE IF EXISTS source +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS source +POSTHOOK: type: DROPTABLE +PREHOOK: query: DROP MATERIALIZED VIEW IF EXISTS test_orc_cmv +PREHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: query: DROP MATERIALIZED VIEW IF EXISTS test_orc_cmv +POSTHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: query: DROP MATERIALIZED VIEW IF EXISTS test_orc_mmcmv +PREHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: query: DROP MATERIALIZED VIEW IF EXISTS test_orc_mmcmv +POSTHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: query: DROP MATERIALIZED VIEW IF EXISTS test_parquet_mmcmv +PREHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: query: DROP MATERIALIZED VIEW IF EXISTS test_parquet_mmcmv +POSTHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: query: DROP MATERIALIZED VIEW IF EXISTS test_avro_mmcmv +PREHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: query: DROP MATERIALIZED VIEW IF EXISTS test_avro_mmcmv +POSTHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: query: DROP MATERIALIZED VIEW IF EXISTS test_textfile_mmcmv +PREHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: query: DROP MATERIALIZED VIEW IF EXISTS test_textfile_mmcmv +POSTHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: query: CREATE TABLE source STORED AS ORC TBLPROPERTIES('transactional'='true') AS (SELECT cint, cfloat, cdouble, cstring1, ctimestamp1 FROM alltypesorc) +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@alltypesorc +PREHOOK: Output: database:default +PREHOOK: Output: default@source +POSTHOOK: query: CREATE TABLE source STORED AS ORC TBLPROPERTIES('transactional'='true') AS (SELECT cint, cfloat, cdouble, cstring1, ctimestamp1 FROM alltypesorc) +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@alltypesorc +POSTHOOK: Output: database:default +POSTHOOK: Output: default@source +POSTHOOK: Lineage: source.cdouble SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cdouble, type:double, comment:null), ] +POSTHOOK: Lineage: source.cfloat SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cfloat, type:float, comment:null), ] +POSTHOOK: Lineage: source.cint SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: source.cstring1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:cstring1, type:string, comment:null), ] +POSTHOOK: Lineage: source.ctimestamp1 SIMPLE [(alltypesorc)alltypesorc.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] +Only query text based automatic rewriting is available for materialized view. Statement has unsupported operator: union. +PREHOOK: query: CREATE MATERIALIZED VIEW test_orc_cmv STORED AS ORC TBLPROPERTIES('transactional'='true') AS ((SELECT * FROM source WHERE cint > 200 LIMIT 10) UNION (SELECT * FROM source WHERE cint < -100 LIMIT 10)) +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@source +PREHOOK: Output: database:default +PREHOOK: Output: default@test_orc_cmv +POSTHOOK: query: CREATE MATERIALIZED VIEW test_orc_cmv STORED AS ORC TBLPROPERTIES('transactional'='true') AS ((SELECT * FROM source WHERE cint > 200 LIMIT 10) UNION (SELECT * FROM source WHERE cint < -100 LIMIT 10)) +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@source +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test_orc_cmv +POSTHOOK: Lineage: test_orc_cmv.cdouble EXPRESSION [(source)source.FieldSchema(name:cdouble, type:double, comment:null), ] +POSTHOOK: Lineage: test_orc_cmv.cfloat EXPRESSION [(source)source.FieldSchema(name:cfloat, type:float, comment:null), ] +POSTHOOK: Lineage: test_orc_cmv.cint EXPRESSION [(source)source.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: test_orc_cmv.cstring1 EXPRESSION [(source)source.FieldSchema(name:cstring1, type:string, comment:null), ] +POSTHOOK: Lineage: test_orc_cmv.ctimestamp1 EXPRESSION [(source)source.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] +Only query text based automatic rewriting is available for materialized view. Statement has unsupported operator: union. +PREHOOK: query: CREATE MATERIALIZED VIEW test_orc_mmcmv STORED AS ORC TBLPROPERTIES('transactional'='true', 'transactional_properties'='insert_only') AS ((SELECT * FROM source WHERE cint > 200 LIMIT 10) UNION (SELECT * FROM source WHERE cint < -100 LIMIT 10)) +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@source +PREHOOK: Output: database:default +PREHOOK: Output: default@test_orc_mmcmv +POSTHOOK: query: CREATE MATERIALIZED VIEW test_orc_mmcmv STORED AS ORC TBLPROPERTIES('transactional'='true', 'transactional_properties'='insert_only') AS ((SELECT * FROM source WHERE cint > 200 LIMIT 10) UNION (SELECT * FROM source WHERE cint < -100 LIMIT 10)) +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@source +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test_orc_mmcmv +POSTHOOK: Lineage: test_orc_mmcmv.cdouble EXPRESSION [(source)source.FieldSchema(name:cdouble, type:double, comment:null), ] +POSTHOOK: Lineage: test_orc_mmcmv.cfloat EXPRESSION [(source)source.FieldSchema(name:cfloat, type:float, comment:null), ] +POSTHOOK: Lineage: test_orc_mmcmv.cint EXPRESSION [(source)source.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: test_orc_mmcmv.cstring1 EXPRESSION [(source)source.FieldSchema(name:cstring1, type:string, comment:null), ] +POSTHOOK: Lineage: test_orc_mmcmv.ctimestamp1 EXPRESSION [(source)source.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] +Only query text based automatic rewriting is available for materialized view. Statement has unsupported operator: union. +PREHOOK: query: CREATE MATERIALIZED VIEW test_parquet_mmcmv STORED AS PARQUET TBLPROPERTIES('transactional'='true', 'transactional_properties'='insert_only') AS ((SELECT * FROM source WHERE cint > 200 LIMIT 10) UNION (SELECT * FROM source WHERE cint < -100 LIMIT 10)) +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@source +PREHOOK: Output: database:default +PREHOOK: Output: default@test_parquet_mmcmv +POSTHOOK: query: CREATE MATERIALIZED VIEW test_parquet_mmcmv STORED AS PARQUET TBLPROPERTIES('transactional'='true', 'transactional_properties'='insert_only') AS ((SELECT * FROM source WHERE cint > 200 LIMIT 10) UNION (SELECT * FROM source WHERE cint < -100 LIMIT 10)) +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@source +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test_parquet_mmcmv +POSTHOOK: Lineage: test_parquet_mmcmv.cdouble EXPRESSION [(source)source.FieldSchema(name:cdouble, type:double, comment:null), ] +POSTHOOK: Lineage: test_parquet_mmcmv.cfloat EXPRESSION [(source)source.FieldSchema(name:cfloat, type:float, comment:null), ] +POSTHOOK: Lineage: test_parquet_mmcmv.cint EXPRESSION [(source)source.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: test_parquet_mmcmv.cstring1 EXPRESSION [(source)source.FieldSchema(name:cstring1, type:string, comment:null), ] +POSTHOOK: Lineage: test_parquet_mmcmv.ctimestamp1 EXPRESSION [(source)source.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] +Only query text based automatic rewriting is available for materialized view. Statement has unsupported operator: union. +PREHOOK: query: CREATE MATERIALIZED VIEW test_avro_mmcmv STORED AS AVRO TBLPROPERTIES('transactional'='true', 'transactional_properties'='insert_only') AS ((SELECT * FROM source WHERE cint > 200 LIMIT 10) UNION (SELECT * FROM source WHERE cint < -100 LIMIT 10)) +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@source +PREHOOK: Output: database:default +PREHOOK: Output: default@test_avro_mmcmv +POSTHOOK: query: CREATE MATERIALIZED VIEW test_avro_mmcmv STORED AS AVRO TBLPROPERTIES('transactional'='true', 'transactional_properties'='insert_only') AS ((SELECT * FROM source WHERE cint > 200 LIMIT 10) UNION (SELECT * FROM source WHERE cint < -100 LIMIT 10)) +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@source +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test_avro_mmcmv +POSTHOOK: Lineage: test_avro_mmcmv.cdouble EXPRESSION [(source)source.FieldSchema(name:cdouble, type:double, comment:null), ] +POSTHOOK: Lineage: test_avro_mmcmv.cfloat EXPRESSION [(source)source.FieldSchema(name:cfloat, type:float, comment:null), ] +POSTHOOK: Lineage: test_avro_mmcmv.cint EXPRESSION [(source)source.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: test_avro_mmcmv.cstring1 EXPRESSION [(source)source.FieldSchema(name:cstring1, type:string, comment:null), ] +POSTHOOK: Lineage: test_avro_mmcmv.ctimestamp1 EXPRESSION [(source)source.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] +Only query text based automatic rewriting is available for materialized view. Statement has unsupported operator: union. +PREHOOK: query: CREATE MATERIALIZED VIEW test_textfile_mmcmv STORED AS TEXTFILE TBLPROPERTIES('transactional'='true', 'transactional_properties'='insert_only') AS ((SELECT * FROM source WHERE cint > 200 LIMIT 10) UNION (SELECT * FROM source WHERE cint < -100 LIMIT 10)) +PREHOOK: type: CREATE_MATERIALIZED_VIEW +PREHOOK: Input: default@source +PREHOOK: Output: database:default +PREHOOK: Output: default@test_textfile_mmcmv +POSTHOOK: query: CREATE MATERIALIZED VIEW test_textfile_mmcmv STORED AS TEXTFILE TBLPROPERTIES('transactional'='true', 'transactional_properties'='insert_only') AS ((SELECT * FROM source WHERE cint > 200 LIMIT 10) UNION (SELECT * FROM source WHERE cint < -100 LIMIT 10)) +POSTHOOK: type: CREATE_MATERIALIZED_VIEW +POSTHOOK: Input: default@source +POSTHOOK: Output: database:default +POSTHOOK: Output: default@test_textfile_mmcmv +POSTHOOK: Lineage: test_textfile_mmcmv.cdouble EXPRESSION [(source)source.FieldSchema(name:cdouble, type:double, comment:null), ] +POSTHOOK: Lineage: test_textfile_mmcmv.cfloat EXPRESSION [(source)source.FieldSchema(name:cfloat, type:float, comment:null), ] +POSTHOOK: Lineage: test_textfile_mmcmv.cint EXPRESSION [(source)source.FieldSchema(name:cint, type:int, comment:null), ] +POSTHOOK: Lineage: test_textfile_mmcmv.cstring1 EXPRESSION [(source)source.FieldSchema(name:cstring1, type:string, comment:null), ] +POSTHOOK: Lineage: test_textfile_mmcmv.ctimestamp1 EXPRESSION [(source)source.FieldSchema(name:ctimestamp1, type:timestamp, comment:null), ] +PREHOOK: query: SELECT * FROM test_orc_cmv ORDER BY cint +PREHOOK: type: QUERY +PREHOOK: Input: default@test_orc_cmv +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT * FROM test_orc_cmv ORDER BY cint +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_orc_cmv +POSTHOOK: Output: hdfs://### HDFS PATH ### +-109813638 -51.0 NULL t32s57Cjt4a250qQgVNAB5T 1969-12-31 16:00:08.451 +-164031131 -51.0 NULL AwVW3sV2gsM 1969-12-31 16:00:08.451 +-204359131 -51.0 NULL 21UE6fJyy 1969-12-31 16:00:08.451 +-285355633 -51.0 NULL LFgU5WT87C2yJ4W4YU0r8Pp 1969-12-31 16:00:08.451 +-468112720 -51.0 NULL XWIExC7NI3bqu6VhR14g2 1969-12-31 16:00:08.451 +-497211600 -51.0 NULL m4eSLx4qihVg1e32 1969-12-31 16:00:08.451 +-552134813 -51.0 NULL 7342q5oFQL8QIl7cO 1969-12-31 16:00:08.451 +-601502867 -51.0 NULL M152O 1969-12-31 16:00:08.451 +-660174857 -51.0 NULL VkXY4IOSO 1969-12-31 16:00:08.451 +-881630661 -51.0 NULL 3e27C1jTdTQPdvCWi4if 1969-12-31 16:00:08.451 +528534767 -11.0 -15431.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:52.176 +528534767 -28.0 -15813.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:55.787 +528534767 -34.0 15007.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:50.434 +528534767 -50.0 -13326.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:46.674 +528534767 27.0 -7824.0 cvLH6Eat2yFsyy7p 1969-12-31 16:00:04.963 +528534767 29.0 7021.0 cvLH6Eat2yFsyy7p 1969-12-31 16:00:15.007 +528534767 31.0 -9566.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:44.187 +528534767 31.0 4963.0 cvLH6Eat2yFsyy7p 1969-12-31 16:00:07.021 +528534767 61.0 -15549.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:44.569 +528534767 NULL -4213.0 cvLH6Eat2yFsyy7p NULL +PREHOOK: query: SELECT * FROM test_orc_mmcmv ORDER BY cint +PREHOOK: type: QUERY +PREHOOK: Input: default@test_orc_mmcmv +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT * FROM test_orc_mmcmv ORDER BY cint +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_orc_mmcmv +POSTHOOK: Output: hdfs://### HDFS PATH ### +-109813638 -51.0 NULL t32s57Cjt4a250qQgVNAB5T 1969-12-31 16:00:08.451 +-164031131 -51.0 NULL AwVW3sV2gsM 1969-12-31 16:00:08.451 +-204359131 -51.0 NULL 21UE6fJyy 1969-12-31 16:00:08.451 +-285355633 -51.0 NULL LFgU5WT87C2yJ4W4YU0r8Pp 1969-12-31 16:00:08.451 +-468112720 -51.0 NULL XWIExC7NI3bqu6VhR14g2 1969-12-31 16:00:08.451 +-497211600 -51.0 NULL m4eSLx4qihVg1e32 1969-12-31 16:00:08.451 +-552134813 -51.0 NULL 7342q5oFQL8QIl7cO 1969-12-31 16:00:08.451 +-601502867 -51.0 NULL M152O 1969-12-31 16:00:08.451 +-660174857 -51.0 NULL VkXY4IOSO 1969-12-31 16:00:08.451 +-881630661 -51.0 NULL 3e27C1jTdTQPdvCWi4if 1969-12-31 16:00:08.451 +528534767 -11.0 -15431.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:52.176 +528534767 -28.0 -15813.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:55.787 +528534767 -34.0 15007.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:50.434 +528534767 -50.0 -13326.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:46.674 +528534767 27.0 -7824.0 cvLH6Eat2yFsyy7p 1969-12-31 16:00:04.963 +528534767 29.0 7021.0 cvLH6Eat2yFsyy7p 1969-12-31 16:00:15.007 +528534767 31.0 -9566.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:44.187 +528534767 31.0 4963.0 cvLH6Eat2yFsyy7p 1969-12-31 16:00:07.021 +528534767 61.0 -15549.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:44.569 +528534767 NULL -4213.0 cvLH6Eat2yFsyy7p NULL +PREHOOK: query: SELECT * FROM test_parquet_mmcmv ORDER BY cint +PREHOOK: type: QUERY +PREHOOK: Input: default@test_parquet_mmcmv +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT * FROM test_parquet_mmcmv ORDER BY cint +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_parquet_mmcmv +POSTHOOK: Output: hdfs://### HDFS PATH ### +-109813638 -51.0 NULL t32s57Cjt4a250qQgVNAB5T 1969-12-31 16:00:08.451 +-164031131 -51.0 NULL AwVW3sV2gsM 1969-12-31 16:00:08.451 +-204359131 -51.0 NULL 21UE6fJyy 1969-12-31 16:00:08.451 +-285355633 -51.0 NULL LFgU5WT87C2yJ4W4YU0r8Pp 1969-12-31 16:00:08.451 +-468112720 -51.0 NULL XWIExC7NI3bqu6VhR14g2 1969-12-31 16:00:08.451 +-497211600 -51.0 NULL m4eSLx4qihVg1e32 1969-12-31 16:00:08.451 +-552134813 -51.0 NULL 7342q5oFQL8QIl7cO 1969-12-31 16:00:08.451 +-601502867 -51.0 NULL M152O 1969-12-31 16:00:08.451 +-660174857 -51.0 NULL VkXY4IOSO 1969-12-31 16:00:08.451 +-881630661 -51.0 NULL 3e27C1jTdTQPdvCWi4if 1969-12-31 16:00:08.451 +528534767 -11.0 -15431.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:52.176 +528534767 -28.0 -15813.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:55.787 +528534767 -34.0 15007.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:50.434 +528534767 -50.0 -13326.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:46.674 +528534767 27.0 -7824.0 cvLH6Eat2yFsyy7p 1969-12-31 16:00:04.963 +528534767 29.0 7021.0 cvLH6Eat2yFsyy7p 1969-12-31 16:00:15.007 +528534767 31.0 -9566.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:44.187 +528534767 31.0 4963.0 cvLH6Eat2yFsyy7p 1969-12-31 16:00:07.021 +528534767 61.0 -15549.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:44.569 +528534767 NULL -4213.0 cvLH6Eat2yFsyy7p NULL +PREHOOK: query: SELECT * FROM test_avro_mmcmv ORDER BY cint +PREHOOK: type: QUERY +PREHOOK: Input: default@test_avro_mmcmv +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT * FROM test_avro_mmcmv ORDER BY cint +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_avro_mmcmv +POSTHOOK: Output: hdfs://### HDFS PATH ### +-109813638 -51.0 NULL t32s57Cjt4a250qQgVNAB5T 1969-12-31 16:00:08.451 +-164031131 -51.0 NULL AwVW3sV2gsM 1969-12-31 16:00:08.451 +-204359131 -51.0 NULL 21UE6fJyy 1969-12-31 16:00:08.451 +-285355633 -51.0 NULL LFgU5WT87C2yJ4W4YU0r8Pp 1969-12-31 16:00:08.451 +-468112720 -51.0 NULL XWIExC7NI3bqu6VhR14g2 1969-12-31 16:00:08.451 +-497211600 -51.0 NULL m4eSLx4qihVg1e32 1969-12-31 16:00:08.451 +-552134813 -51.0 NULL 7342q5oFQL8QIl7cO 1969-12-31 16:00:08.451 +-601502867 -51.0 NULL M152O 1969-12-31 16:00:08.451 +-660174857 -51.0 NULL VkXY4IOSO 1969-12-31 16:00:08.451 +-881630661 -51.0 NULL 3e27C1jTdTQPdvCWi4if 1969-12-31 16:00:08.451 +528534767 -11.0 -15431.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:52.176 +528534767 -28.0 -15813.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:55.787 +528534767 -34.0 15007.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:50.434 +528534767 -50.0 -13326.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:46.674 +528534767 27.0 -7824.0 cvLH6Eat2yFsyy7p 1969-12-31 16:00:04.963 +528534767 29.0 7021.0 cvLH6Eat2yFsyy7p 1969-12-31 16:00:15.007 +528534767 31.0 -9566.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:44.187 +528534767 31.0 4963.0 cvLH6Eat2yFsyy7p 1969-12-31 16:00:07.021 +528534767 61.0 -15549.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:44.569 +528534767 NULL -4213.0 cvLH6Eat2yFsyy7p NULL +PREHOOK: query: SELECT * FROM test_textfile_mmcmv ORDER BY cint +PREHOOK: type: QUERY +PREHOOK: Input: default@test_textfile_mmcmv +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT * FROM test_textfile_mmcmv ORDER BY cint +POSTHOOK: type: QUERY +POSTHOOK: Input: default@test_textfile_mmcmv +POSTHOOK: Output: hdfs://### HDFS PATH ### +-109813638 -51.0 NULL t32s57Cjt4a250qQgVNAB5T 1969-12-31 16:00:08.451 +-164031131 -51.0 NULL AwVW3sV2gsM 1969-12-31 16:00:08.451 +-204359131 -51.0 NULL 21UE6fJyy 1969-12-31 16:00:08.451 +-285355633 -51.0 NULL LFgU5WT87C2yJ4W4YU0r8Pp 1969-12-31 16:00:08.451 +-468112720 -51.0 NULL XWIExC7NI3bqu6VhR14g2 1969-12-31 16:00:08.451 +-497211600 -51.0 NULL m4eSLx4qihVg1e32 1969-12-31 16:00:08.451 +-552134813 -51.0 NULL 7342q5oFQL8QIl7cO 1969-12-31 16:00:08.451 +-601502867 -51.0 NULL M152O 1969-12-31 16:00:08.451 +-660174857 -51.0 NULL VkXY4IOSO 1969-12-31 16:00:08.451 +-881630661 -51.0 NULL 3e27C1jTdTQPdvCWi4if 1969-12-31 16:00:08.451 +528534767 -11.0 -15431.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:52.176 +528534767 -28.0 -15813.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:55.787 +528534767 -34.0 15007.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:50.434 +528534767 -50.0 -13326.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:46.674 +528534767 27.0 -7824.0 cvLH6Eat2yFsyy7p 1969-12-31 16:00:04.963 +528534767 29.0 7021.0 cvLH6Eat2yFsyy7p 1969-12-31 16:00:15.007 +528534767 31.0 -9566.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:44.187 +528534767 31.0 4963.0 cvLH6Eat2yFsyy7p 1969-12-31 16:00:07.021 +528534767 61.0 -15549.0 cvLH6Eat2yFsyy7p 1969-12-31 15:59:44.569 +528534767 NULL -4213.0 cvLH6Eat2yFsyy7p NULL +PREHOOK: query: DROP MATERIALIZED VIEW IF EXISTS test_orc_cmv +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@test_orc_cmv +PREHOOK: Output: default@test_orc_cmv +POSTHOOK: query: DROP MATERIALIZED VIEW IF EXISTS test_orc_cmv +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@test_orc_cmv +POSTHOOK: Output: default@test_orc_cmv +PREHOOK: query: DROP MATERIALIZED VIEW IF EXISTS test_orc_mmcmv +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@test_orc_mmcmv +PREHOOK: Output: default@test_orc_mmcmv +POSTHOOK: query: DROP MATERIALIZED VIEW IF EXISTS test_orc_mmcmv +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@test_orc_mmcmv +POSTHOOK: Output: default@test_orc_mmcmv +PREHOOK: query: DROP MATERIALIZED VIEW IF EXISTS test_parquet_mmcmv +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@test_parquet_mmcmv +PREHOOK: Output: default@test_parquet_mmcmv +POSTHOOK: query: DROP MATERIALIZED VIEW IF EXISTS test_parquet_mmcmv +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@test_parquet_mmcmv +POSTHOOK: Output: default@test_parquet_mmcmv +PREHOOK: query: DROP MATERIALIZED VIEW IF EXISTS test_avro_mmcmv +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@test_avro_mmcmv +PREHOOK: Output: default@test_avro_mmcmv +POSTHOOK: query: DROP MATERIALIZED VIEW IF EXISTS test_avro_mmcmv +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@test_avro_mmcmv +POSTHOOK: Output: default@test_avro_mmcmv +PREHOOK: query: DROP MATERIALIZED VIEW IF EXISTS test_textfile_mmcmv +PREHOOK: type: DROP_MATERIALIZED_VIEW +PREHOOK: Input: default@test_textfile_mmcmv +PREHOOK: Output: default@test_textfile_mmcmv +POSTHOOK: query: DROP MATERIALIZED VIEW IF EXISTS test_textfile_mmcmv +POSTHOOK: type: DROP_MATERIALIZED_VIEW +POSTHOOK: Input: default@test_textfile_mmcmv +POSTHOOK: Output: default@test_textfile_mmcmv +PREHOOK: query: DROP TABLE IF EXISTS source +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@source +PREHOOK: Output: default@source +POSTHOOK: query: DROP TABLE IF EXISTS source +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@source +POSTHOOK: Output: default@source From 46aad58f939a04fc2ee7f2f303fd34f8cb7093d7 Mon Sep 17 00:00:00 2001 From: Sourabh Badhya Date: Wed, 2 Nov 2022 16:03:43 +0530 Subject: [PATCH 2/6] Fix tests - 1 --- itests/src/test/resources/testconfiguration.properties | 3 +++ .../results/clientpositive/llap/materialized_view_create.q.out | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index 0b589fe4d369..5b94366b8618 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -51,6 +51,9 @@ minillap.query.files=\ binary_output_format.q,\ bucket5.q,\ bucket6.q,\ + cmv_direct.q,\ + cmv_direct_with_specified_locations.q,\ + cmv_direct_with_suffixed_locations.q,\ create_genericudaf.q,\ create_table.q,\ create_udaf.q,\ diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create.q.out index 7f2b51485c59..132ad39b95e4 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_create.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_create.q.out @@ -373,4 +373,4 @@ POSTHOOK: Input: default@acid_cmv_part@k=0 POSTHOOK: Input: default@acid_cmv_part@k=10 POSTHOOK: Input: default@acid_cmv_part@k=100 #### A masked pattern was here #### -{"CBOPlan":"{\n \"rels\": [\n {\n \"id\": \"0\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"acid_cmv_part\"\n ],\n \"table:alias\": \"acid_cmv_part\",\n \"inputs\": [],\n \"rowCount\": 202.0,\n \"avgRowSize\": 200.0,\n \"rowType\": [\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"value\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"k\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"BLOCK__OFFSET__INSIDE__FILE\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"INPUT__FILE__NAME\"\n },\n {\n \"fields\": [\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"writeid\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"bucketid\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"rowid\"\n }\n ],\n \"name\": \"ROW__ID\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"ROW__IS__DELETED\"\n }\n ],\n \"partitionColumns\": [\n \"k\"\n ],\n \"colStats\": [\n {\n \"name\": \"value\",\n \"ndv\": 40\n },\n {\n \"name\": \"k\",\n \"ndv\": 3\n }\n ]\n },\n {\n \"id\": \"1\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"k\",\n \"value\"\n ],\n \"exprs\": [\n {\n \"input\": 1,\n \"name\": \"$1\"\n },\n {\n \"input\": 0,\n \"name\": \"$0\"\n }\n ],\n \"rowCount\": 202.0\n }\n ]\n}","optimizedSQL":"SELECT `k`, `value`\nFROM `default`.`acid_cmv_part`","cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-0":{"ROOT STAGE":"TRUE"}},"STAGE PLANS":{"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"TableScan":{"alias:":"acid_cmv_part","columns:":["value"],"database:":"default","table:":"acid_cmv_part","isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"k (type: string), value (type: string)","columnExprMap:":{"_col0":"k","_col1":"value"},"outputColumnNames:":["_col0","_col1"],"OperatorId:":"SEL_1","children":{"ListSink":{"OperatorId:":"LIST_SINK_3"}}}}}}}}}} +{"CBOPlan":"{\n \"rels\": [\n {\n \"id\": \"0\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan\",\n \"table\": [\n \"default\",\n \"acid_cmv_part\"\n ],\n \"table:alias\": \"acid_cmv_part\",\n \"inputs\": [],\n \"rowCount\": 270.0,\n \"avgRowSize\": 200.0,\n \"rowType\": [\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"value\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"k\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"BLOCK__OFFSET__INSIDE__FILE\"\n },\n {\n \"type\": \"VARCHAR\",\n \"nullable\": true,\n \"precision\": 2147483647,\n \"name\": \"INPUT__FILE__NAME\"\n },\n {\n \"fields\": [\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"writeid\"\n },\n {\n \"type\": \"INTEGER\",\n \"nullable\": true,\n \"name\": \"bucketid\"\n },\n {\n \"type\": \"BIGINT\",\n \"nullable\": true,\n \"name\": \"rowid\"\n }\n ],\n \"name\": \"ROW__ID\"\n },\n {\n \"type\": \"BOOLEAN\",\n \"nullable\": true,\n \"name\": \"ROW__IS__DELETED\"\n }\n ],\n \"partitionColumns\": [\n \"k\"\n ],\n \"colStats\": [\n {\n \"name\": \"value\",\n \"ndv\": 54\n },\n {\n \"name\": \"k\",\n \"ndv\": 3\n }\n ]\n },\n {\n \"id\": \"1\",\n \"relOp\": \"org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject\",\n \"fields\": [\n \"k\",\n \"value\"\n ],\n \"exprs\": [\n {\n \"input\": 1,\n \"name\": \"$1\"\n },\n {\n \"input\": 0,\n \"name\": \"$0\"\n }\n ],\n \"rowCount\": 270.0\n }\n ]\n}","optimizedSQL":"SELECT `k`, `value`\nFROM `default`.`acid_cmv_part`","cboInfo":"Plan optimized by CBO.","STAGE DEPENDENCIES":{"Stage-0":{"ROOT STAGE":"TRUE"}},"STAGE PLANS":{"Stage-0":{"Fetch Operator":{"limit:":"-1","Processor Tree:":{"TableScan":{"alias:":"acid_cmv_part","columns:":["value"],"database:":"default","table:":"acid_cmv_part","isTempTable:":"false","OperatorId:":"TS_0","children":{"Select Operator":{"expressions:":"k (type: string), value (type: string)","columnExprMap:":{"_col0":"k","_col1":"value"},"outputColumnNames:":["_col0","_col1"],"OperatorId:":"SEL_1","children":{"ListSink":{"OperatorId:":"LIST_SINK_3"}}}}}}}}}} From e0ef416dcf59de513a5f3ccb0552916438fac591 Mon Sep 17 00:00:00 2001 From: Sourabh Badhya Date: Mon, 7 Nov 2022 13:23:47 +0530 Subject: [PATCH 3/6] Address review comments - 1 --- .../org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 66eed9a4808c..4e543a4cc56c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -7265,7 +7265,7 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) LoadTableDesc ltd = null; ListBucketingCtx lbCtx = null; Map partSpec = null; - boolean isMmTable = false, isMmCreate = false, isDirectInsertCreate = false, isNonNativeTable = false; + boolean isMmTable = false, isMmCreate = false, isNonNativeTable = false; Long writeId = null; HiveTxnManager txnMgr = getTxnMgr(); @@ -7626,7 +7626,7 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) if (!isNonNativeTable && !destTableIsTemporary && (isCtas || isCMV)) { destTableIsFullAcid = AcidUtils.isFullAcidTable(tblProps); acidOperation = getAcidType(dest); - isDirectInsert = isDirectInsertCreate = isDirectInsert(destTableIsFullAcid, acidOperation); + isDirectInsert = isDirectInsert(destTableIsFullAcid, acidOperation); if (isDirectInsert || isMmTable) { destinationPath = getCtasOrCMVLocation(tblDesc, viewDesc, createTableUseSuffix); if (createTableUseSuffix) { @@ -7929,7 +7929,7 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) destinationPath, currentTableId, destTableIsFullAcid, destTableIsTemporary,//this was 1/4 acid destTableIsMaterialization, queryTmpdir, rsCtx, dpCtx, lbCtx, fsRS, canBeMerged, destinationTable, writeId, isMmCreate, destType, qb, isDirectInsert, acidOperation, moveTaskId); - if (isMmCreate || isDirectInsertCreate) { + if (isMmCreate || ((qb.isCTAS() || qb.isMaterializedView()) && isDirectInsert)) { // Add FSD so that the LoadTask compilation could fix up its path to avoid the move. if (tableDesc != null) { tableDesc.setWriter(fileSinkDesc); From 032f693844b86744bfc3ca7ad66227d033f726d5 Mon Sep 17 00:00:00 2001 From: Sourabh Badhya Date: Tue, 15 Nov 2022 12:24:19 +0530 Subject: [PATCH 4/6] Address review comments - 2 --- .../apache/hadoop/hive/ql/exec/Utilities.java | 12 +++++++++++ .../hive/ql/parse/SemanticAnalyzer.java | 21 +++++++------------ .../hadoop/hive/ql/parse/TaskCompiler.java | 16 ++------------ 3 files changed, 21 insertions(+), 28 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java index 0eb568614e5d..b2c3fbbda1fd 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java @@ -5010,4 +5010,16 @@ public static boolean arePathsEqualOrWithin(Path p1, Path p2) { return ((p1.toString().toLowerCase().indexOf(p2.toString().toLowerCase()) > -1) || (p2.toString().toLowerCase().indexOf(p1.toString().toLowerCase()) > -1)) ? true : false; } + + public static String getTableOrMVSuffix(Context context, boolean createTableOrMVUseSuffix) { + String suffix = ""; + if (createTableOrMVUseSuffix) { + long txnId = Optional.ofNullable(context) + .map(ctx -> ctx.getHiveTxnManager().getCurrentTxnId()).orElse(0L); + if (txnId != 0) { + suffix = AcidUtils.getPathSuffix(txnId); + } + } + return suffix; + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 4e543a4cc56c..5097e476fba8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -7929,7 +7929,7 @@ protected Operator genFileSinkPlan(String dest, QB qb, Operator input) destinationPath, currentTableId, destTableIsFullAcid, destTableIsTemporary,//this was 1/4 acid destTableIsMaterialization, queryTmpdir, rsCtx, dpCtx, lbCtx, fsRS, canBeMerged, destinationTable, writeId, isMmCreate, destType, qb, isDirectInsert, acidOperation, moveTaskId); - if (isMmCreate || ((qb.isCTAS() || qb.isMaterializedView()) && isDirectInsert)) { + if (isMmCreate || (qb.isCTAS() || qb.isMaterializedView()) && isDirectInsert) { // Add FSD so that the LoadTask compilation could fix up its path to avoid the move. if (tableDesc != null) { tableDesc.setWriter(fileSinkDesc); @@ -8001,13 +8001,11 @@ protected boolean enableColumnStatsCollecting() { private Path getCtasOrCMVLocation(CreateTableDesc tblDesc, CreateMaterializedViewDesc viewDesc, boolean createTableWithSuffix) throws SemanticException { Path location; - String protoName; String[] names; Table tbl; try { if (tblDesc != null) { - protoName = tblDesc.getDbTableName(); - names = Utilities.getDbTableName(protoName); + names = Utilities.getDbTableName(tblDesc.getDbTableName()); // Handle table translation initially and if not present // use default table path. @@ -8017,8 +8015,7 @@ private Path getCtasOrCMVLocation(CreateTableDesc tblDesc, CreateMaterializedVie tbl = tblDesc.toTable(conf); tbl = db.getTranslateTableDryrun(tbl.getTTable()); } else { - protoName = viewDesc.getViewName(); - names = Utilities.getDbTableName(protoName); + names = Utilities.getDbTableName(viewDesc.getViewName()); tbl = viewDesc.toTable(conf); } @@ -8031,9 +8028,8 @@ private Path getCtasOrCMVLocation(CreateTableDesc tblDesc, CreateMaterializedVie } if (createTableWithSuffix) { - long txnId = ctx.getHiveTxnManager().getCurrentTxnId(); - String suffix = AcidUtils.getPathSuffix(txnId); - location = new Path(location.toString() + suffix); + location = new Path(location.toString() + + Utilities.getTableOrMVSuffix(ctx, createTableWithSuffix)); } return location; @@ -8335,11 +8331,8 @@ private void handleLineage(Table destinationTable, LoadTableDesc ltd, Operator o Path tlocation = null; String tName = Utilities.getDbTableName(tableDesc.getDbTableName())[1]; try { - String suffix = ""; - if (AcidUtils.isTableSoftDeleteEnabled(destinationTable, conf)) { - long txnId = ctx.getHiveTxnManager().getCurrentTxnId(); - suffix = AcidUtils.getPathSuffix(txnId); - } + String suffix = Utilities.getTableOrMVSuffix(ctx, + AcidUtils.isTableSoftDeleteEnabled(destinationTable, conf)); Warehouse wh = new Warehouse(conf); tlocation = wh.getDefaultTablePath(db.getDatabase(tableDesc.getDatabaseName()), tName + suffix, tableDesc.isExternal()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java index 79728007c75b..f40567a3dfd0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java @@ -522,11 +522,11 @@ private Path getDefaultCtasOrCMVLocation(final ParseContext pCtx) throws Semanti protoName = pCtx.getCreateTable().getDbTableName(); isExternal = pCtx.getCreateTable().isExternal(); createTableOrMVUseSuffix &= AcidUtils.isTransactionalTable(pCtx.getCreateTable()); - suffix = getTableOrMVSuffix(pCtx, createTableOrMVUseSuffix); + suffix = Utilities.getTableOrMVSuffix(pCtx.getContext(), createTableOrMVUseSuffix); } else if (pCtx.getQueryProperties().isMaterializedView()) { protoName = pCtx.getCreateViewDesc().getViewName(); createTableOrMVUseSuffix &= AcidUtils.isTransactionalView(pCtx.getCreateViewDesc()); - suffix = getTableOrMVSuffix(pCtx, createTableOrMVUseSuffix); + suffix = Utilities.getTableOrMVSuffix(pCtx.getContext(), createTableOrMVUseSuffix); } String[] names = Utilities.getDbTableName(protoName); if (!db.databaseExists(names[0])) { @@ -539,18 +539,6 @@ private Path getDefaultCtasOrCMVLocation(final ParseContext pCtx) throws Semanti } } - public String getTableOrMVSuffix(ParseContext pCtx, boolean createTableOrMVUseSuffix) { - String suffix = ""; - if (createTableOrMVUseSuffix) { - long txnId = Optional.ofNullable(pCtx.getContext()) - .map(ctx -> ctx.getHiveTxnManager().getCurrentTxnId()).orElse(0L); - if (txnId != 0) { - suffix = AcidUtils.getPathSuffix(txnId); - } - } - return suffix; - } - private void patchUpAfterCTASorMaterializedView(List> rootTasks, Set inputs, Set outputs, Task createTask, boolean createTaskAfterMoveTask) { From 780a5841e680aa9735ac225e6f6b20f13dc57701 Mon Sep 17 00:00:00 2001 From: Sourabh Badhya Date: Fri, 18 Nov 2022 09:11:31 +0530 Subject: [PATCH 5/6] Remove unused import --- ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java | 1 - 1 file changed, 1 deletion(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java index f40567a3dfd0..193af2d695c8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java @@ -98,7 +98,6 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; /** From f7235de2be70c4c7ac71f91477efce76ce6dc88c Mon Sep 17 00:00:00 2001 From: Sourabh Badhya Date: Fri, 18 Nov 2022 17:42:00 +0530 Subject: [PATCH 6/6] Address review comments - 3 --- .../org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 5097e476fba8..64dd079ebd2e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -8002,10 +8002,11 @@ private Path getCtasOrCMVLocation(CreateTableDesc tblDesc, CreateMaterializedVie boolean createTableWithSuffix) throws SemanticException { Path location; String[] names; + String protoName = null; Table tbl; try { if (tblDesc != null) { - names = Utilities.getDbTableName(tblDesc.getDbTableName()); + protoName = tblDesc.getDbTableName(); // Handle table translation initially and if not present // use default table path. @@ -8015,9 +8016,10 @@ private Path getCtasOrCMVLocation(CreateTableDesc tblDesc, CreateMaterializedVie tbl = tblDesc.toTable(conf); tbl = db.getTranslateTableDryrun(tbl.getTTable()); } else { - names = Utilities.getDbTableName(viewDesc.getViewName()); + protoName = viewDesc.getViewName(); tbl = viewDesc.toTable(conf); } + names = Utilities.getDbTableName(protoName); Warehouse wh = new Warehouse(conf); if (tbl.getSd() == null