[LENS-1545]: Fixed a Bug and made changes for PreparedQuery.

lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java

@@ -1830,30 +1830,19 @@ public XFact getXFactTable(FactTable ft) throws LensException {
         for (UpdatePeriod updatePeriod : updatePeriods) {
           tableNames.add(updatePeriodToTableMap.get(updatePeriod));
         }
-        if (tableNames.size() <= 1) {
-          XStorageTableElement tblElement = JAXBUtils.getXStorageTableFromHiveTable(
-            getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(cft.getName(), storageName)));
-          tblElement.setStorageName(storageName);
-          for (UpdatePeriod p : updatePeriods) {
-            tblElement.getUpdatePeriods().getUpdatePeriod().add(XUpdatePeriod.valueOf(p.name()));
-          }
-          factTable.getStorageTables().getStorageTable().add(tblElement);
-        } else {
-          // Multiple storage tables.
-          XStorageTableElement tblElement = new XStorageTableElement();
-          tblElement.setStorageName(storageName);
-          XUpdatePeriods xUpdatePeriods = new XUpdatePeriods();
-          tblElement.setUpdatePeriods(xUpdatePeriods);
-          for (Map.Entry entry : updatePeriodToTableMap.entrySet()) {
-            XUpdatePeriodTableDescriptor updatePeriodTableDescriptor = new XUpdatePeriodTableDescriptor();
-            updatePeriodTableDescriptor.setTableDesc(getStorageTableDescFromHiveTable(
+        XStorageTableElement tblElement = new XStorageTableElement();
+        tblElement.setStorageName(storageName);
+        XUpdatePeriods xUpdatePeriods = new XUpdatePeriods();
+        tblElement.setUpdatePeriods(xUpdatePeriods);
+        for (Map.Entry entry : updatePeriodToTableMap.entrySet()) {
+          XUpdatePeriodTableDescriptor updatePeriodTableDescriptor = new XUpdatePeriodTableDescriptor();
+          updatePeriodTableDescriptor.setTableDesc(getStorageTableDescFromHiveTable(
               this.getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(cft.getName(),
-                (String) entry.getValue()))));
-            updatePeriodTableDescriptor.setUpdatePeriod(XUpdatePeriod.valueOf(((UpdatePeriod) entry.getKey()).name()));
-            xUpdatePeriods.getUpdatePeriodTableDescriptor().add(updatePeriodTableDescriptor);
-          }
-          factTable.getStorageTables().getStorageTable().add(tblElement);
+                  (String) entry.getValue()))));
+          updatePeriodTableDescriptor.setUpdatePeriod(XUpdatePeriod.valueOf(((UpdatePeriod) entry.getKey()).name()));
+          xUpdatePeriods.getUpdatePeriodTableDescriptor().add(updatePeriodTableDescriptor);
         }
+        factTable.getStorageTables().getStorageTable().add(tblElement);
       }
       fact = factTable;
     }

lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java

@@ -30,6 +30,7 @@
 import java.text.SimpleDateFormat;
 import java.util.*;
 
+import org.apache.lens.api.metastore.XFact;
 import org.apache.lens.cube.error.LensCubeErrorCode;
 import org.apache.lens.cube.metadata.ExprColumn.ExprSpec;
 import org.apache.lens.cube.metadata.ReferencedDimAttribute.ChainRefCol;
@@ -47,6 +48,7 @@
 import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
 import org.apache.hadoop.hive.metastore.api.Database;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
 import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat;
 import org.apache.hadoop.hive.ql.metadata.Hive;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -89,6 +91,7 @@ public class TestCubeMetastoreClient {
   private static final String CUBE_NAME_WITH_PROPS = "testMetastoreCubeWithProps";
   private static final String DERIVED_CUBE_NAME = "derivedTestMetastoreCube";
   private static final String DERIVED_CUBE_NAME_WITH_PROPS = "derivedTestMetastoreCubeWithProps";
+  private static final String X_FACT_NAME = "testMetastoreXFact";
   private static final Map<String, String> CUBE_PROPERTIES = new HashMap<>();
   private static HiveConf conf = new HiveConf(TestCubeMetastoreClient.class);
   private static FieldSchema dtPart = new FieldSchema(getDatePartitionKey(), serdeConstants.STRING_TYPE_NAME,
@@ -144,6 +147,11 @@ public static void setup() throws HiveException, AlreadyExistsException, LensExc
     conf.set(LensConfConstants.AUTHORIZER_CLASS, "org.apache.lens.cube.parse.MockAuthorizer");
     LensAuthorizer.get().init(conf);
 
+    try {
+      Hive.get().dropDatabase(TestCubeMetastoreClient.class.getSimpleName(), true, true, true);
+    } catch (NoSuchObjectException e) {
+      fail();
+    }
     Database database = new Database();
     database.setName(TestCubeMetastoreClient.class.getSimpleName());
     Hive.get(conf).createDatabase(database);
@@ -1591,6 +1599,48 @@ public void testCubeFactWithTwoTimedParts() throws Exception {
     assertFalse(client.latestPartitionExists(cubeFact.getName(), c1, testDtPart.getName()));
   }
 
+  @Test(priority = 2)
+  public void testGetXFactTable() throws Exception {
+    List<FieldSchema> factColumns = new ArrayList<>(cubeMeasures.size());
+    for (CubeMeasure measure : cubeMeasures) {
+      factColumns.add(measure.getColumn());
+    }
+
+    // add one dimension of the cube
+    factColumns.add(new FieldSchema("zipcode", "int", "zip"));
+    FieldSchema itPart = new FieldSchema("it", "string", "date part");
+    FieldSchema etPart = new FieldSchema("et", "string", "date part");
+    StorageTableDesc s1 = new StorageTableDesc(TextInputFormat.class, HiveIgnoreKeyTextOutputFormat.class,
+        Lists.newArrayList(getDatePartition(), itPart, etPart), Lists.newArrayList(getDatePartitionKey(),
+        itPart.getName(), etPart.getName()));
+
+    Set<UpdatePeriod> c1Storage = new HashSet<UpdatePeriod>();
+    c1Storage.add(HOURLY);
+    Map<String, Set<UpdatePeriod>> updatePeriods = new HashMap<String, Set<UpdatePeriod>>();
+    updatePeriods.put(c1, c1Storage);
+
+    Map<String, StorageTableDesc> storageTableDescs = getHashMap("HOURLY_c1", s1);
+
+    Map<UpdatePeriod, String> updatePeriodMap = new HashMap<>();
+    updatePeriodMap.put(HOURLY, HOURLY.toString() + "_" + c1);
+
+    Map<String, Set<String>> storageTablePartitionColumns = new HashMap<String, Set<String>>();
+    storageTablePartitionColumns.put(c1, new HashSet<>());
+
+    Map<String, Map<UpdatePeriod, String>> storageUpdatePeriodMap = new HashMap<String, Map<UpdatePeriod, String>>();
+
+    storageUpdatePeriodMap.put(c1, updatePeriodMap);
+
+    client.createCubeFactTable(CUBE_NAME_WITH_PROPS, X_FACT_NAME, factColumns, updatePeriods, 0.0d, new HashMap<String,
+        String>(), storageTableDescs, storageUpdatePeriodMap, storageTablePartitionColumns);
+
+    CubeFactTable cubeFact = new CubeFactTable(CUBE_NAME_WITH_PROPS, X_FACT_NAME, factColumns, updatePeriods, 0.0d,
+        new HashMap<String, String>(), storageUpdatePeriodMap, new HashMap<String, Set<String>>());
+    XFact xfact = client.getXFactTable(cubeFact);
+    assertEquals(xfact.getCubeName(), CUBE_NAME_WITH_PROPS);
+    assertEquals(xfact.getName(), X_FACT_NAME.toLowerCase());
+  }
+
   @Test(priority = 2)
   public void testCubeFactWithThreeTimedParts() throws Exception {
     String factName = "testMetastoreFact3TimedParts";

lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java

@@ -593,19 +593,25 @@ public DriverQueryPlan explain(AbstractQueryContext explainCtx) throws LensExcep
         + explainKeyword + " ");
     }
     log.info("{} Explain Query : {}", getFullyQualifiedName(), explainQuery);
-    QueryContext explainQueryCtx = QueryContext.createContextWithSingleDriver(explainQuery, null,
-      new LensConf(), explainConf, this, explainCtx.getLensSessionIdentifier(), false);
-    QueryResult result = null;
-    try {
-      result = executeInternal(explainQueryCtx, explainQuery);
-      if (result.error != null) {
-        throw new LensException("Query explain failed!", result.error);
-      }
-    } finally {
-      if (result != null) {
-        result.close();
+
+    boolean validateThroughPrepare = explainCtx.getDriverConf(this).getBoolean(JDBC_VALIDATE_THROUGH_PREPARE_OR_EXPLAIN,
+        DEFAULT_JDBC_VALIDATE_THROUGH_PREPARE_OR_EXPLAIN);
+    if (validateThroughPrepare) {
+      QueryContext explainQueryCtx = QueryContext.createContextWithSingleDriver(explainQuery, null, new LensConf(),
+          explainConf, this, explainCtx.getLensSessionIdentifier(), false);
+      QueryResult result = null;
+      try {
+        result = executeInternal(explainQueryCtx, explainQuery);
+        if (result.error != null) {
+          throw new LensException("Query explain failed!", result.error);
+        }
+      } finally {
+        if (result != null) {
+          result.close();
+        }
       }
     }
+
     JDBCQueryPlan jqp = new JDBCQueryPlan(calculateQueryCost(explainCtx));
     explainCtx.getDriverContext().setDriverQueryPlan(this, jqp);
     return jqp;
@@ -621,8 +627,8 @@ public void validate(AbstractQueryContext pContext) throws LensException {
     if (pContext.getDriverQuery(this) == null) {
       throw new NullPointerException("Null driver query for " + pContext.getUserQuery());
     }
-    boolean validateThroughPrepare = pContext.getDriverConf(this).getBoolean(JDBC_VALIDATE_THROUGH_PREPARE,
-      DEFAULT_JDBC_VALIDATE_THROUGH_PREPARE);
+    boolean validateThroughPrepare = pContext.getDriverConf(this).getBoolean(JDBC_VALIDATE_THROUGH_PREPARE_OR_EXPLAIN,
+      DEFAULT_JDBC_VALIDATE_THROUGH_PREPARE_OR_EXPLAIN);
     if (validateThroughPrepare) {
       PreparedStatement stmt;
       // Estimate queries need to get connection from estimate pool to make sure

lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriverConfConstants.java

@@ -94,11 +94,12 @@ public enum ConnectionPoolProperties {
   /** The Constant DEFAULT_JDBC_EXPLAIN_KEYWORD_BEFORE_SELECT. */
   public static final boolean DEFAULT_JDBC_EXPLAIN_KEYWORD_BEFORE_SELECT = true;
 
-  /** The Constant JDBC_VALIDATE_THROUGH_PREPARE. */
-  public static final String JDBC_VALIDATE_THROUGH_PREPARE = JDBC_DRIVER_PFX + "validate.through.prepare";
+  /** The Constant JDBC_VALIDATE_THROUGH_PREPARE_OR_EXPLAIN. */
+  public static final String JDBC_VALIDATE_THROUGH_PREPARE_OR_EXPLAIN =
+    JDBC_DRIVER_PFX + "validate.through.prepare.or.explain";
 
   /** The Constant DEFAULT_JDBC_VALIDATE_THROUGH_PREPARE. */
-  public static final boolean DEFAULT_JDBC_VALIDATE_THROUGH_PREPARE = true;
+  public static final boolean DEFAULT_JDBC_VALIDATE_THROUGH_PREPARE_OR_EXPLAIN = true;
 
   /** The Constant JDBC_VALIDATE_SKIP_WARNINGS */
   public static final String JDBC_VALIDATE_SKIP_WARNINGS = JDBC_DRIVER_PFX + "validate.skip.warnings";

lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java

@@ -49,10 +49,17 @@ public class PreparedQueryContext extends AbstractQueryContext implements Delaye
   private final QueryPrepareHandle prepareHandle;
 
   /**
-   * The prepared time.
+   * The prepare start time.
    */
   @Getter
-  private final Date preparedTime;
+  private final Date prepareStartTime;
+
+  /**
+   * The prepare end time.
+   */
+  @Getter
+  @Setter
+  private Date prepareEndTime = null;
 
   /**
    * The prepared user.
@@ -94,7 +101,7 @@ public PreparedQueryContext(String query, String user, Configuration conf, Colle
   public PreparedQueryContext(String query, String user, Configuration conf, LensConf qconf, Collection<LensDriver>
     drivers) {
     super(query, user, qconf, conf, drivers, true);
-    this.preparedTime = new Date();
+    this.prepareStartTime = new Date();
     this.preparedUser = user;
     this.prepareHandle = new QueryPrepareHandle(UUID.randomUUID());
     this.conf = conf;
@@ -119,9 +126,9 @@ public int compareTo(Delayed o) {
   @Override
   public long getDelay(TimeUnit units) {
     long delayMillis;
-    if (this.preparedTime != null) {
+    if (this.prepareStartTime != null) {
       Date now = new Date();
-      long elapsedMills = now.getTime() - this.preparedTime.getTime();
+      long elapsedMills = now.getTime() - this.prepareStartTime.getTime();
       delayMillis = millisInWeek - elapsedMills;
       return units.convert(delayMillis, TimeUnit.MILLISECONDS);
     } else {
@@ -147,7 +154,7 @@ public void updateConf(Map<String, String> confoverlay) {
    * @return the lens prepared query
    */
   public LensPreparedQuery toPreparedQuery() {
-    return new LensPreparedQuery(prepareHandle, userQuery, preparedTime, preparedUser, getDriverContext()
+    return new LensPreparedQuery(prepareHandle, userQuery, prepareStartTime, preparedUser, getDriverContext()
         .getSelectedDriver() != null ? getDriverContext().getSelectedDriver().getFullyQualifiedName() : null,
         getDriverContext().getSelectedDriverQuery(), lensConf);
   }

lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java

@@ -35,6 +35,7 @@
 import org.apache.lens.api.query.QueryStatus;
 import org.apache.lens.server.api.error.LensException;
 import org.apache.lens.server.api.query.FinishedLensQuery;
+import org.apache.lens.server.api.query.PreparedQueryContext;
 import org.apache.lens.server.api.query.QueryContext;
 import org.apache.lens.server.session.LensSessionImpl;
 import org.apache.lens.server.util.UtilityMethods;
@@ -47,6 +48,7 @@
 import org.apache.hadoop.conf.Configuration;
 
 import com.google.common.collect.Lists;
+
 import lombok.extern.slf4j.Slf4j;
 
 /**
@@ -105,6 +107,20 @@ public void createFinishedQueriesTable() throws Exception {
       log.warn("Unable to create finished queries table", e);
     }
   }
+
+  public void createPreparedQueriesTable() throws Exception {
+    String sql = "CREATE TABLE if not exists prepared_queries (handle varchar(255) NOT NULL unique,  userquery "
+      + "varchar(20000),  submitter varchar(255) NOT NULL,  timetaken bigint,  queryname varchar(255) DEFAULT NULL, "
+      + "drivername varchar(10000) DEFAULT NULL,  driverquery varchar(1000000),  starttime bigint)";
+    try {
+      QueryRunner runner = new QueryRunner(ds);
+      runner.update(sql);
+      log.info("Created prepared_queries queries table");
+    } catch (SQLException e) {
+      log.warn("Unable to create prepared_queries queries table", e);
+    }
+  }
+
   public void createFailedAttemptsTable() throws Exception {
     String sql = "CREATE TABLE if not exists failed_attempts (handle varchar(255) not null,"
       + "attempt_number int, drivername varchar(10000), progress float, progressmessage varchar(10000), "
@@ -821,4 +837,35 @@ public boolean deleteActiveSession(LensSessionHandle sessionId) throws LensExcep
 
     return result;
   }
+
+  /**
+   * DAO method to insert a new Prepared query into Table.
+   *
+   * @param preparedQueryContext to be inserted
+   * @throws SQLException the exception
+   */
+  public void insertPreparedQuery(PreparedQueryContext preparedQueryContext) throws LensException {
+    String sql = "insert into prepared_queries (handle, userquery, submitter, timetaken, queryname, drivername, "
+        + "driverquery, starttime)" + " values (?,?,?,?,?,?,?,?)";
+    Connection conn = null;
+    try {
+      conn = getConnection();
+      conn.setAutoCommit(false);
+      QueryRunner runner = new QueryRunner();
+
+      long timeTaken =
+          preparedQueryContext.getPrepareEndTime().getTime() - preparedQueryContext.getPrepareStartTime().getTime();
+
+      runner.update(conn, sql, preparedQueryContext.getPrepareHandle().getQueryHandleString(),
+          preparedQueryContext.getUserQuery(), preparedQueryContext.getSubmittedUser(), timeTaken,
+          preparedQueryContext.getQueryName(), preparedQueryContext.getDriverContext().getSelectedDriver().toString(),
+          preparedQueryContext.getSelectedDriverQuery(), preparedQueryContext.getPrepareStartTime().getTime());
+      conn.commit();
+    } catch (SQLException e) {
+      log.error("Failed to insert prepared query into database with error, " + e);
+      throw new LensException(e);
+  } finally {
+      DbUtils.closeQuietly(conn);
+    }
+  }
 }

lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java

@@ -147,7 +147,9 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE
   /**
    * The Constant PREPARED_QUERY_PURGER_COUNTER.
    */
-  public static final String PREPARED_QUERY_PURGER_COUNTER = "prepared-query-purger-errors";
+  public static final String PREPARED_QUERY_PURGER_ERROR_COUNTER = "prepared-query-purger-errors";
+
+  public static final String PREPARED_QUERY_INSERT_ERROR_COUNTER = "prepared-query-insert-errors";
 
   /**
    * The millis in week.
@@ -1302,13 +1304,13 @@ public void run() {
           destroyPreparedQuery(prepared);
           log.info("Purged prepared query: {}", prepared.getPrepareHandle());
         } catch (LensException e) {
-          incrCounter(PREPARED_QUERY_PURGER_COUNTER);
+          incrCounter(PREPARED_QUERY_PURGER_ERROR_COUNTER);
           log.error("Error closing prepared query ", e);
         } catch (InterruptedException e) {
           log.info("PreparedQueryPurger has been interrupted, exiting");
           return;
         } catch (Exception e) {
-          incrCounter(PREPARED_QUERY_PURGER_COUNTER);
+          incrCounter(PREPARED_QUERY_PURGER_ERROR_COUNTER);
           log.error("Error in prepared query purger", e);
         }
       }
@@ -1415,6 +1417,7 @@ private void initalizeFinishedQueryStore(Configuration conf) {
       this.lensServerDao.createFailedAttemptsTable();
       this.lensServerDao.createActiveSessionsTable();
       this.lensServerDao.createActiveQueriesTable();
+      this.lensServerDao.createPreparedQueriesTable();
     } catch (Exception e) {
       log.warn("Unable to create finished query tables, query purger will not purge queries", e);
     }
@@ -2054,9 +2057,10 @@ public QueryPrepareHandle prepare(LensSessionHandle sessionHandle, String query,
       acquire(sessionHandle);
       prepared = prepareQuery(sessionHandle, query, lensConf, SubmitOp.PREPARE);
       prepared.setQueryName(queryName);
-      prepared.getSelectedDriver().prepare(prepared);
+      lensServerDao.insertPreparedQuery(prepared);
       return prepared.getPrepareHandle();
     } catch (LensException e) {
+      incrCounter(PREPARED_QUERY_INSERT_ERROR_COUNTER);
       if (prepared != null) {
         destroyPreparedQuery(prepared);
       }
@@ -2087,6 +2091,7 @@ private PreparedQueryContext prepareQuery(LensSessionHandle sessionHandle, Strin
     preparedQueries.put(prepared.getPrepareHandle(), prepared);
     preparedQueryQueue.add(prepared);
     incrCounter(PREPARED_QUERIES_COUNTER);
+    prepared.setPrepareEndTime(new Date());
     return prepared;
   }
 
@@ -3031,7 +3036,7 @@ public List<QueryPrepareHandle> getAllPreparedQueries(LensSessionHandle sessionH
             continue;
           }
         }
-        long queryPrepTime = preparedQueryContext.getPreparedTime().getTime();
+        long queryPrepTime = preparedQueryContext.getPrepareStartTime().getTime();
         if (fromTime <= queryPrepTime && queryPrepTime < toTime) {
           continue;
         }