From 98ab7a54a5cbec36457afefcc1f355a7bca53ed0 Mon Sep 17 00:00:00 2001
From: Rajashekhar Choukimath <rajashekhar.c@inmobi.com>
Date: Wed, 18 Jul 2018 16:04:30 +0530
Subject: [PATCH 01/25] exposing jmx port for monitoring the threads, cpu and
 memory

---
 tools/conf-pseudo-distr/lens-env.sh | 5 ++++-
 tools/conf/lens-env.sh              | 5 ++++-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/tools/conf-pseudo-distr/lens-env.sh b/tools/conf-pseudo-distr/lens-env.sh
index 456e00133..c4ccdefa5 100644
--- a/tools/conf-pseudo-distr/lens-env.sh
+++ b/tools/conf-pseudo-distr/lens-env.sh
@@ -33,8 +33,11 @@
 # java heap size we want to set for the client. Default is 1024MB
 #export LENS_CLIENT_HEAP=
 
+# jmx port for monitoring the threads, cpu and memory
+JMX_OPTS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=8045 -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false"
+
 # any additional opts you want to set for lens server.
-export LENS_SERVER_OPTS="-XX:PermSize=256m -XX:MaxPermSize=256m"
+export LENS_SERVER_OPTS="-XX:PermSize=256m -XX:MaxPermSize=256m $JMX_OPTS"
 
 # java heap size we want to set for the lens server. Default is 1024MB
 #export LENS_SERVER_HEAP=
diff --git a/tools/conf/lens-env.sh b/tools/conf/lens-env.sh
index 456e00133..c4ccdefa5 100644
--- a/tools/conf/lens-env.sh
+++ b/tools/conf/lens-env.sh
@@ -33,8 +33,11 @@
 # java heap size we want to set for the client. Default is 1024MB
 #export LENS_CLIENT_HEAP=
 
+# jmx port for monitoring the threads, cpu and memory
+JMX_OPTS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=8045 -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false"
+
 # any additional opts you want to set for lens server.
-export LENS_SERVER_OPTS="-XX:PermSize=256m -XX:MaxPermSize=256m"
+export LENS_SERVER_OPTS="-XX:PermSize=256m -XX:MaxPermSize=256m $JMX_OPTS"
 
 # java heap size we want to set for the lens server. Default is 1024MB
 #export LENS_SERVER_HEAP=

From d3b41b38dd5a7b8cb0493c84d3b30e6cbbd76e90 Mon Sep 17 00:00:00 2001
From: Rajashekhar Choukimath <rajashekhar.c@inmobi.com>
Date: Thu, 16 Aug 2018 16:00:31 +0530
Subject: [PATCH 02/25] Fix for RNA-10440: Lens driver not picking hadoop conf

---
 .../src/main/java/org/apache/lens/server/LensServerConf.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lens-server/src/main/java/org/apache/lens/server/LensServerConf.java b/lens-server/src/main/java/org/apache/lens/server/LensServerConf.java
index 8d2a3fce4..526239c2b 100644
--- a/lens-server/src/main/java/org/apache/lens/server/LensServerConf.java
+++ b/lens-server/src/main/java/org/apache/lens/server/LensServerConf.java
@@ -42,7 +42,7 @@ private static final class ConfHolder {
     static {
       HIVE_CONF.addResource("lensserver-default.xml");
       HIVE_CONF.addResource("lens-site.xml");
-      Configuration conf = new Configuration(false);
+      Configuration conf = new Configuration(true);
       conf.addResource("lens-site.xml");
       for (Map.Entry<String, String> prop : conf) {
         if (!prop.getKey().startsWith(LensConfConstants.SERVER_PFX)) {

From 69e9e64cd33603f90e192ea99e2149333f3e710b Mon Sep 17 00:00:00 2001
From: Rajashekhar Choukimath <rajashekhar.c@inmobi.com>
Date: Thu, 16 Aug 2018 16:10:45 +0530
Subject: [PATCH 03/25] added the files

---
 tools/conf-pseudo-distr/lens-env.sh | 5 +----
 tools/conf/lens-env.sh              | 5 +----
 2 files changed, 2 insertions(+), 8 deletions(-)

diff --git a/tools/conf-pseudo-distr/lens-env.sh b/tools/conf-pseudo-distr/lens-env.sh
index c4ccdefa5..456e00133 100644
--- a/tools/conf-pseudo-distr/lens-env.sh
+++ b/tools/conf-pseudo-distr/lens-env.sh
@@ -33,11 +33,8 @@
 # java heap size we want to set for the client. Default is 1024MB
 #export LENS_CLIENT_HEAP=
 
-# jmx port for monitoring the threads, cpu and memory
-JMX_OPTS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=8045 -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false"
-
 # any additional opts you want to set for lens server.
-export LENS_SERVER_OPTS="-XX:PermSize=256m -XX:MaxPermSize=256m $JMX_OPTS"
+export LENS_SERVER_OPTS="-XX:PermSize=256m -XX:MaxPermSize=256m"
 
 # java heap size we want to set for the lens server. Default is 1024MB
 #export LENS_SERVER_HEAP=
diff --git a/tools/conf/lens-env.sh b/tools/conf/lens-env.sh
index c4ccdefa5..456e00133 100644
--- a/tools/conf/lens-env.sh
+++ b/tools/conf/lens-env.sh
@@ -33,11 +33,8 @@
 # java heap size we want to set for the client. Default is 1024MB
 #export LENS_CLIENT_HEAP=
 
-# jmx port for monitoring the threads, cpu and memory
-JMX_OPTS="-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=8045 -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false"
-
 # any additional opts you want to set for lens server.
-export LENS_SERVER_OPTS="-XX:PermSize=256m -XX:MaxPermSize=256m $JMX_OPTS"
+export LENS_SERVER_OPTS="-XX:PermSize=256m -XX:MaxPermSize=256m"
 
 # java heap size we want to set for the lens server. Default is 1024MB
 #export LENS_SERVER_HEAP=

From 3dffffb6bcd1c1959e056e79f28f4bd740147b97 Mon Sep 17 00:00:00 2001
From: Rajashekhar Choukimath <rajashekhar.c@inmobi.com>
Date: Mon, 27 Aug 2018 16:44:06 +0530
Subject: [PATCH 04/25] Adding support for Partition Columns to queried without
 giving additional xml tag valid.columns

---
 .../lens/cube/metadata/CubeFactTable.java     | 55 ++++++++++++++++---
 .../cube/metadata/CubeMetastoreClient.java    | 12 ++--
 .../cube/metadata/CubeVirtualFactTable.java   |  7 ++-
 .../apache/lens/cube/metadata/FactTable.java  |  7 +++
 .../apache/lens/cube/metadata/JAXBUtils.java  | 28 ++++++++--
 .../lens/cube/metadata/MetastoreUtil.java     |  4 ++
 .../lens/cube/parse/StorageCandidate.java     | 26 ++-------
 .../metadata/TestCubeMetastoreClient.java     |  7 ++-
 8 files changed, 101 insertions(+), 45 deletions(-)

diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java
index 8f3f4aa98..218e15350 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeFactTable.java
@@ -40,12 +40,14 @@ public class CubeFactTable extends AbstractCubeTable implements FactTable {
   private final Map<String, Map<UpdatePeriod, String>> storagePrefixUpdatePeriodMap;
   private String cubeName;
   private final Map<String, Set<UpdatePeriod>> storageUpdatePeriods;
+  private Map<String, Set<String>> storageTablePartitionColumns;
 
   public CubeFactTable(Table hiveTable) {
     super(hiveTable);
     this.storageUpdatePeriods = getUpdatePeriods(getName(), getProperties());
     this.cubeName = this.getProperties().get(MetastoreUtil.getFactCubeNameKey(getName()));
     this.storagePrefixUpdatePeriodMap = getUpdatePeriodMap(getName(), getProperties());
+    this.storageTablePartitionColumns = getStorageTablePartitionColumns(getName(), getProperties());
   }
 
   public CubeFactTable(String cubeName, String factName, List<FieldSchema> columns,
@@ -61,17 +63,20 @@ public CubeFactTable(String cubeName, String factName, List<FieldSchema> columns
   public CubeFactTable(String cubeName, String factName, List<FieldSchema> columns,
     Map<String, Set<UpdatePeriod>> storageUpdatePeriods, double weight, Map<String, String> properties) {
     this(cubeName, factName, columns, storageUpdatePeriods, weight, properties,
-      new HashMap<String, Map<UpdatePeriod, String>>());
+      new HashMap<String, Map<UpdatePeriod, String>>(), new HashMap<String, Set<String>>());
 
   }
 
   public CubeFactTable(String cubeName, String factName, List<FieldSchema> columns,
-    Map<String, Set<UpdatePeriod>> storageUpdatePeriods, double weight, Map<String, String> properties,
-    Map<String, Map<UpdatePeriod, String>> storagePrefixUpdatePeriodMap) {
+                       Map<String, Set<UpdatePeriod>> storageUpdatePeriods, double weight,
+                       Map<String, String> properties,
+                       Map<String, Map<UpdatePeriod, String>> storagePrefixUpdatePeriodMap,
+                       Map<String, Set<String>> storageTablePartitionColumns) {
     super(factName, columns, properties, weight);
     this.cubeName = cubeName;
     this.storageUpdatePeriods = storageUpdatePeriods;
     this.storagePrefixUpdatePeriodMap = storagePrefixUpdatePeriodMap;
+    this.storageTablePartitionColumns = storageTablePartitionColumns;
     addProperties();
   }
 
@@ -90,10 +95,20 @@ protected void addProperties() {
     this.getProperties().put(MetastoreUtil.getFactCubeNameKey(getName()), cubeName);
     addUpdatePeriodProperies(getName(), getProperties(), storageUpdatePeriods);
     addStorageTableProperties(getName(), getProperties(), storagePrefixUpdatePeriodMap);
+    addStorageTableParititionColumns(getName(), getProperties(), storageTablePartitionColumns);
   }
 
-  private void addStorageTableProperties(String name, Map<String, String> properties,
-    Map<String, Map<UpdatePeriod, String>> storageUpdatePeriodMap) {
+  private void addStorageTableParititionColumns(String name, Map<String, String> properties,
+                                                Map<String, Set<String>> storageTablePartitionColumns) {
+    for (String storage : storageTablePartitionColumns.keySet()) {
+      String partitionColumnKey = MetastoreUtil.getPartitionColumnKey(name, storage);
+      String partitionColumnValues = StringUtils.join(storageTablePartitionColumns.get(storage), ',');
+      properties.put(partitionColumnKey, partitionColumnValues);
+    }
+  }
+
+  private void addStorageTableProperties(String name, Map<String, String> properties, Map<String, Map<UpdatePeriod,
+          String>> storageUpdatePeriodMap) {
     for (String storageName : storageUpdatePeriodMap.keySet()) {
       String prefix = MetastoreUtil.getFactKeyPrefix(name) + "." + storageName;
       for (Map.Entry updatePeriodEntry : storageUpdatePeriodMap.get(storageName).entrySet()) {
@@ -104,12 +119,12 @@ private void addStorageTableProperties(String name, Map<String, String> properti
   }
 
   private static void addUpdatePeriodProperies(String name, Map<String, String> props,
-    Map<String, Set<UpdatePeriod>> updatePeriods) {
+                                               Map<String, Set<UpdatePeriod>> updatePeriods) {
     if (updatePeriods != null) {
       props.put(MetastoreUtil.getFactStorageListKey(name), MetastoreUtil.getStr(updatePeriods.keySet()));
       for (Map.Entry<String, Set<UpdatePeriod>> entry : updatePeriods.entrySet()) {
         props.put(MetastoreUtil.getFactUpdatePeriodKey(name, entry.getKey()),
-          MetastoreUtil.getNamedStr(entry.getValue()));
+                MetastoreUtil.getNamedStr(entry.getValue()));
       }
     }
   }
@@ -151,10 +166,36 @@ private Map<String, Set<UpdatePeriod>> getUpdatePeriods(String name, Map<String,
     return storageUpdatePeriods;
   }
 
+  private Map<String, Set<String>> getStorageTablePartitionColumns(String name, Map<String, String> props) {
+    Map<String, Set<String>> storageTablePartitionColumns = new HashMap<>();
+    String storagesStr = props.get(MetastoreUtil.getFactStorageListKey(name));
+    if (!StringUtils.isBlank(storagesStr)) {
+      String[] storages = storagesStr.split(",");
+      for (String storage : storages) {
+        String partitionColumnKey = MetastoreUtil.getPartitionColumnKey(name, storage);
+        String partitionColumnValues = props.get(partitionColumnKey);
+        if (StringUtils.isNotEmpty(partitionColumnValues)) {
+          storageTablePartitionColumns.put(partitionColumnKey,
+                  new HashSet<String>(Arrays.asList(partitionColumnValues.split(","))));
+        }
+
+      }
+    }
+    return storageTablePartitionColumns;
+  }
+
   public Map<String, Set<UpdatePeriod>> getUpdatePeriods() {
     return storageUpdatePeriods;
   }
 
+  public Set<String> getPartitionColumns(String storageName) {
+    String partitionColumnKey = MetastoreUtil.getPartitionColumnKey(getName(), storageName);
+    if (storageTablePartitionColumns.containsKey(partitionColumnKey)) {
+      return storageTablePartitionColumns.get(partitionColumnKey);
+    }
+    return new HashSet<String>();
+  }
+
   @Override
   public int hashCode() {
     return super.hashCode();
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
index e6afcff9f..d8267eeb8 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
@@ -29,19 +29,16 @@
 import java.util.concurrent.ConcurrentHashMap;
 
 import org.apache.lens.api.metastore.*;
-
 import org.apache.lens.cube.authorization.AuthorizationUtil;
 import org.apache.lens.cube.error.LensCubeErrorCode;
 import org.apache.lens.cube.metadata.Storage.LatestInfo;
 import org.apache.lens.cube.metadata.Storage.LatestPartColumnInfo;
 import org.apache.lens.cube.metadata.timeline.PartitionTimeline;
 import org.apache.lens.cube.metadata.timeline.PartitionTimelineFactory;
-
 import org.apache.lens.server.api.LensConfConstants;
 import org.apache.lens.server.api.authorization.ActionType;
 import org.apache.lens.server.api.authorization.Authorizer;
 import org.apache.lens.server.api.authorization.LensPrivilegeObject;
-
 import org.apache.lens.server.api.error.LensException;
 import org.apache.lens.server.api.metastore.DataCompletenessChecker;
 import org.apache.lens.server.api.util.LensUtil;
@@ -68,7 +65,6 @@
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
-
 import lombok.extern.slf4j.Slf4j;
 
 /**
@@ -339,11 +335,12 @@ public void verifyStorageExists(AbstractCubeTable cdt, String storage) throws Le
 
   public void createCubeFactTable(String cubeName, String factName, List<FieldSchema> columns,
     Map<String, Set<UpdatePeriod>> storageAggregatePeriods, double weight, Map<String, String> properties,
-    Map<String, StorageTableDesc> storageTableDescs, Map<String, Map<UpdatePeriod, String>> storageUpdatePeriodMap)
+    Map<String, StorageTableDesc> storageTableDescs, Map<String, Map<UpdatePeriod, String>> storageUpdatePeriodMap,
+    Map<String, Set<String>> storageTablePartitionColumns)
     throws LensException {
     checkIfAuthorized();
     CubeFactTable factTable = new CubeFactTable(cubeName, factName, columns, storageAggregatePeriods, weight,
-      properties, storageUpdatePeriodMap);
+      properties, storageUpdatePeriodMap, storageTablePartitionColumns);
     createCubeTable(factTable, storageTableDescs);
     // do a get to update cache
     getFactTable(factName);
@@ -410,7 +407,8 @@ public void createFactTable(XFact fact) throws LensException {
         xf.getWeight(),
         addFactColStartTimePropertyToFactProperties(xf),
         JAXBUtils.tableDescPrefixMapFromXStorageTables(xf.getStorageTables()),
-        JAXBUtils.storageTablePrefixMapOfStorage(xf.getStorageTables()));
+        JAXBUtils.storageTablePrefixMapOfStorage(xf.getStorageTables()),
+        JAXBUtils.getStorageTablePartitionColumnsFromTableDescs(xf.getStorageTables()));
     }
   }
 
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java
index 96948ccb6..4881640ab 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeVirtualFactTable.java
@@ -25,8 +25,8 @@
 import org.apache.hadoop.hive.ql.metadata.Table;
 
 import com.google.common.base.Optional;
-
 import com.google.common.collect.Lists;
+
 import lombok.Getter;
 import lombok.Setter;
 
@@ -195,4 +195,9 @@ public String getSourceFactName() {
   public Map<String, String> getSourceFactProperties() {
     return getSourceCubeFactTable().getProperties();
   }
+
+  @Override
+  public Set<String> getPartitionColumns(String storage) {
+    return new HashSet<String>();
+  }
 }
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java
index e351d7fd9..ab39e60d9 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/FactTable.java
@@ -86,6 +86,13 @@ public interface FactTable extends Named {
    */
   Set<String> getAllFieldNames();
 
+  /**
+   *  Set of all partition columns for the fact
+   * @param storage Name of the stroage
+   * @return Set of all partition columns
+   */
+  Set<String> getPartitionColumns(String storage);
+
   /**
    *tag for checking data completeness
    *
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/JAXBUtils.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/JAXBUtils.java
index 7dfb16683..623425e9d 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/JAXBUtils.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/JAXBUtils.java
@@ -710,12 +710,14 @@ public static CubeDimensionTable cubeDimTableFromDimTable(XDimensionTable dimens
   public static CubeFactTable cubeFactFromFactTable(XFactTable fact) throws LensException {
     List<FieldSchema> columns = fieldSchemaListFromColumns(fact.getColumns());
 
-    Map<String, Set<UpdatePeriod>> storageUpdatePeriods = getFactUpdatePeriodsFromStorageTables(
-      fact.getStorageTables());
-    Map<String, Map<UpdatePeriod, String>> storageTablePrefixMap = storageTablePrefixMapOfStorage(
-      fact.getStorageTables());
+    Map<String, Set<UpdatePeriod>> storageUpdatePeriods =
+            getFactUpdatePeriodsFromStorageTables(fact.getStorageTables());
+    Map<String, Map<UpdatePeriod, String>> storageTablePrefixMap =
+            storageTablePrefixMapOfStorage(fact.getStorageTables());
+    Map<String, Set<String>> storageTablePartitionColumns =
+            getStorageTablePartitionColumnsFromTableDescs(fact.getStorageTables());
     return new CubeFactTable(fact.getCubeName(), fact.getName(), columns, storageUpdatePeriods, fact.getWeight(),
-      mapFromXProperties(fact.getProperties()), storageTablePrefixMap);
+            mapFromXProperties(fact.getProperties()), storageTablePrefixMap, storageTablePartitionColumns);
   }
 
   public static CubeVirtualFactTable cubeVirtualFactFromFactTable(XVirtualFactTable fact, FactTable sourceFactTable)
@@ -893,6 +895,22 @@ public static Map<String, StorageTableDesc> tableDescPrefixMapFromXStorageTables
     return storageTablePrefixToDescMap;
   }
 
+  public static Map<String, Set<String>> getStorageTablePartitionColumnsFromTableDescs(XStorageTables storageTables) {
+    Map<String, Set<String>> storageTablePartitionColumns = new HashMap<>();
+    if (storageTables != null && !storageTables.getStorageTable().isEmpty()) {
+      for (XStorageTableElement xStorageTableElement : storageTables.getStorageTable()) {
+        Set<String> partitionColumns = new HashSet<>();
+        if (xStorageTableElement.getTableDesc() != null && xStorageTableElement.getTableDesc().getPartCols() != null) {
+          for (XColumn xColumn : xStorageTableElement.getTableDesc().getPartCols().getColumn()) {
+            partitionColumns.add(xColumn.getName());
+          }
+        }
+        storageTablePartitionColumns.put(xStorageTableElement.getStorageName(), partitionColumns);
+      }
+    }
+    return storageTablePartitionColumns;
+  }
+
   public static Map<String, Map<UpdatePeriod, String>> storageTablePrefixMapOfStorage(XStorageTables storageTables) {
     Map<String, Map<UpdatePeriod, String>> storageTableMap = new HashMap<>();
     if (storageTables != null && !storageTables.getStorageTable().isEmpty()) {
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java
index 44411f20b..2c25d614c 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/MetastoreUtil.java
@@ -374,6 +374,10 @@ public static String getValidColumnsKey(String name) {
     return getFactKeyPrefix(name) + VALID_COLUMNS_SFX;
   }
 
+  public static String getPartitionColumnKey(String name, String storage) {
+    return getFactKeyPrefix(name) + "." + storage + PARTCOLS_SFX;
+  }
+
   public static String getRestrictedColumnsKey(String name) {
     return getCubePrefix(name) + RESTRICTED_COLUMNS_SFX;
   }
diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
index 193ce6b3d..2f873deaa 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
@@ -20,31 +20,12 @@
 
 import static java.util.Comparator.naturalOrder;
 
-import static org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode;
-import static org.apache.lens.cube.parse.CandidateTablePruneCause.SkipUpdatePeriodCode;
-import static org.apache.lens.cube.parse.CandidateTablePruneCause.timeDimNotSupported;
-import static org.apache.lens.cube.parse.StorageUtil.getFallbackRange;
-import static org.apache.lens.cube.parse.StorageUtil.processCubeColForDataCompleteness;
-import static org.apache.lens.cube.parse.StorageUtil.processExpressionsForCompleteness;
+import static org.apache.lens.cube.parse.CandidateTablePruneCause.*;
+import static org.apache.lens.cube.parse.StorageUtil.*;
 
 import java.text.DateFormat;
 import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.OptionalDouble;
-import java.util.Set;
-import java.util.TimeZone;
-import java.util.TreeSet;
+import java.util.*;
 import java.util.stream.Stream;
 
 import org.apache.lens.cube.metadata.*;
@@ -348,6 +329,7 @@ public Collection<String> getColumns() {
       factColumns = fact.getValidColumns();
       if (factColumns == null) {
         factColumns = fact.getAllFieldNames();
+        factColumns.addAll(fact.getPartitionColumns(storageName));
       }
     }
     return factColumns;
diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
index 6f054c453..9c47e5fda 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
@@ -28,7 +28,6 @@
 import static org.testng.Assert.*;
 
 import java.text.SimpleDateFormat;
-
 import java.util.*;
 
 import org.apache.lens.cube.error.LensCubeErrorCode;
@@ -997,10 +996,12 @@ public void testUpdatePeriodTableDescriptions() throws LensException, HiveExcept
     Map<String, Map<UpdatePeriod, String>> storageUpdatePeriodMap = getHashMap(c1,
       getHashMap(HOURLY, HOURLY + "_" + c1, DAILY, DAILY + "_" + c1), c2, getHashMap(HOURLY, c2, DAILY, c2));
 
+    Map<String, Set<String>> storageTablePartitionColumns = getHashMap("pl_pi", new HashSet<String>(Arrays.asList("pt"
+            , "it")), "pl_p3", new HashSet<String>(Arrays.asList("pt", "et")));
     CubeFactTable cubeFact = new CubeFactTable(CUBE_NAME, factName, factColumns, updatePeriods, 0L, null,
-      storageUpdatePeriodMap);
+            storageUpdatePeriodMap, storageTablePartitionColumns);
     client.createCubeFactTable(CUBE_NAME, factName, factColumns, updatePeriods, 0L, null, storageTables,
-      storageUpdatePeriodMap);
+            storageUpdatePeriodMap, storageTablePartitionColumns);
 
     assertTrue(client.tableExists(factName));
     Table cubeTbl = client.getHiveTable(factName);

From 7783450f889550dd7ca85ae877f668b9d50e5793 Mon Sep 17 00:00:00 2001
From: Rajashekhar Choukimath <rajashekhar.c@inmobi.com>
Date: Mon, 27 Aug 2018 16:52:49 +0530
Subject: [PATCH 05/25] reverting old change

---
 .../src/main/java/org/apache/lens/server/LensServerConf.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lens-server/src/main/java/org/apache/lens/server/LensServerConf.java b/lens-server/src/main/java/org/apache/lens/server/LensServerConf.java
index 526239c2b..8d2a3fce4 100644
--- a/lens-server/src/main/java/org/apache/lens/server/LensServerConf.java
+++ b/lens-server/src/main/java/org/apache/lens/server/LensServerConf.java
@@ -42,7 +42,7 @@ private static final class ConfHolder {
     static {
       HIVE_CONF.addResource("lensserver-default.xml");
       HIVE_CONF.addResource("lens-site.xml");
-      Configuration conf = new Configuration(true);
+      Configuration conf = new Configuration(false);
       conf.addResource("lens-site.xml");
       for (Map.Entry<String, String> prop : conf) {
         if (!prop.getKey().startsWith(LensConfConstants.SERVER_PFX)) {

From 149a192cb6385f13b28445bb2d7cf01b50d7a55b Mon Sep 17 00:00:00 2001
From: Rajashekhar Choukimath <rajashekhar.c@inmobi.com>
Date: Mon, 27 Aug 2018 17:30:59 +0530
Subject: [PATCH 06/25] resolved imports

---
 .../lens/cube/parse/StorageCandidate.java     | 37 +++++++++++++++++--
 1 file changed, 33 insertions(+), 4 deletions(-)

diff --git a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
index 2f873deaa..ff67b8c63 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/parse/StorageCandidate.java
@@ -20,15 +20,44 @@
 
 import static java.util.Comparator.naturalOrder;
 
-import static org.apache.lens.cube.parse.CandidateTablePruneCause.*;
-import static org.apache.lens.cube.parse.StorageUtil.*;
+import static org.apache.lens.cube.parse.CandidateTablePruneCause.CandidateTablePruneCode;
+import static org.apache.lens.cube.parse.CandidateTablePruneCause.SkipUpdatePeriodCode;
+import static org.apache.lens.cube.parse.CandidateTablePruneCause.timeDimNotSupported;
+import static org.apache.lens.cube.parse.StorageUtil.getFallbackRange;
+import static org.apache.lens.cube.parse.StorageUtil.processCubeColForDataCompleteness;
+import static org.apache.lens.cube.parse.StorageUtil.processExpressionsForCompleteness;
 
 import java.text.DateFormat;
 import java.text.SimpleDateFormat;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.OptionalDouble;
+import java.util.Set;
+import java.util.TimeZone;
+import java.util.TreeSet;
 import java.util.stream.Stream;
 
-import org.apache.lens.cube.metadata.*;
+import org.apache.lens.cube.metadata.AbstractCubeTable;
+import org.apache.lens.cube.metadata.CubeFactTable;
+import org.apache.lens.cube.metadata.CubeInterface;
+import org.apache.lens.cube.metadata.DateUtil;
+import org.apache.lens.cube.metadata.Dimension;
+import org.apache.lens.cube.metadata.FactPartition;
+import org.apache.lens.cube.metadata.FactTable;
+import org.apache.lens.cube.metadata.MetastoreConstants;
+import org.apache.lens.cube.metadata.MetastoreUtil;
+import org.apache.lens.cube.metadata.TimeRange;
+import org.apache.lens.cube.metadata.UpdatePeriod;
 import org.apache.lens.server.api.error.LensException;
 import org.apache.lens.server.api.metastore.DataCompletenessChecker;
 

From c7aa7856c4ea31a6c30bb0cf6521bfc860665a7d Mon Sep 17 00:00:00 2001
From: Rajashekhar Choukimath <rajashekhar.c@im1552-x1.corp.inmobi.com>
Date: Mon, 10 Sep 2018 18:32:31 +0530
Subject: [PATCH 07/25] added Example for supporting the non-timed partition as
 filter while querying

---
 .../examples/PopulateSampleMetastore.java     |  1 +
 .../apache/lens/examples/SampleMetastore.java |  1 +
 .../src/main/resources/cube-queries.sql       |  2 +-
 .../src/main/resources/fact3-local-part.xml   | 19 +++++++
 .../src/main/resources/fact3-local1.data      |  8 +++
 .../src/main/resources/fact3-local2.data      |  8 +++
 lens-examples/src/main/resources/fact3.xml    | 51 +++++++++++++++++++
 .../src/main/resources/sample-cube.xml        |  1 +
 8 files changed, 90 insertions(+), 1 deletion(-)
 create mode 100644 lens-examples/src/main/resources/fact3-local-part.xml
 create mode 100644 lens-examples/src/main/resources/fact3-local1.data
 create mode 100644 lens-examples/src/main/resources/fact3-local2.data
 create mode 100644 lens-examples/src/main/resources/fact3.xml

diff --git a/lens-examples/src/main/java/org/apache/lens/examples/PopulateSampleMetastore.java b/lens-examples/src/main/java/org/apache/lens/examples/PopulateSampleMetastore.java
index 7c8fd2cc5..5fb57dcbb 100644
--- a/lens-examples/src/main/java/org/apache/lens/examples/PopulateSampleMetastore.java
+++ b/lens-examples/src/main/java/org/apache/lens/examples/PopulateSampleMetastore.java
@@ -190,6 +190,7 @@ public void populateFactTables() throws Exception {
     createFactPartition("fact2-local-part1.xml", "fact2", "local");
     createFactPartition("fact2-local-part2.xml", "fact2", "local");
     createFactPartition("fact2-local-part3.xml", "fact2", "local");
+    createFactPartitions("fact3-local-part.xml", "fact3", "local");
     createFactPartition("raw-local-part1.xml", "rawfact", "local");
     createFactPartition("raw-local-part2.xml", "rawfact", "local");
     createFactPartition("raw-local-part3.xml", "rawfact", "local");
diff --git a/lens-examples/src/main/java/org/apache/lens/examples/SampleMetastore.java b/lens-examples/src/main/java/org/apache/lens/examples/SampleMetastore.java
index 1aa993b6d..ac7470b08 100644
--- a/lens-examples/src/main/java/org/apache/lens/examples/SampleMetastore.java
+++ b/lens-examples/src/main/java/org/apache/lens/examples/SampleMetastore.java
@@ -167,6 +167,7 @@ private void createFact(String factSpec) {
   private void createFacts() throws JAXBException, IOException {
     createFact("fact1.xml");
     createFact("fact2.xml");
+    createFact("fact3.xml");
     createFact("rawfact.xml");
     createFact("sales-raw-fact.xml");
     createFact("sales-aggr-fact1.xml");
diff --git a/lens-examples/src/main/resources/cube-queries.sql b/lens-examples/src/main/resources/cube-queries.sql
index 016917589..f3d6cc7fe 100644
--- a/lens-examples/src/main/resources/cube-queries.sql
+++ b/lens-examples/src/main/resources/cube-queries.sql
@@ -16,7 +16,7 @@
 -- specific language governing permissions and limitations
 -- under the License.
 --
-
+cube select dim1, measure2 from sample_cube where time_range_in(dt, '2014-06-25-00', '2014-06-26-00') and nontimedim = 'nonTimeDimValue2'
 cube select measure2 from sample_cube where time_range_in(dt, '2014-06-24-23', '2014-06-25-00')
 cube select measure2 from sample_cube where time_range_in(dt, '2014-06-25-00', '2014-06-26-00')
 cube select measure2 from sample_cube where time_range_in(dt, '2014-06-24-23', '2014-06-26-01')
diff --git a/lens-examples/src/main/resources/fact3-local-part.xml b/lens-examples/src/main/resources/fact3-local-part.xml
new file mode 100644
index 000000000..05ed81517
--- /dev/null
+++ b/lens-examples/src/main/resources/fact3-local-part.xml
@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<x_partition_list xmlns="uri:lens:cube:0.1">
+    <partition fact_or_dimension_table_name="fact3" location="examples/data/fact3-local1" update_period="DAILY">
+        <non_time_partition_spec>
+            <part_spec_element key="nontimedim" value="nonTimeDimValue1"/>
+        </non_time_partition_spec>
+        <time_partition_spec>
+            <part_spec_element key="dt" value="2014-06-25T00:00:00"/>
+        </time_partition_spec>
+    </partition>
+    <partition fact_or_dimension_table_name="fact3" location="examples/data/fact3-local2" update_period="DAILY">
+        <non_time_partition_spec>
+            <part_spec_element key="nontimedim" value="nonTimeDimValue2"/>
+        </non_time_partition_spec>
+        <time_partition_spec>
+            <part_spec_element key="dt" value="2014-06-25T00:00:00"/>
+        </time_partition_spec>
+    </partition>
+</x_partition_list>
diff --git a/lens-examples/src/main/resources/fact3-local1.data b/lens-examples/src/main/resources/fact3-local1.data
new file mode 100644
index 000000000..273765d64
--- /dev/null
+++ b/lens-examples/src/main/resources/fact3-local1.data
@@ -0,0 +1,8 @@
+21,1,100,100,100
+22,2,200,200,200
+23,3,300,300,300
+24,4,400,400,400
+25,5,500,500,500
+26,6,600,600,600
+27,7,700,700,700
+28,8,800,800,800
diff --git a/lens-examples/src/main/resources/fact3-local2.data b/lens-examples/src/main/resources/fact3-local2.data
new file mode 100644
index 000000000..2f7f83149
--- /dev/null
+++ b/lens-examples/src/main/resources/fact3-local2.data
@@ -0,0 +1,8 @@
+21,1,2400,100,100
+22,2,4800,200,200
+23,3,7200,300,300
+24,4,9600,400,400
+25,5,12000,500,500
+26,6,14400,600,600
+27,7,16800,700,700
+28,8,19200,800,800
diff --git a/lens-examples/src/main/resources/fact3.xml b/lens-examples/src/main/resources/fact3.xml
new file mode 100644
index 000000000..a8a93f881
--- /dev/null
+++ b/lens-examples/src/main/resources/fact3.xml
@@ -0,0 +1,51 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements. See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership. The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License. You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied. See the License for the
+  specific language governing permissions and limitations
+  under the License.
+
+-->
+<x_fact_table cube_name="sample_cube" name="fact3" weight="100.0" xmlns="uri:lens:cube:0.1"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="uri:lens:cube:0.1 cube-0.1.xsd ">
+  <columns>
+    <column comment="" name="dim1" _type="INT"/>
+    <column comment="" name="measure2" _type="BIGINT"/>
+    <column comment="" name="measure3" _type="INT"/>
+    <column comment="" name="measure4" _type="FLOAT"/>
+    <column comment="" name="measure5" _type="FLOAT" start_time="2015-01-01"/>
+    <column comment="" name="measure6" _type="FLOAT" end_time="2015-01-01"/>
+  </columns>
+  <properties>
+    <property name="cube.fact.is.aggregated" value="true"/>
+  </properties>
+  <storage_tables>
+    <storage_table>
+      <update_periods>
+        <update_period>HOURLY</update_period>
+        <update_period>DAILY</update_period>
+      </update_periods>
+      <storage_name>local</storage_name>
+      <table_desc external="true" field_delimiter="," table_location="/tmp/examples/fact3_local">
+        <part_cols>
+          <column comment="Time column" name="dt" _type="STRING"/>
+          <column comment="Non Time column" name="nontimedim" _type="STRING"/>
+        </part_cols>
+        <time_part_cols>dt</time_part_cols>
+      </table_desc>
+    </storage_table>
+  </storage_tables>
+</x_fact_table>
diff --git a/lens-examples/src/main/resources/sample-cube.xml b/lens-examples/src/main/resources/sample-cube.xml
index 6b17da535..464fb0bdb 100644
--- a/lens-examples/src/main/resources/sample-cube.xml
+++ b/lens-examples/src/main/resources/sample-cube.xml
@@ -53,6 +53,7 @@
         <property name="is_ui_visible" value="true"/>
       </tags>
     </dim_attribute>
+    <dim_attribute name="nonTimeDim" _type="STRING"/>
   </dim_attributes>
   <expressions>
     <expression name="expr_msr5" _type="DOUBLE">

From c0cb51ae1c86063baf3247cc9bbd4358b8ce3a89 Mon Sep 17 00:00:00 2001
From: Rajashekhar <rajashekhar.c@inmobi.com>
Date: Thu, 20 Sep 2018 17:54:14 +0530
Subject: [PATCH 08/25] fixed the yaml

---
 lens-examples/src/test/resources/yaml/sample-cube.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lens-examples/src/test/resources/yaml/sample-cube.yaml b/lens-examples/src/test/resources/yaml/sample-cube.yaml
index 5ccc1af60..6aa74bd1c 100644
--- a/lens-examples/src/test/resources/yaml/sample-cube.yaml
+++ b/lens-examples/src/test/resources/yaml/sample-cube.yaml
@@ -33,6 +33,7 @@ dimAttributes:
     startTime: 2013-12-01T00:00:00
   dim3: type: INT
   dim4: tags:, category: primary, is_ui_visible: true, type: BIGINT
+  nonTimeDim: type: STRING
 expressions:
   expr_msr5:
     exprSpec:

From 65174d27d68a5cf01a1bc277d787c2de191ee123 Mon Sep 17 00:00:00 2001
From: Rajashekhar <rajashekhar.c@inmobi.com>
Date: Thu, 20 Sep 2018 18:41:10 +0530
Subject: [PATCH 09/25] licence

---
 .../src/main/resources/fact3-local-part.xml   | 20 +++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/lens-examples/src/main/resources/fact3-local-part.xml b/lens-examples/src/main/resources/fact3-local-part.xml
index 05ed81517..ba52c3f37 100644
--- a/lens-examples/src/main/resources/fact3-local-part.xml
+++ b/lens-examples/src/main/resources/fact3-local-part.xml
@@ -1,4 +1,24 @@
 <?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements. See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership. The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License. You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied. See the License for the
+  specific language governing permissions and limitations
+  under the License.
+
+-->
 <x_partition_list xmlns="uri:lens:cube:0.1">
     <partition fact_or_dimension_table_name="fact3" location="examples/data/fact3-local1" update_period="DAILY">
         <non_time_partition_spec>

From 0519d9d59ee06e362f967288e7d508a8f932b461 Mon Sep 17 00:00:00 2001
From: Rajashekhar <rajashekhar.c@inmobi.com>
Date: Tue, 2 Jul 2019 23:12:25 +0530
Subject: [PATCH 10/25] made few changes to the code for azure deployment

---
 .../cube/metadata/CubeMetastoreClient.java    | 33 +++----
 .../org/apache/lens/driver/es/ESDriver.java   |  8 --
 .../apache/lens/driver/hive/HiveDriver.java   | 12 ---
 .../lens/driver/hive/TestHiveDriver.java      |  1 -
 .../apache/lens/driver/jdbc/JDBCDriver.java   | 85 ++-----------------
 .../lens/server/api/driver/LensDriver.java    |  9 --
 .../api/query/PreparedQueryContext.java       | 19 +++--
 .../lens/server/api/driver/MockDriver.java    | 11 ---
 .../lens/server/query/LensServerDAO.java      | 30 +++++++
 .../query/QueryExecutionServiceImpl.java      | 14 ++-
 pom.xml                                       |  2 +-
 11 files changed, 72 insertions(+), 152 deletions(-)

diff --git a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
index 34069d9a7..357d4debc 100644
--- a/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
+++ b/lens-cube/src/main/java/org/apache/lens/cube/metadata/CubeMetastoreClient.java
@@ -1830,30 +1830,19 @@ public XFact getXFactTable(FactTable ft) throws LensException {
         for (UpdatePeriod updatePeriod : updatePeriods) {
           tableNames.add(updatePeriodToTableMap.get(updatePeriod));
         }
-        if (tableNames.size() <= 1) {
-          XStorageTableElement tblElement = JAXBUtils.getXStorageTableFromHiveTable(
-            getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(cft.getName(), storageName)));
-          tblElement.setStorageName(storageName);
-          for (UpdatePeriod p : updatePeriods) {
-            tblElement.getUpdatePeriods().getUpdatePeriod().add(XUpdatePeriod.valueOf(p.name()));
-          }
-          factTable.getStorageTables().getStorageTable().add(tblElement);
-        } else {
-          // Multiple storage tables.
-          XStorageTableElement tblElement = new XStorageTableElement();
-          tblElement.setStorageName(storageName);
-          XUpdatePeriods xUpdatePeriods = new XUpdatePeriods();
-          tblElement.setUpdatePeriods(xUpdatePeriods);
-          for (Map.Entry entry : updatePeriodToTableMap.entrySet()) {
-            XUpdatePeriodTableDescriptor updatePeriodTableDescriptor = new XUpdatePeriodTableDescriptor();
-            updatePeriodTableDescriptor.setTableDesc(getStorageTableDescFromHiveTable(
+        XStorageTableElement tblElement = new XStorageTableElement();
+        tblElement.setStorageName(storageName);
+        XUpdatePeriods xUpdatePeriods = new XUpdatePeriods();
+        tblElement.setUpdatePeriods(xUpdatePeriods);
+        for (Map.Entry entry : updatePeriodToTableMap.entrySet()) {
+          XUpdatePeriodTableDescriptor updatePeriodTableDescriptor = new XUpdatePeriodTableDescriptor();
+          updatePeriodTableDescriptor.setTableDesc(getStorageTableDescFromHiveTable(
               this.getHiveTable(MetastoreUtil.getFactOrDimtableStorageTableName(cft.getName(),
-                (String) entry.getValue()))));
-            updatePeriodTableDescriptor.setUpdatePeriod(XUpdatePeriod.valueOf(((UpdatePeriod) entry.getKey()).name()));
-            xUpdatePeriods.getUpdatePeriodTableDescriptor().add(updatePeriodTableDescriptor);
-          }
-          factTable.getStorageTables().getStorageTable().add(tblElement);
+                  (String) entry.getValue()))));
+          updatePeriodTableDescriptor.setUpdatePeriod(XUpdatePeriod.valueOf(((UpdatePeriod) entry.getKey()).name()));
+          xUpdatePeriods.getUpdatePeriodTableDescriptor().add(updatePeriodTableDescriptor);
         }
+        factTable.getStorageTables().getStorageTable().add(tblElement);
       }
       fact = factTable;
     }
diff --git a/lens-driver-es/src/main/java/org/apache/lens/driver/es/ESDriver.java b/lens-driver-es/src/main/java/org/apache/lens/driver/es/ESDriver.java
index e735c0c4b..5a7c6f044 100644
--- a/lens-driver-es/src/main/java/org/apache/lens/driver/es/ESDriver.java
+++ b/lens-driver-es/src/main/java/org/apache/lens/driver/es/ESDriver.java
@@ -28,7 +28,6 @@
 import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.lens.api.query.QueryHandle;
-import org.apache.lens.api.query.QueryPrepareHandle;
 import org.apache.lens.cube.metadata.CubeMetastoreClient;
 import org.apache.lens.cube.parse.HQLParser;
 import org.apache.lens.driver.es.client.ESClient;
@@ -119,13 +118,6 @@ public DriverQueryPlan explainAndPrepare(PreparedQueryContext pContext) throws L
     return explain(pContext);
   }
 
-  @Override
-  public void closePreparedQuery(QueryPrepareHandle handle) {
-    /**
-     * Elastic search does not have a concept of prepared query.
-     */
-  }
-
   @Override
   public LensResultSet execute(QueryContext context) throws LensException {
     handleContextMap.put(context.getQueryHandle(), context);
diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
index 61fef15a3..562e139da 100644
--- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
+++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
@@ -34,7 +34,6 @@
 import org.apache.lens.api.LensSessionHandle;
 import org.apache.lens.api.Priority;
 import org.apache.lens.api.query.QueryHandle;
-import org.apache.lens.api.query.QueryPrepareHandle;
 import org.apache.lens.cube.query.cost.FactPartitionBasedQueryCostCalculator;
 import org.apache.lens.server.api.LensConfConstants;
 import org.apache.lens.server.api.driver.*;
@@ -467,17 +466,6 @@ public void prepare(PreparedQueryContext pContext) throws LensException {
     // NO OP
   }
 
-  /*
-   * (non-Javadoc)
-   *
-   * @see org.apache.lens.server.api.driver.LensDriver#closePreparedQuery(org.apache.lens.api.query.QueryPrepareHandle)
-   */
-  @Override
-  public void closePreparedQuery(QueryPrepareHandle handle) throws LensException {
-    // NO OP
-
-  }
-
   /*
    * (non-Javadoc)
    *
diff --git a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
index 49bfb683e..05f97178b 100644
--- a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
+++ b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
@@ -757,7 +757,6 @@ public void testExplain() throws Exception {
     validateExecuteAsync(qctx, DriverQueryState.SUCCESSFUL, true, false);
 
     driver.closeQuery(qctx.getQueryHandle());
-    driver.closePreparedQuery(pctx.getPrepareHandle());
     assertHandleSize(handleSize);
   }
 
diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
index 8dd299a11..fbacf544e 100644
--- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
+++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
@@ -31,14 +31,11 @@
 import java.io.ObjectOutput;
 import java.sql.*;
 import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Map;
 import java.util.concurrent.*;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.lens.api.LensConf;
 import org.apache.lens.api.query.QueryHandle;
-import org.apache.lens.api.query.QueryPrepareHandle;
 import org.apache.lens.cube.parse.HQLParser;
 import org.apache.lens.cube.query.cost.StaticCostCalculator;
 import org.apache.lens.server.api.driver.*;
@@ -624,17 +621,9 @@ public void validate(AbstractQueryContext pContext) throws LensException {
     boolean validateThroughPrepare = pContext.getDriverConf(this).getBoolean(JDBC_VALIDATE_THROUGH_PREPARE,
       DEFAULT_JDBC_VALIDATE_THROUGH_PREPARE);
     if (validateThroughPrepare) {
-      PreparedStatement stmt;
       // Estimate queries need to get connection from estimate pool to make sure
       // we are not blocked by data queries.
-      stmt = prepareInternal(pContext, true, true, "validate-");
-      if (stmt != null) {
-        try {
-          stmt.close();
-        } catch (SQLException e) {
-          throw new LensException();
-        }
-      }
+      prepareInternal(pContext, true, true, "validate-");
     }
   }
 
@@ -692,8 +681,6 @@ protected final ConnectionProvider getConnectionProvider() {
     return connectionProvider;
   }
 
-  private final Map<QueryPrepareHandle, PreparedStatement> preparedQueries = new HashMap<>();
-
   /**
    * Internally prepare the query
    *
@@ -701,12 +688,12 @@ protected final ConnectionProvider getConnectionProvider() {
    * @return prepared statement of the query
    * @throws LensException
    */
-  private PreparedStatement prepareInternal(AbstractQueryContext pContext) throws LensException {
+  private void prepareInternal(AbstractQueryContext pContext) throws LensException {
     if (pContext.getDriverQuery(this) == null) {
       throw new NullPointerException("Null driver query for " + pContext.getUserQuery());
     }
     checkConfigured();
-    return prepareInternal(pContext, false, false, "prepare-");
+    prepareInternal(pContext, false, false, "prepare-");
   }
 
 
@@ -719,7 +706,7 @@ private PreparedStatement prepareInternal(AbstractQueryContext pContext) throws
    * @return prepared statement
    * @throws LensException
    */
-  private PreparedStatement prepareInternal(AbstractQueryContext pContext,
+  private void prepareInternal(AbstractQueryContext pContext,
     boolean calledForEstimate,
     boolean checkConfigured,
     String metricCallStack) throws LensException {
@@ -736,33 +723,10 @@ private PreparedStatement prepareInternal(AbstractQueryContext pContext,
     MethodMetricsContext sqlRewriteGauge = MethodMetricsFactory.createMethodGauge(pContext.getDriverConf(this), true,
       metricCallStack + COLUMNAR_SQL_REWRITE_GAUGE);
     String rewrittenQuery = rewriteQuery(pContext);
+    pContext.setSelectedDriverQuery(rewrittenQuery);
     sqlRewriteGauge.markSuccess();
     MethodMetricsContext jdbcPrepareGauge = MethodMetricsFactory.createMethodGauge(pContext.getDriverConf(this), true,
       metricCallStack + JDBC_PREPARE_GAUGE);
-
-    PreparedStatement stmt = null;
-    Connection conn = null;
-    try {
-      conn = calledForEstimate ? getEstimateConnection() : getConnection();
-      stmt = conn.prepareStatement(rewrittenQuery);
-      if (!pContext.getDriverConf(this).getBoolean(JDBC_VALIDATE_SKIP_WARNINGS,
-        DEFAULT_JDBC_VALIDATE_SKIP_WARNINGS) && stmt.getWarnings() != null) {
-        throw new LensException(stmt.getWarnings());
-      }
-    } catch (SQLException sql) {
-      handleJDBCSQLException(sql);
-    } finally {
-      if (conn != null) {
-        try {
-          conn.close();
-        } catch (SQLException e) {
-          log.error("Error closing connection: {}", rewrittenQuery, e);
-        }
-      }
-      jdbcPrepareGauge.markSuccess();
-    }
-    log.info("Prepared: {}", rewrittenQuery);
-    return stmt;
   }
 
   /**
@@ -797,14 +761,7 @@ private String getSqlSynataxExceptions(Throwable e) {
    */
   @Override
   public void prepare(PreparedQueryContext pContext) throws LensException {
-    if (preparedQueries.containsKey(pContext.getPrepareHandle())) {
-      // already prepared
-      return;
-    }
-    PreparedStatement stmt = prepareInternal(pContext);
-    if (stmt != null) {
-      preparedQueries.put(pContext.getPrepareHandle(), stmt);
-    }
+    prepareInternal(pContext);
   }
 
   /**
@@ -821,24 +778,6 @@ public DriverQueryPlan explainAndPrepare(PreparedQueryContext pContext) throws L
     return new JDBCQueryPlan(calculateQueryCost(pContext));
   }
 
-  /**
-   * Close the prepare query specified by the prepared handle, releases all the resources held by the prepared query.
-   *
-   * @param handle The query handle
-   * @throws LensException the lens exception
-   */
-  @Override
-  public void closePreparedQuery(QueryPrepareHandle handle) throws LensException {
-    checkConfigured();
-    try {
-      if (preparedQueries.get(handle) != null) {
-        preparedQueries.get(handle).close();
-      }
-    } catch (SQLException e) {
-      throw new LensException(e);
-    }
-  }
-
   /**
    * Blocking execute of the query.
    *
@@ -887,6 +826,7 @@ public void executeAsync(QueryContext context) throws LensException {
     String rewrittenQuery = rewriteQuery(context);
     JdbcQueryContext jdbcCtx = new JdbcQueryContext(context);
     jdbcCtx.setRewrittenQuery(rewrittenQuery);
+    context.setSelectedDriverQuery(rewrittenQuery);
     try {
       Future<QueryResult> future = asyncQueryPool.submit(new QueryCallable(jdbcCtx, logSegregationContext));
       jdbcCtx.setResultFuture(future);
@@ -1021,17 +961,6 @@ public void close() throws LensException {
           log.warn("{} Error closing query : {}", getFullyQualifiedName(), query.getHandleId(), e);
         }
       }
-      for (QueryPrepareHandle query : preparedQueries.keySet()) {
-        try {
-          try {
-            preparedQueries.get(query).close();
-          } catch (SQLException e) {
-            throw new LensException();
-          }
-        } catch (LensException e) {
-          log.warn("{} Error closing prapared query : {}", getFullyQualifiedName(), query, e);
-        }
-      }
     } finally {
       queryContextMap.clear();
     }
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java
index 0080b97b9..0fbd8d428 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java
@@ -22,7 +22,6 @@
 
 import org.apache.lens.api.Priority;
 import org.apache.lens.api.query.QueryHandle;
-import org.apache.lens.api.query.QueryPrepareHandle;
 import org.apache.lens.server.api.driver.hooks.DriverQueryHook;
 import org.apache.lens.server.api.error.LensException;
 import org.apache.lens.server.api.events.LensEventListener;
@@ -97,14 +96,6 @@ public interface LensDriver extends Externalizable {
    */
   DriverQueryPlan explainAndPrepare(PreparedQueryContext pContext) throws LensException;
 
-  /**
-   * Close the prepare query specified by the prepared handle, releases all the resources held by the prepared query.
-   *
-   * @param handle The query handle
-   * @throws LensException the lens exception
-   */
-  void closePreparedQuery(QueryPrepareHandle handle) throws LensException;
-
   /**
    * Blocking execute of the query
    * <p></p>
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java
index 0b08459c1..69ee7c4fe 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/PreparedQueryContext.java
@@ -49,10 +49,17 @@ public class PreparedQueryContext extends AbstractQueryContext implements Delaye
   private final QueryPrepareHandle prepareHandle;
 
   /**
-   * The prepared time.
+   * The prepare start time.
    */
   @Getter
-  private final Date preparedTime;
+  private final Date prepareStartTime;
+
+  /**
+   * The prepare end time.
+   */
+  @Getter
+  @Setter
+  private Date prepareEndTime = null;
 
   /**
    * The prepared user.
@@ -94,7 +101,7 @@ public PreparedQueryContext(String query, String user, Configuration conf, Colle
   public PreparedQueryContext(String query, String user, Configuration conf, LensConf qconf, Collection<LensDriver>
     drivers) {
     super(query, user, qconf, conf, drivers, true);
-    this.preparedTime = new Date();
+    this.prepareStartTime = new Date();
     this.preparedUser = user;
     this.prepareHandle = new QueryPrepareHandle(UUID.randomUUID());
     this.conf = conf;
@@ -119,9 +126,9 @@ public int compareTo(Delayed o) {
   @Override
   public long getDelay(TimeUnit units) {
     long delayMillis;
-    if (this.preparedTime != null) {
+    if (this.prepareStartTime != null) {
       Date now = new Date();
-      long elapsedMills = now.getTime() - this.preparedTime.getTime();
+      long elapsedMills = now.getTime() - this.prepareStartTime.getTime();
       delayMillis = millisInWeek - elapsedMills;
       return units.convert(delayMillis, TimeUnit.MILLISECONDS);
     } else {
@@ -147,7 +154,7 @@ public void updateConf(Map<String, String> confoverlay) {
    * @return the lens prepared query
    */
   public LensPreparedQuery toPreparedQuery() {
-    return new LensPreparedQuery(prepareHandle, userQuery, preparedTime, preparedUser, getDriverContext()
+    return new LensPreparedQuery(prepareHandle, userQuery, prepareStartTime, preparedUser, getDriverContext()
         .getSelectedDriver() != null ? getDriverContext().getSelectedDriver().getFullyQualifiedName() : null,
         getDriverContext().getSelectedDriverQuery(), lensConf);
   }
diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java b/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java
index 92724c6a0..f8c25bcce 100644
--- a/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java
+++ b/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java
@@ -271,17 +271,6 @@ public DriverQueryPlan explainAndPrepare(PreparedQueryContext pContext) throws L
     return p;
   }
 
-  /*
-   * (non-Javadoc)
-   *
-   * @see org.apache.lens.server.api.driver.LensDriver#closePreparedQuery(org.apache.lens.api.query.QueryPrepareHandle)
-   */
-  @Override
-  public void closePreparedQuery(QueryPrepareHandle handle) throws LensException {
-    // TODO Auto-generated method stub
-
-  }
-
   /*
    * (non-Javadoc)
    *
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java b/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
index 983365d7b..24748fc14 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
@@ -35,6 +35,7 @@
 import org.apache.lens.api.query.QueryStatus;
 import org.apache.lens.server.api.error.LensException;
 import org.apache.lens.server.api.query.FinishedLensQuery;
+import org.apache.lens.server.api.query.PreparedQueryContext;
 import org.apache.lens.server.api.query.QueryContext;
 import org.apache.lens.server.session.LensSessionImpl;
 import org.apache.lens.server.util.UtilityMethods;
@@ -47,6 +48,7 @@
 import org.apache.hadoop.conf.Configuration;
 
 import com.google.common.collect.Lists;
+
 import lombok.extern.slf4j.Slf4j;
 
 /**
@@ -821,4 +823,32 @@ public boolean deleteActiveSession(LensSessionHandle sessionId) throws LensExcep
 
     return result;
   }
+
+  /**
+   * DAO method to insert a new Prepared query into Table.
+   *
+   * @param preparedQueryContext to be inserted
+   * @throws SQLException the exception
+   */
+  public void insertPreparedQuery(PreparedQueryContext preparedQueryContext) throws SQLException {
+    String sql = "insert into prepared_queries (handle, userquery, submitter, timetaken, queryname, drivername, "
+        + "driverquery, starttime)" + " values (?,?,?,?,?,?,?)";
+    Connection conn = null;
+    try {
+      conn = getConnection();
+      conn.setAutoCommit(false);
+      QueryRunner runner = new QueryRunner();
+
+      long timeTaken =
+          preparedQueryContext.getPrepareEndTime().getTime() - preparedQueryContext.getPrepareStartTime().getTime();
+
+      runner.update(conn, sql, preparedQueryContext.getPrepareHandle().getQueryHandleString(),
+          preparedQueryContext.getUserQuery(), preparedQueryContext.getSubmittedUser(), timeTaken,
+          preparedQueryContext.getQueryName(), preparedQueryContext.getDriverContext().getSelectedDriver(),
+          preparedQueryContext.getSelectedDriverQuery(), preparedQueryContext.getPrepareStartTime());
+      conn.commit();
+    } finally {
+      DbUtils.closeQuietly(conn);
+    }
+  }
 }
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
index 9e5f2e6dc..623d3eb66 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
@@ -149,6 +149,8 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE
    */
   public static final String PREPARED_QUERY_PURGER_COUNTER = "prepared-query-purger-errors";
 
+  public static final String PREPARED_QUERY_INSERT_COUNTER = "prepared-query-insert-errors";
+
   /**
    * The millis in week.
    */
@@ -2055,6 +2057,12 @@ public QueryPrepareHandle prepare(LensSessionHandle sessionHandle, String query,
       prepared = prepareQuery(sessionHandle, query, lensConf, SubmitOp.PREPARE);
       prepared.setQueryName(queryName);
       prepared.getSelectedDriver().prepare(prepared);
+      try {
+        lensServerDao.insertPreparedQuery(prepared);
+      } catch (Exception e) {
+        incrCounter(PREPARED_QUERY_INSERT_COUNTER);
+      }
+
       return prepared.getPrepareHandle();
     } catch (LensException e) {
       if (prepared != null) {
@@ -2087,6 +2095,7 @@ private PreparedQueryContext prepareQuery(LensSessionHandle sessionHandle, Strin
     preparedQueries.put(prepared.getPrepareHandle(), prepared);
     preparedQueryQueue.add(prepared);
     incrCounter(PREPARED_QUERIES_COUNTER);
+    prepared.setPrepareEndTime(new Date());
     return prepared;
   }
 
@@ -3031,7 +3040,7 @@ public List<QueryPrepareHandle> getAllPreparedQueries(LensSessionHandle sessionH
             continue;
           }
         }
-        long queryPrepTime = preparedQueryContext.getPreparedTime().getTime();
+        long queryPrepTime = preparedQueryContext.getPrepareStartTime().getTime();
         if (fromTime <= queryPrepTime && queryPrepTime < toTime) {
           continue;
         }
@@ -3080,9 +3089,6 @@ public boolean destroyPrepared(LensSessionHandle sessionHandle, QueryPrepareHand
    * @throws LensException the lens exception
    */
   private void destroyPreparedQuery(PreparedQueryContext ctx) throws LensException {
-    if (ctx.getSelectedDriver() != null) {
-      ctx.getSelectedDriver().closePreparedQuery(ctx.getPrepareHandle());
-    }
     preparedQueries.remove(ctx.getPrepareHandle());
     preparedQueryQueue.remove(ctx);
     decrCounter(PREPARED_QUERIES_COUNTER);
diff --git a/pom.xml b/pom.xml
index 9525332af..3deff5df4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -103,7 +103,7 @@
     <war.plugin.version>2.1.1</war.plugin.version>
     <license.plugin.version>2.6</license.plugin.version>
     <buildnumber.plugin.version>1.0</buildnumber.plugin.version>
-    <findbugs.plugin.version>3.0.1</findbugs.plugin.version>
+    <findbugs.plugin.version>3.0.4</findbugs.plugin.version>
     <antrun.plugin.version>1.8</antrun.plugin.version>
     <cobertura.plugin.version>2.7</cobertura.plugin.version>
 

From bc6f628facb8e7b0b7967e06f5af1358fa6cc4b4 Mon Sep 17 00:00:00 2001
From: Rajashekhar <rajashekhar.c@inmobi.com>
Date: Wed, 3 Jul 2019 17:51:44 +0530
Subject: [PATCH 11/25] Fixed the test case, removed instances where Prepare
 was hitting the underlying JDBC Driver

---
 .../lens/driver/jdbc/TestJdbcDriver.java      | 35 -------------------
 1 file changed, 35 deletions(-)

diff --git a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
index c08b04a27..6be4ac208 100644
--- a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
+++ b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
@@ -293,22 +293,6 @@ public void testEstimate() throws Exception {
 
   }
 
-  /**
-   * Test estimate failing
-   *
-   * @throws Exception the exception
-   */
-  @Test
-  public void testEstimateFailing() throws Exception {
-    String query2 = "SELECT * FROM estimate_test2"; // Select query against non existing table
-    try {
-      driver.estimate(createExplainContext(query2, baseConf));
-      Assert.fail("Running estimate on a non existing table.");
-    } catch (LensException ex) {
-      Assert.assertEquals(LensUtil.getCauseMessage(ex), "user lacks privilege or object not found: ESTIMATE_TEST2");
-    }
-  }
-
   /**
    * Test estimate failing
    *
@@ -326,7 +310,6 @@ public void testEstimateGauges() throws Exception {
     String driverQualifiledName = driver.getFullyQualifiedName();
     Assert.assertTrue(reg.getGauges().keySet().containsAll(Arrays.asList(
       "lens.MethodMetricGauge.TestJdbcDriver-"+driverQualifiledName+"-validate-columnar-sql-rewrite",
-      "lens.MethodMetricGauge.TestJdbcDriver-"+driverQualifiledName+"-validate-jdbc-prepare-statement",
       "lens.MethodMetricGauge.TestJdbcDriver-"+driverQualifiledName+"-validate-thru-prepare",
       "lens.MethodMetricGauge.TestJdbcDriver-"+driverQualifiledName+"-jdbc-check-allowed-query")));
   }
@@ -359,7 +342,6 @@ public void testMetricsEnabled() throws Exception {
     cost = driver.estimate(pContext2);
     Assert.assertEquals(cost, JDBC_COST);
     driver.prepare(pContext2);
-    driver.explainAndPrepare(pContext2);
   }
 
   /**
@@ -653,23 +635,6 @@ public void testPrepare() throws Exception {
     driver.prepare(pContext);
   }
 
-  /**
-   * Test prepare failing
-   *
-   * @throws Exception the exception
-   */
-  @Test
-  public void testPrepareFailing() throws Exception {
-    String query = "SELECT * FROM prepare_test2"; // Select query against non existing table
-    try {
-      PreparedQueryContext pContext = new PreparedQueryContext(query, "SA", baseConf, drivers);
-      driver.prepare(pContext);
-      Assert.fail("Running prepare on a non existing table.");
-    } catch (LensException ex) {
-      Assert.assertEquals(LensUtil.getCauseMessage(ex), "user lacks privilege or object not found: PREPARE_TEST2");
-    }
-  }
-
   /**
    * Test prepare skip warnings
    *

From 2d6fb3dd5a214f14facfa5ef24e422ad56899556 Mon Sep 17 00:00:00 2001
From: Rajashekhar <rajashekhar.c@inmobi.com>
Date: Mon, 8 Jul 2019 16:55:23 +0530
Subject: [PATCH 12/25] reversed few changes

---
 .../java/org/apache/lens/driver/es/ESDriver.java    |  8 ++++++++
 .../org/apache/lens/driver/hive/HiveDriver.java     | 12 ++++++++++++
 .../org/apache/lens/driver/hive/TestHiveDriver.java |  1 +
 .../apache/lens/server/api/driver/LensDriver.java   |  9 +++++++++
 .../apache/lens/server/api/driver/MockDriver.java   | 11 +++++++++++
 .../server/query/QueryExecutionServiceImpl.java     | 13 +++++++------
 6 files changed, 48 insertions(+), 6 deletions(-)

diff --git a/lens-driver-es/src/main/java/org/apache/lens/driver/es/ESDriver.java b/lens-driver-es/src/main/java/org/apache/lens/driver/es/ESDriver.java
index 5a7c6f044..e735c0c4b 100644
--- a/lens-driver-es/src/main/java/org/apache/lens/driver/es/ESDriver.java
+++ b/lens-driver-es/src/main/java/org/apache/lens/driver/es/ESDriver.java
@@ -28,6 +28,7 @@
 import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.lens.api.query.QueryHandle;
+import org.apache.lens.api.query.QueryPrepareHandle;
 import org.apache.lens.cube.metadata.CubeMetastoreClient;
 import org.apache.lens.cube.parse.HQLParser;
 import org.apache.lens.driver.es.client.ESClient;
@@ -118,6 +119,13 @@ public DriverQueryPlan explainAndPrepare(PreparedQueryContext pContext) throws L
     return explain(pContext);
   }
 
+  @Override
+  public void closePreparedQuery(QueryPrepareHandle handle) {
+    /**
+     * Elastic search does not have a concept of prepared query.
+     */
+  }
+
   @Override
   public LensResultSet execute(QueryContext context) throws LensException {
     handleContextMap.put(context.getQueryHandle(), context);
diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
index 562e139da..61fef15a3 100644
--- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
+++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
@@ -34,6 +34,7 @@
 import org.apache.lens.api.LensSessionHandle;
 import org.apache.lens.api.Priority;
 import org.apache.lens.api.query.QueryHandle;
+import org.apache.lens.api.query.QueryPrepareHandle;
 import org.apache.lens.cube.query.cost.FactPartitionBasedQueryCostCalculator;
 import org.apache.lens.server.api.LensConfConstants;
 import org.apache.lens.server.api.driver.*;
@@ -466,6 +467,17 @@ public void prepare(PreparedQueryContext pContext) throws LensException {
     // NO OP
   }
 
+  /*
+   * (non-Javadoc)
+   *
+   * @see org.apache.lens.server.api.driver.LensDriver#closePreparedQuery(org.apache.lens.api.query.QueryPrepareHandle)
+   */
+  @Override
+  public void closePreparedQuery(QueryPrepareHandle handle) throws LensException {
+    // NO OP
+
+  }
+
   /*
    * (non-Javadoc)
    *
diff --git a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
index 05f97178b..49bfb683e 100644
--- a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
+++ b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
@@ -757,6 +757,7 @@ public void testExplain() throws Exception {
     validateExecuteAsync(qctx, DriverQueryState.SUCCESSFUL, true, false);
 
     driver.closeQuery(qctx.getQueryHandle());
+    driver.closePreparedQuery(pctx.getPrepareHandle());
     assertHandleSize(handleSize);
   }
 
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java
index 0fbd8d428..0080b97b9 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java
@@ -22,6 +22,7 @@
 
 import org.apache.lens.api.Priority;
 import org.apache.lens.api.query.QueryHandle;
+import org.apache.lens.api.query.QueryPrepareHandle;
 import org.apache.lens.server.api.driver.hooks.DriverQueryHook;
 import org.apache.lens.server.api.error.LensException;
 import org.apache.lens.server.api.events.LensEventListener;
@@ -96,6 +97,14 @@ public interface LensDriver extends Externalizable {
    */
   DriverQueryPlan explainAndPrepare(PreparedQueryContext pContext) throws LensException;
 
+  /**
+   * Close the prepare query specified by the prepared handle, releases all the resources held by the prepared query.
+   *
+   * @param handle The query handle
+   * @throws LensException the lens exception
+   */
+  void closePreparedQuery(QueryPrepareHandle handle) throws LensException;
+
   /**
    * Blocking execute of the query
    * <p></p>
diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java b/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java
index f8c25bcce..92724c6a0 100644
--- a/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java
+++ b/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java
@@ -271,6 +271,17 @@ public DriverQueryPlan explainAndPrepare(PreparedQueryContext pContext) throws L
     return p;
   }
 
+  /*
+   * (non-Javadoc)
+   *
+   * @see org.apache.lens.server.api.driver.LensDriver#closePreparedQuery(org.apache.lens.api.query.QueryPrepareHandle)
+   */
+  @Override
+  public void closePreparedQuery(QueryPrepareHandle handle) throws LensException {
+    // TODO Auto-generated method stub
+
+  }
+
   /*
    * (non-Javadoc)
    *
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
index 623d3eb66..73e916cea 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
@@ -148,7 +148,7 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE
    * The Constant PREPARED_QUERY_PURGER_COUNTER.
    */
   public static final String PREPARED_QUERY_PURGER_COUNTER = "prepared-query-purger-errors";
-
+  
   public static final String PREPARED_QUERY_INSERT_COUNTER = "prepared-query-insert-errors";
 
   /**
@@ -2058,12 +2058,13 @@ public QueryPrepareHandle prepare(LensSessionHandle sessionHandle, String query,
       prepared.setQueryName(queryName);
       prepared.getSelectedDriver().prepare(prepared);
       try {
-        lensServerDao.insertPreparedQuery(prepared);
-      } catch (Exception e) {
-        incrCounter(PREPARED_QUERY_INSERT_COUNTER);
-      }
-
+          lensServerDao.insertPreparedQuery(prepared);
+        } catch (Exception e) {
+          incrCounter(PREPARED_QUERY_INSERT_COUNTER);
+        }
+      
       return prepared.getPrepareHandle();
+      
     } catch (LensException e) {
       if (prepared != null) {
         destroyPreparedQuery(prepared);

From 0caecd738d6c19975ea0007ac8669e48e0828058 Mon Sep 17 00:00:00 2001
From: Rajashekhar <rajashekhar.c@inmobi.com>
Date: Mon, 8 Jul 2019 17:09:48 +0530
Subject: [PATCH 13/25] removing not needed commits

---
 .../apache/lens/driver/jdbc/JDBCDriver.java   | 90 +++++++++++++++++--
 1 file changed, 82 insertions(+), 8 deletions(-)

diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
index fbacf544e..ca973b907 100644
--- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
+++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
@@ -31,11 +31,14 @@
 import java.io.ObjectOutput;
 import java.sql.*;
 import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.concurrent.*;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.lens.api.LensConf;
 import org.apache.lens.api.query.QueryHandle;
+import org.apache.lens.api.query.QueryPrepareHandle;
 import org.apache.lens.cube.parse.HQLParser;
 import org.apache.lens.cube.query.cost.StaticCostCalculator;
 import org.apache.lens.server.api.driver.*;
@@ -551,6 +554,9 @@ public String getPlan() {
 
   @Override
   public QueryCost estimate(AbstractQueryContext qctx) throws LensException {
+    String rewrittenQuery = rewriteQuery(qctx);
+    qctx.setSelectedDriverQuery(rewrittenQuery);
+
     MethodMetricsContext validateGauge = MethodMetricsFactory.createMethodGauge(qctx.getDriverConf(this), true,
       VALIDATE_GAUGE);
     validate(qctx);
@@ -621,9 +627,17 @@ public void validate(AbstractQueryContext pContext) throws LensException {
     boolean validateThroughPrepare = pContext.getDriverConf(this).getBoolean(JDBC_VALIDATE_THROUGH_PREPARE,
       DEFAULT_JDBC_VALIDATE_THROUGH_PREPARE);
     if (validateThroughPrepare) {
+      PreparedStatement stmt;
       // Estimate queries need to get connection from estimate pool to make sure
       // we are not blocked by data queries.
-      prepareInternal(pContext, true, true, "validate-");
+      stmt = prepareInternal(pContext, true, true, "validate-");
+      if (stmt != null) {
+        try {
+          stmt.close();
+        } catch (SQLException e) {
+          throw new LensException();
+        }
+      }
     }
   }
 
@@ -681,6 +695,8 @@ protected final ConnectionProvider getConnectionProvider() {
     return connectionProvider;
   }
 
+  private final Map<QueryPrepareHandle, PreparedStatement> preparedQueries = new HashMap<>();
+
   /**
    * Internally prepare the query
    *
@@ -688,12 +704,12 @@ protected final ConnectionProvider getConnectionProvider() {
    * @return prepared statement of the query
    * @throws LensException
    */
-  private void prepareInternal(AbstractQueryContext pContext) throws LensException {
+  private PreparedStatement prepareInternal(AbstractQueryContext pContext) throws LensException {
     if (pContext.getDriverQuery(this) == null) {
       throw new NullPointerException("Null driver query for " + pContext.getUserQuery());
     }
     checkConfigured();
-    prepareInternal(pContext, false, false, "prepare-");
+    return prepareInternal(pContext, false, false, "prepare-");
   }
 
 
@@ -706,7 +722,7 @@ private void prepareInternal(AbstractQueryContext pContext) throws LensException
    * @return prepared statement
    * @throws LensException
    */
-  private void prepareInternal(AbstractQueryContext pContext,
+  private PreparedStatement prepareInternal(AbstractQueryContext pContext,
     boolean calledForEstimate,
     boolean checkConfigured,
     String metricCallStack) throws LensException {
@@ -722,11 +738,34 @@ private void prepareInternal(AbstractQueryContext pContext,
     // Only create a prepared statement and then close it
     MethodMetricsContext sqlRewriteGauge = MethodMetricsFactory.createMethodGauge(pContext.getDriverConf(this), true,
       metricCallStack + COLUMNAR_SQL_REWRITE_GAUGE);
-    String rewrittenQuery = rewriteQuery(pContext);
-    pContext.setSelectedDriverQuery(rewrittenQuery);
+    String rewrittenQuery = pContext.getSelectedDriverQuery();
     sqlRewriteGauge.markSuccess();
     MethodMetricsContext jdbcPrepareGauge = MethodMetricsFactory.createMethodGauge(pContext.getDriverConf(this), true,
       metricCallStack + JDBC_PREPARE_GAUGE);
+
+    PreparedStatement stmt = null;
+    Connection conn = null;
+    try {
+      conn = calledForEstimate ? getEstimateConnection() : getConnection();
+      stmt = conn.prepareStatement(rewrittenQuery);
+      if (!pContext.getDriverConf(this).getBoolean(JDBC_VALIDATE_SKIP_WARNINGS,
+        DEFAULT_JDBC_VALIDATE_SKIP_WARNINGS) && stmt.getWarnings() != null) {
+        throw new LensException(stmt.getWarnings());
+      }
+    } catch (SQLException sql) {
+      handleJDBCSQLException(sql);
+    } finally {
+      if (conn != null) {
+        try {
+          conn.close();
+        } catch (SQLException e) {
+          log.error("Error closing connection: {}", rewrittenQuery, e);
+        }
+      }
+      jdbcPrepareGauge.markSuccess();
+    }
+    log.info("Prepared: {}", rewrittenQuery);
+    return stmt;
   }
 
   /**
@@ -761,7 +800,14 @@ private String getSqlSynataxExceptions(Throwable e) {
    */
   @Override
   public void prepare(PreparedQueryContext pContext) throws LensException {
-    prepareInternal(pContext);
+    if (preparedQueries.containsKey(pContext.getPrepareHandle())) {
+      // already prepared
+      return;
+    }
+    PreparedStatement stmt = prepareInternal(pContext);
+    if (stmt != null) {
+      preparedQueries.put(pContext.getPrepareHandle(), stmt);
+    }
   }
 
   /**
@@ -778,6 +824,24 @@ public DriverQueryPlan explainAndPrepare(PreparedQueryContext pContext) throws L
     return new JDBCQueryPlan(calculateQueryCost(pContext));
   }
 
+  /**
+   * Close the prepare query specified by the prepared handle, releases all the resources held by the prepared query.
+   *
+   * @param handle The query handle
+   * @throws LensException the lens exception
+   */
+  @Override
+  public void closePreparedQuery(QueryPrepareHandle handle) throws LensException {
+    checkConfigured();
+    try {
+      if (preparedQueries.get(handle) != null) {
+        preparedQueries.get(handle).close();
+      }
+    } catch (SQLException e) {
+      throw new LensException(e);
+    }
+  }
+
   /**
    * Blocking execute of the query.
    *
@@ -826,7 +890,6 @@ public void executeAsync(QueryContext context) throws LensException {
     String rewrittenQuery = rewriteQuery(context);
     JdbcQueryContext jdbcCtx = new JdbcQueryContext(context);
     jdbcCtx.setRewrittenQuery(rewrittenQuery);
-    context.setSelectedDriverQuery(rewrittenQuery);
     try {
       Future<QueryResult> future = asyncQueryPool.submit(new QueryCallable(jdbcCtx, logSegregationContext));
       jdbcCtx.setResultFuture(future);
@@ -961,6 +1024,17 @@ public void close() throws LensException {
           log.warn("{} Error closing query : {}", getFullyQualifiedName(), query.getHandleId(), e);
         }
       }
+      for (QueryPrepareHandle query : preparedQueries.keySet()) {
+        try {
+          try {
+            preparedQueries.get(query).close();
+          } catch (SQLException e) {
+            throw new LensException();
+          }
+        } catch (LensException e) {
+          log.warn("{} Error closing prapared query : {}", getFullyQualifiedName(), query, e);
+        }
+      }
     } finally {
       queryContextMap.clear();
     }

From 2ce78dc66e7ab0748444df65a4b07ffa9f6547dc Mon Sep 17 00:00:00 2001
From: Rajashekhar <rajashekhar.c@inmobi.com>
Date: Tue, 9 Jul 2019 11:54:09 +0530
Subject: [PATCH 14/25] undid the unit test cases

---
 .../lens/driver/jdbc/TestJdbcDriver.java      | 35 +++++++++++++++++++
 1 file changed, 35 insertions(+)

diff --git a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
index 6be4ac208..c08b04a27 100644
--- a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
+++ b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
@@ -293,6 +293,22 @@ public void testEstimate() throws Exception {
 
   }
 
+  /**
+   * Test estimate failing
+   *
+   * @throws Exception the exception
+   */
+  @Test
+  public void testEstimateFailing() throws Exception {
+    String query2 = "SELECT * FROM estimate_test2"; // Select query against non existing table
+    try {
+      driver.estimate(createExplainContext(query2, baseConf));
+      Assert.fail("Running estimate on a non existing table.");
+    } catch (LensException ex) {
+      Assert.assertEquals(LensUtil.getCauseMessage(ex), "user lacks privilege or object not found: ESTIMATE_TEST2");
+    }
+  }
+
   /**
    * Test estimate failing
    *
@@ -310,6 +326,7 @@ public void testEstimateGauges() throws Exception {
     String driverQualifiledName = driver.getFullyQualifiedName();
     Assert.assertTrue(reg.getGauges().keySet().containsAll(Arrays.asList(
       "lens.MethodMetricGauge.TestJdbcDriver-"+driverQualifiledName+"-validate-columnar-sql-rewrite",
+      "lens.MethodMetricGauge.TestJdbcDriver-"+driverQualifiledName+"-validate-jdbc-prepare-statement",
       "lens.MethodMetricGauge.TestJdbcDriver-"+driverQualifiledName+"-validate-thru-prepare",
       "lens.MethodMetricGauge.TestJdbcDriver-"+driverQualifiledName+"-jdbc-check-allowed-query")));
   }
@@ -342,6 +359,7 @@ public void testMetricsEnabled() throws Exception {
     cost = driver.estimate(pContext2);
     Assert.assertEquals(cost, JDBC_COST);
     driver.prepare(pContext2);
+    driver.explainAndPrepare(pContext2);
   }
 
   /**
@@ -635,6 +653,23 @@ public void testPrepare() throws Exception {
     driver.prepare(pContext);
   }
 
+  /**
+   * Test prepare failing
+   *
+   * @throws Exception the exception
+   */
+  @Test
+  public void testPrepareFailing() throws Exception {
+    String query = "SELECT * FROM prepare_test2"; // Select query against non existing table
+    try {
+      PreparedQueryContext pContext = new PreparedQueryContext(query, "SA", baseConf, drivers);
+      driver.prepare(pContext);
+      Assert.fail("Running prepare on a non existing table.");
+    } catch (LensException ex) {
+      Assert.assertEquals(LensUtil.getCauseMessage(ex), "user lacks privilege or object not found: PREPARE_TEST2");
+    }
+  }
+
   /**
    * Test prepare skip warnings
    *

From 2400d8efee13a00e12a9f4f86e0c7c3766fe1ab6 Mon Sep 17 00:00:00 2001
From: Rajashekhar <rajashekhar.c@inmobi.com>
Date: Tue, 9 Jul 2019 12:00:03 +0530
Subject: [PATCH 15/25] made one more change

---
 .../apache/lens/server/query/QueryExecutionServiceImpl.java   | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
index 73e916cea..9ad8f92e8 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
@@ -2056,7 +2056,6 @@ public QueryPrepareHandle prepare(LensSessionHandle sessionHandle, String query,
       acquire(sessionHandle);
       prepared = prepareQuery(sessionHandle, query, lensConf, SubmitOp.PREPARE);
       prepared.setQueryName(queryName);
-      prepared.getSelectedDriver().prepare(prepared);
       try {
           lensServerDao.insertPreparedQuery(prepared);
         } catch (Exception e) {
@@ -3090,6 +3089,9 @@ public boolean destroyPrepared(LensSessionHandle sessionHandle, QueryPrepareHand
    * @throws LensException the lens exception
    */
   private void destroyPreparedQuery(PreparedQueryContext ctx) throws LensException {
+    if (ctx.getSelectedDriver() != null) {
+      ctx.getSelectedDriver().closePreparedQuery(ctx.getPrepareHandle());
+    }
     preparedQueries.remove(ctx.getPrepareHandle());
     preparedQueryQueue.remove(ctx);
     decrCounter(PREPARED_QUERIES_COUNTER);

From 5fa28003f40932aa37ab118640c58b48854eb6af Mon Sep 17 00:00:00 2001
From: Rajashekhar <rajashekhar.c@inmobi.com>
Date: Mon, 15 Jul 2019 13:32:57 +0530
Subject: [PATCH 16/25] fixed code review comments

---
 .../apache/lens/driver/jdbc/JDBCDriver.java   | 32 +++++++++++--------
 .../driver/jdbc/JDBCDriverConfConstants.java  |  7 ++--
 2 files changed, 23 insertions(+), 16 deletions(-)

diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
index 8dd299a11..62f615b93 100644
--- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
+++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
@@ -593,19 +593,25 @@ public DriverQueryPlan explain(AbstractQueryContext explainCtx) throws LensExcep
         + explainKeyword + " ");
     }
     log.info("{} Explain Query : {}", getFullyQualifiedName(), explainQuery);
-    QueryContext explainQueryCtx = QueryContext.createContextWithSingleDriver(explainQuery, null,
-      new LensConf(), explainConf, this, explainCtx.getLensSessionIdentifier(), false);
-    QueryResult result = null;
-    try {
-      result = executeInternal(explainQueryCtx, explainQuery);
-      if (result.error != null) {
-        throw new LensException("Query explain failed!", result.error);
-      }
-    } finally {
-      if (result != null) {
-        result.close();
+
+    boolean validateThroughPrepare = explainCtx.getDriverConf(this).getBoolean(JDBC_VALIDATE_THROUGH_PREPARE_OR_EXPLAIN,
+        DEFAULT_JDBC_VALIDATE_THROUGH_PREPARE_OR_EXPLAIN);
+    if (validateThroughPrepare) {
+      QueryContext explainQueryCtx = QueryContext.createContextWithSingleDriver(explainQuery, null, new LensConf(),
+          explainConf, this, explainCtx.getLensSessionIdentifier(), false);
+      QueryResult result = null;
+      try {
+        result = executeInternal(explainQueryCtx, explainQuery);
+        if (result.error != null) {
+          throw new LensException("Query explain failed!", result.error);
+        }
+      } finally {
+        if (result != null) {
+          result.close();
+        }
       }
     }
+
     JDBCQueryPlan jqp = new JDBCQueryPlan(calculateQueryCost(explainCtx));
     explainCtx.getDriverContext().setDriverQueryPlan(this, jqp);
     return jqp;
@@ -621,8 +627,8 @@ public void validate(AbstractQueryContext pContext) throws LensException {
     if (pContext.getDriverQuery(this) == null) {
       throw new NullPointerException("Null driver query for " + pContext.getUserQuery());
     }
-    boolean validateThroughPrepare = pContext.getDriverConf(this).getBoolean(JDBC_VALIDATE_THROUGH_PREPARE,
-      DEFAULT_JDBC_VALIDATE_THROUGH_PREPARE);
+    boolean validateThroughPrepare = pContext.getDriverConf(this).getBoolean(JDBC_VALIDATE_THROUGH_PREPARE_OR_EXPLAIN,
+      DEFAULT_JDBC_VALIDATE_THROUGH_PREPARE_OR_EXPLAIN);
     if (validateThroughPrepare) {
       PreparedStatement stmt;
       // Estimate queries need to get connection from estimate pool to make sure
diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriverConfConstants.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriverConfConstants.java
index f2bfb69c3..95eb55bcb 100644
--- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriverConfConstants.java
+++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriverConfConstants.java
@@ -94,11 +94,12 @@ public enum ConnectionPoolProperties {
   /** The Constant DEFAULT_JDBC_EXPLAIN_KEYWORD_BEFORE_SELECT. */
   public static final boolean DEFAULT_JDBC_EXPLAIN_KEYWORD_BEFORE_SELECT = true;
 
-  /** The Constant JDBC_VALIDATE_THROUGH_PREPARE. */
-  public static final String JDBC_VALIDATE_THROUGH_PREPARE = JDBC_DRIVER_PFX + "validate.through.prepare";
+  /** The Constant JDBC_VALIDATE_THROUGH_PREPARE_OR_EXPLAIN. */
+  public static final String JDBC_VALIDATE_THROUGH_PREPARE_OR_EXPLAIN =
+    JDBC_DRIVER_PFX + "validate.through.prepare.or.explain";
 
   /** The Constant DEFAULT_JDBC_VALIDATE_THROUGH_PREPARE. */
-  public static final boolean DEFAULT_JDBC_VALIDATE_THROUGH_PREPARE = true;
+  public static final boolean DEFAULT_JDBC_VALIDATE_THROUGH_PREPARE_OR_EXPLAIN = true;
 
   /** The Constant JDBC_VALIDATE_SKIP_WARNINGS */
   public static final String JDBC_VALIDATE_SKIP_WARNINGS = JDBC_DRIVER_PFX + "validate.skip.warnings";

From 926b60440f5f7e7ba3c0bc181a99437acbd35b69 Mon Sep 17 00:00:00 2001
From: Rajashekhar <rajashekhar.c@inmobi.com>
Date: Mon, 15 Jul 2019 13:34:59 +0530
Subject: [PATCH 17/25] fixed code review comments

---
 .../apache/lens/driver/jdbc/JDBCDriver.java   | 37 ++++++++++---------
 .../driver/jdbc/JDBCDriverConfConstants.java  |  7 ++--
 2 files changed, 24 insertions(+), 20 deletions(-)

diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
index ca973b907..62f615b93 100644
--- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
+++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
@@ -554,9 +554,6 @@ public String getPlan() {
 
   @Override
   public QueryCost estimate(AbstractQueryContext qctx) throws LensException {
-    String rewrittenQuery = rewriteQuery(qctx);
-    qctx.setSelectedDriverQuery(rewrittenQuery);
-
     MethodMetricsContext validateGauge = MethodMetricsFactory.createMethodGauge(qctx.getDriverConf(this), true,
       VALIDATE_GAUGE);
     validate(qctx);
@@ -596,19 +593,25 @@ public DriverQueryPlan explain(AbstractQueryContext explainCtx) throws LensExcep
         + explainKeyword + " ");
     }
     log.info("{} Explain Query : {}", getFullyQualifiedName(), explainQuery);
-    QueryContext explainQueryCtx = QueryContext.createContextWithSingleDriver(explainQuery, null,
-      new LensConf(), explainConf, this, explainCtx.getLensSessionIdentifier(), false);
-    QueryResult result = null;
-    try {
-      result = executeInternal(explainQueryCtx, explainQuery);
-      if (result.error != null) {
-        throw new LensException("Query explain failed!", result.error);
-      }
-    } finally {
-      if (result != null) {
-        result.close();
+
+    boolean validateThroughPrepare = explainCtx.getDriverConf(this).getBoolean(JDBC_VALIDATE_THROUGH_PREPARE_OR_EXPLAIN,
+        DEFAULT_JDBC_VALIDATE_THROUGH_PREPARE_OR_EXPLAIN);
+    if (validateThroughPrepare) {
+      QueryContext explainQueryCtx = QueryContext.createContextWithSingleDriver(explainQuery, null, new LensConf(),
+          explainConf, this, explainCtx.getLensSessionIdentifier(), false);
+      QueryResult result = null;
+      try {
+        result = executeInternal(explainQueryCtx, explainQuery);
+        if (result.error != null) {
+          throw new LensException("Query explain failed!", result.error);
+        }
+      } finally {
+        if (result != null) {
+          result.close();
+        }
       }
     }
+
     JDBCQueryPlan jqp = new JDBCQueryPlan(calculateQueryCost(explainCtx));
     explainCtx.getDriverContext().setDriverQueryPlan(this, jqp);
     return jqp;
@@ -624,8 +627,8 @@ public void validate(AbstractQueryContext pContext) throws LensException {
     if (pContext.getDriverQuery(this) == null) {
       throw new NullPointerException("Null driver query for " + pContext.getUserQuery());
     }
-    boolean validateThroughPrepare = pContext.getDriverConf(this).getBoolean(JDBC_VALIDATE_THROUGH_PREPARE,
-      DEFAULT_JDBC_VALIDATE_THROUGH_PREPARE);
+    boolean validateThroughPrepare = pContext.getDriverConf(this).getBoolean(JDBC_VALIDATE_THROUGH_PREPARE_OR_EXPLAIN,
+      DEFAULT_JDBC_VALIDATE_THROUGH_PREPARE_OR_EXPLAIN);
     if (validateThroughPrepare) {
       PreparedStatement stmt;
       // Estimate queries need to get connection from estimate pool to make sure
@@ -738,7 +741,7 @@ private PreparedStatement prepareInternal(AbstractQueryContext pContext,
     // Only create a prepared statement and then close it
     MethodMetricsContext sqlRewriteGauge = MethodMetricsFactory.createMethodGauge(pContext.getDriverConf(this), true,
       metricCallStack + COLUMNAR_SQL_REWRITE_GAUGE);
-    String rewrittenQuery = pContext.getSelectedDriverQuery();
+    String rewrittenQuery = rewriteQuery(pContext);
     sqlRewriteGauge.markSuccess();
     MethodMetricsContext jdbcPrepareGauge = MethodMetricsFactory.createMethodGauge(pContext.getDriverConf(this), true,
       metricCallStack + JDBC_PREPARE_GAUGE);
diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriverConfConstants.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriverConfConstants.java
index f2bfb69c3..95eb55bcb 100644
--- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriverConfConstants.java
+++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriverConfConstants.java
@@ -94,11 +94,12 @@ public enum ConnectionPoolProperties {
   /** The Constant DEFAULT_JDBC_EXPLAIN_KEYWORD_BEFORE_SELECT. */
   public static final boolean DEFAULT_JDBC_EXPLAIN_KEYWORD_BEFORE_SELECT = true;
 
-  /** The Constant JDBC_VALIDATE_THROUGH_PREPARE. */
-  public static final String JDBC_VALIDATE_THROUGH_PREPARE = JDBC_DRIVER_PFX + "validate.through.prepare";
+  /** The Constant JDBC_VALIDATE_THROUGH_PREPARE_OR_EXPLAIN. */
+  public static final String JDBC_VALIDATE_THROUGH_PREPARE_OR_EXPLAIN =
+    JDBC_DRIVER_PFX + "validate.through.prepare.or.explain";
 
   /** The Constant DEFAULT_JDBC_VALIDATE_THROUGH_PREPARE. */
-  public static final boolean DEFAULT_JDBC_VALIDATE_THROUGH_PREPARE = true;
+  public static final boolean DEFAULT_JDBC_VALIDATE_THROUGH_PREPARE_OR_EXPLAIN = true;
 
   /** The Constant JDBC_VALIDATE_SKIP_WARNINGS */
   public static final String JDBC_VALIDATE_SKIP_WARNINGS = JDBC_DRIVER_PFX + "validate.skip.warnings";

From 353dd563c77769e1bad9578396ba66f840d3797f Mon Sep 17 00:00:00 2001
From: Rajashekhar <rajashekhar.c@inmobi.com>
Date: Mon, 15 Jul 2019 13:36:45 +0530
Subject: [PATCH 18/25] fixed code review comments

---
 .../server/query/QueryExecutionServiceImpl.java     | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
index 9ad8f92e8..c3d78cd19 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
@@ -148,7 +148,7 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE
    * The Constant PREPARED_QUERY_PURGER_COUNTER.
    */
   public static final String PREPARED_QUERY_PURGER_COUNTER = "prepared-query-purger-errors";
-  
+
   public static final String PREPARED_QUERY_INSERT_COUNTER = "prepared-query-insert-errors";
 
   /**
@@ -2057,13 +2057,12 @@ public QueryPrepareHandle prepare(LensSessionHandle sessionHandle, String query,
       prepared = prepareQuery(sessionHandle, query, lensConf, SubmitOp.PREPARE);
       prepared.setQueryName(queryName);
       try {
-          lensServerDao.insertPreparedQuery(prepared);
-        } catch (Exception e) {
-          incrCounter(PREPARED_QUERY_INSERT_COUNTER);
-        }
-      
+        lensServerDao.insertPreparedQuery(prepared);
+      } catch (Exception e) {
+        incrCounter(PREPARED_QUERY_INSERT_COUNTER);
+      }
+
       return prepared.getPrepareHandle();
-      
     } catch (LensException e) {
       if (prepared != null) {
         destroyPreparedQuery(prepared);

From fc4358ec15aabc74aad9540e47aec29a23d38130 Mon Sep 17 00:00:00 2001
From: Rajashekhar <rajashekhar.c@inmobi.com>
Date: Tue, 16 Jul 2019 15:19:03 +0530
Subject: [PATCH 19/25] made some changes

---
 .../lens/server/query/LensServerDAO.java      | 21 ++++-
 .../query/QueryExecutionServiceImpl.java      |  1 +
 .../apache/lens/server/query/TestLensDAO.java | 81 ++++++++++++++++++-
 3 files changed, 97 insertions(+), 6 deletions(-)

diff --git a/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java b/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
index 24748fc14..453f4a3dc 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
@@ -107,6 +107,21 @@ public void createFinishedQueriesTable() throws Exception {
       log.warn("Unable to create finished queries table", e);
     }
   }
+
+  public void createPreparedQueriesTable() throws Exception {
+    String sql = "CREATE TABLE if not exists prepared_queries (handle varchar(255) NOT NULL unique,  userquery " +
+        "varchar(20000),  submitter varchar(255) NOT NULL,  timetaken bigint,  queryname varchar(255) DEFAULT NULL,  " +
+        "drivername varchar(10000) DEFAULT NULL,  driverquery varchar(1000000),  starttime bigint)";
+    try {
+      QueryRunner runner = new QueryRunner(ds);
+      runner.update(sql);
+      log.info("Created prepared_queries queries table");
+    } catch (SQLException e) {
+      log.warn("Unable to create prepared_queries queries table", e);
+    }
+  }
+    
+    
   public void createFailedAttemptsTable() throws Exception {
     String sql = "CREATE TABLE if not exists failed_attempts (handle varchar(255) not null,"
       + "attempt_number int, drivername varchar(10000), progress float, progressmessage varchar(10000), "
@@ -832,7 +847,7 @@ public boolean deleteActiveSession(LensSessionHandle sessionId) throws LensExcep
    */
   public void insertPreparedQuery(PreparedQueryContext preparedQueryContext) throws SQLException {
     String sql = "insert into prepared_queries (handle, userquery, submitter, timetaken, queryname, drivername, "
-        + "driverquery, starttime)" + " values (?,?,?,?,?,?,?)";
+        + "driverquery, starttime)" + " values (?,?,?,?,?,?,?,?)";
     Connection conn = null;
     try {
       conn = getConnection();
@@ -844,8 +859,8 @@ public void insertPreparedQuery(PreparedQueryContext preparedQueryContext) throw
 
       runner.update(conn, sql, preparedQueryContext.getPrepareHandle().getQueryHandleString(),
           preparedQueryContext.getUserQuery(), preparedQueryContext.getSubmittedUser(), timeTaken,
-          preparedQueryContext.getQueryName(), preparedQueryContext.getDriverContext().getSelectedDriver(),
-          preparedQueryContext.getSelectedDriverQuery(), preparedQueryContext.getPrepareStartTime());
+          preparedQueryContext.getQueryName(), preparedQueryContext.getDriverContext().getSelectedDriver().toString(),
+          preparedQueryContext.getSelectedDriverQuery(), preparedQueryContext.getPrepareStartTime().getTime());
       conn.commit();
     } finally {
       DbUtils.closeQuietly(conn);
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
index c3d78cd19..940a81c78 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
@@ -1417,6 +1417,7 @@ private void initalizeFinishedQueryStore(Configuration conf) {
       this.lensServerDao.createFailedAttemptsTable();
       this.lensServerDao.createActiveSessionsTable();
       this.lensServerDao.createActiveQueriesTable();
+      this.lensServerDao.createPreparedQueriesTable();
     } catch (Exception e) {
       log.warn("Unable to create finished query tables, query purger will not purge queries", e);
     }
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestLensDAO.java b/lens-server/src/test/java/org/apache/lens/server/query/TestLensDAO.java
index c74174561..f8da1f38b 100644
--- a/lens-server/src/test/java/org/apache/lens/server/query/TestLensDAO.java
+++ b/lens-server/src/test/java/org/apache/lens/server/query/TestLensDAO.java
@@ -18,10 +18,13 @@
  */
 package org.apache.lens.server.query;
 
+import java.nio.charset.Charset;
 import java.sql.Connection;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.sql.Statement;
+import java.util.Collection;
+import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
 
@@ -34,21 +37,34 @@
 import org.apache.lens.api.query.LensQuery;
 import org.apache.lens.api.query.QueryHandle;
 import org.apache.lens.api.query.QueryStatus;
+import org.apache.lens.driver.hive.EmbeddedThriftConnection;
+import org.apache.lens.driver.hive.HiveDriver;
+import org.apache.lens.driver.hive.ThriftConnection;
+import org.apache.lens.driver.jdbc.JDBCDriver;
 import org.apache.lens.driver.jdbc.JDBCResultSet;
 import org.apache.lens.server.LensJerseyTest;
 import org.apache.lens.server.LensServices;
+import org.apache.lens.server.api.LensConfConstants;
 import org.apache.lens.server.api.driver.LensDriver;
 import org.apache.lens.server.api.driver.MockDriver;
+import org.apache.lens.server.api.driver.hooks.DriverQueryHook;
 import org.apache.lens.server.api.error.LensException;
 import org.apache.lens.server.api.query.*;
-
+import org.apache.lens.server.api.user.MockDriverQueryHook;
+import org.apache.lens.server.util.UtilityMethods;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.dbutils.BasicRowProcessor;
+import org.apache.commons.dbutils.BeanProcessor;
+import org.apache.commons.dbutils.QueryRunner;
+import org.apache.commons.dbutils.ResultSetHandler;
+import org.apache.commons.dbutils.handlers.BeanHandler;
 import org.apache.hadoop.conf.Configuration;
-
 import org.codehaus.jackson.map.ObjectMapper;
 import org.testng.Assert;
 import org.testng.annotations.Test;
 
 import com.google.common.collect.Lists;
+
 import lombok.extern.slf4j.Slf4j;
 
 /**
@@ -213,4 +229,63 @@ public void testLensServerDAO() throws Exception {
 
     service.closeSession(session);
   }
-}
+
+  public void testPreparedQueryDAO() throws Exception {
+    QueryExecutionServiceImpl service = LensServices.get().getService(QueryExecutionService.NAME);
+    Connection conn = null;
+    Statement stmt = null;
+    final ObjectMapper MAPPER = new ObjectMapper();
+
+    try {
+      conn = service.lensServerDao.getConnection();
+      stmt = conn.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY);
+      ResultSet rs = stmt.executeQuery("SELECT handle FROM finished_queries");
+
+      JDBCResultSet jdbcResultSet = new JDBCResultSet(null, rs, false);
+      JDBCResultSet.JDBCResultSetMetadata jdbcRsMeta =
+          (JDBCResultSet.JDBCResultSetMetadata) jdbcResultSet.getMetadata();
+
+      String jsonMetadata = MAPPER.writeValueAsString(jdbcRsMeta);
+
+      log.info("@@@JSON {}" + jsonMetadata);
+
+    } catch (SQLException ex) {
+      log.error("Error creating result set ", ex);
+    } finally {
+      if (stmt != null) {
+        stmt.close();
+      }
+      if (conn != null) {
+        conn.close();
+      }
+    }
+
+    service.lensServerDao.createPreparedQueriesTable();
+    Configuration driverConf = new Configuration();
+    try {
+      PreparedQueryContext preparedQueryContext = new PreparedQueryContext("query", "user1", driverConf,
+          createDriver(driverConf));
+      preparedQueryContext.setPrepareEndTime(new Date());
+      service.lensServerDao.insertPreparedQuery(preparedQueryContext);
+    } catch (Exception e) {
+      Assert.fail("it shouldn't be coming in this catch block");
+    }
+
+
+  }
+
+  protected Collection<LensDriver> createDriver(Configuration driverConf) throws LensException {
+
+    driverConf.addResource("drivers/jdbc/jdbc1/jdbcdriver-site.xml");
+    driverConf.setClass(HiveDriver.HIVE_CONNECTION_CLASS, EmbeddedThriftConnection.class, ThriftConnection.class);
+    driverConf.setClass(LensConfConstants.DRIVER_HOOK_CLASSES_SFX, MockDriverQueryHook.class, DriverQueryHook.class);
+    driverConf.set("hive.lock.manager", "org.apache.hadoop.hive.ql.lockmgr.EmbeddedLockManager");
+    driverConf.setBoolean(HiveDriver.HS2_CALCULATE_PRIORITY, true);
+    JDBCDriver driver = new JDBCDriver();
+    driver.configure(driverConf, "jdbc", "jdbc1");
+    Collection<LensDriver> drivers = Lists.<LensDriver>newArrayList(driver);
+
+    System.out.println("TestJDBCDriver created");
+    return drivers;
+  }
+}
\ No newline at end of file

From fe57a2e400e132e83c967afdfc5381260bb66271 Mon Sep 17 00:00:00 2001
From: Rajashekhar <rajashekhar.c@inmobi.com>
Date: Tue, 16 Jul 2019 15:54:30 +0530
Subject: [PATCH 20/25] added the files

---
 .../org/apache/lens/server/query/LensServerDAO.java |  9 ++++-----
 .../org/apache/lens/server/query/TestLensDAO.java   | 13 ++++---------
 2 files changed, 8 insertions(+), 14 deletions(-)

diff --git a/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java b/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
index 453f4a3dc..3c492dbb9 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
@@ -109,9 +109,9 @@ public void createFinishedQueriesTable() throws Exception {
   }
 
   public void createPreparedQueriesTable() throws Exception {
-    String sql = "CREATE TABLE if not exists prepared_queries (handle varchar(255) NOT NULL unique,  userquery " +
-        "varchar(20000),  submitter varchar(255) NOT NULL,  timetaken bigint,  queryname varchar(255) DEFAULT NULL,  " +
-        "drivername varchar(10000) DEFAULT NULL,  driverquery varchar(1000000),  starttime bigint)";
+    String sql = "CREATE TABLE if not exists prepared_queries (handle varchar(255) NOT NULL unique,  userquery "
+      + "varchar(20000),  submitter varchar(255) NOT NULL,  timetaken bigint,  queryname varchar(255) DEFAULT NULL, "
+      + "drivername varchar(10000) DEFAULT NULL,  driverquery varchar(1000000),  starttime bigint)";
     try {
       QueryRunner runner = new QueryRunner(ds);
       runner.update(sql);
@@ -120,8 +120,7 @@ public void createPreparedQueriesTable() throws Exception {
       log.warn("Unable to create prepared_queries queries table", e);
     }
   }
-    
-    
+
   public void createFailedAttemptsTable() throws Exception {
     String sql = "CREATE TABLE if not exists failed_attempts (handle varchar(255) not null,"
       + "attempt_number int, drivername varchar(10000), progress float, progressmessage varchar(10000), "
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestLensDAO.java b/lens-server/src/test/java/org/apache/lens/server/query/TestLensDAO.java
index f8da1f38b..23b9813a5 100644
--- a/lens-server/src/test/java/org/apache/lens/server/query/TestLensDAO.java
+++ b/lens-server/src/test/java/org/apache/lens/server/query/TestLensDAO.java
@@ -18,7 +18,6 @@
  */
 package org.apache.lens.server.query;
 
-import java.nio.charset.Charset;
 import java.sql.Connection;
 import java.sql.ResultSet;
 import java.sql.SQLException;
@@ -51,14 +50,9 @@
 import org.apache.lens.server.api.error.LensException;
 import org.apache.lens.server.api.query.*;
 import org.apache.lens.server.api.user.MockDriverQueryHook;
-import org.apache.lens.server.util.UtilityMethods;
-import org.apache.commons.codec.binary.Base64;
-import org.apache.commons.dbutils.BasicRowProcessor;
-import org.apache.commons.dbutils.BeanProcessor;
-import org.apache.commons.dbutils.QueryRunner;
-import org.apache.commons.dbutils.ResultSetHandler;
-import org.apache.commons.dbutils.handlers.BeanHandler;
+
 import org.apache.hadoop.conf.Configuration;
+
 import org.codehaus.jackson.map.ObjectMapper;
 import org.testng.Assert;
 import org.testng.annotations.Test;
@@ -288,4 +282,5 @@ protected Collection<LensDriver> createDriver(Configuration driverConf) throws L
     System.out.println("TestJDBCDriver created");
     return drivers;
   }
-}
\ No newline at end of file
+}
+

From 0cdd720f1b0fbd971657d3091e140adf140d0747 Mon Sep 17 00:00:00 2001
From: Rajashekhar <rajashekhar.c@inmobi.com>
Date: Fri, 19 Jul 2019 15:15:24 +0530
Subject: [PATCH 21/25] added the code review fix

---
 .../apache/lens/server/query/LensServerDAO.java  |  9 ++++++---
 .../server/query/QueryExecutionServiceImpl.java  | 16 ++++++----------
 2 files changed, 12 insertions(+), 13 deletions(-)

diff --git a/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java b/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
index 3c492dbb9..28b678b3b 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
@@ -120,7 +120,7 @@ public void createPreparedQueriesTable() throws Exception {
       log.warn("Unable to create prepared_queries queries table", e);
     }
   }
-
+  
   public void createFailedAttemptsTable() throws Exception {
     String sql = "CREATE TABLE if not exists failed_attempts (handle varchar(255) not null,"
       + "attempt_number int, drivername varchar(10000), progress float, progressmessage varchar(10000), "
@@ -844,7 +844,7 @@ public boolean deleteActiveSession(LensSessionHandle sessionId) throws LensExcep
    * @param preparedQueryContext to be inserted
    * @throws SQLException the exception
    */
-  public void insertPreparedQuery(PreparedQueryContext preparedQueryContext) throws SQLException {
+  public void insertPreparedQuery(PreparedQueryContext preparedQueryContext) throws LensException {
     String sql = "insert into prepared_queries (handle, userquery, submitter, timetaken, queryname, drivername, "
         + "driverquery, starttime)" + " values (?,?,?,?,?,?,?,?)";
     Connection conn = null;
@@ -861,7 +861,10 @@ public void insertPreparedQuery(PreparedQueryContext preparedQueryContext) throw
           preparedQueryContext.getQueryName(), preparedQueryContext.getDriverContext().getSelectedDriver().toString(),
           preparedQueryContext.getSelectedDriverQuery(), preparedQueryContext.getPrepareStartTime().getTime());
       conn.commit();
-    } finally {
+    } catch (SQLException e) {
+      log.error("Failed to insert prepared query into database with error, " + e);
+      throw new LensException(e);
+  } finally {
       DbUtils.closeQuietly(conn);
     }
   }
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
index 940a81c78..18bd2d06f 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
@@ -147,9 +147,9 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE
   /**
    * The Constant PREPARED_QUERY_PURGER_COUNTER.
    */
-  public static final String PREPARED_QUERY_PURGER_COUNTER = "prepared-query-purger-errors";
+  public static final String PREPARED_QUERY_PURGER_ERROR_COUNTER = "prepared-query-purger-errors";
 
-  public static final String PREPARED_QUERY_INSERT_COUNTER = "prepared-query-insert-errors";
+  public static final String PREPARED_QUERY_INSERT_ERROR_COUNTER = "prepared-query-insert-errors";
 
   /**
    * The millis in week.
@@ -1304,13 +1304,13 @@ public void run() {
           destroyPreparedQuery(prepared);
           log.info("Purged prepared query: {}", prepared.getPrepareHandle());
         } catch (LensException e) {
-          incrCounter(PREPARED_QUERY_PURGER_COUNTER);
+          incrCounter(PREPARED_QUERY_PURGER_ERROR_COUNTER);
           log.error("Error closing prepared query ", e);
         } catch (InterruptedException e) {
           log.info("PreparedQueryPurger has been interrupted, exiting");
           return;
         } catch (Exception e) {
-          incrCounter(PREPARED_QUERY_PURGER_COUNTER);
+          incrCounter(PREPARED_QUERY_PURGER_ERROR_COUNTER);
           log.error("Error in prepared query purger", e);
         }
       }
@@ -2057,14 +2057,10 @@ public QueryPrepareHandle prepare(LensSessionHandle sessionHandle, String query,
       acquire(sessionHandle);
       prepared = prepareQuery(sessionHandle, query, lensConf, SubmitOp.PREPARE);
       prepared.setQueryName(queryName);
-      try {
-        lensServerDao.insertPreparedQuery(prepared);
-      } catch (Exception e) {
-        incrCounter(PREPARED_QUERY_INSERT_COUNTER);
-      }
-
+      lensServerDao.insertPreparedQuery(prepared);
       return prepared.getPrepareHandle();
     } catch (LensException e) {
+      incrCounter(PREPARED_QUERY_INSERT_ERROR_COUNTER);
       if (prepared != null) {
         destroyPreparedQuery(prepared);
       }

From dc67c27cfb5f6cbf344daaa72bf0cd9fb96e7d5d Mon Sep 17 00:00:00 2001
From: Rajashekhar <rajashekhar.c@inmobi.com>
Date: Mon, 22 Jul 2019 17:43:42 +0530
Subject: [PATCH 22/25] added a unit test case

---
 .../metadata/TestCubeMetastoreClient.java     | 56 ++++++++++++++++++-
 1 file changed, 53 insertions(+), 3 deletions(-)

diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
index 6e76eda87..355f485fd 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
@@ -24,12 +24,12 @@
 import static org.apache.lens.cube.metadata.MetastoreUtil.*;
 import static org.apache.lens.cube.metadata.UpdatePeriod.*;
 import static org.apache.lens.server.api.util.LensUtil.getHashMap;
-
 import static org.testng.Assert.*;
 
 import java.text.SimpleDateFormat;
 import java.util.*;
 
+import org.apache.lens.api.metastore.XFact;
 import org.apache.lens.cube.error.LensCubeErrorCode;
 import org.apache.lens.cube.metadata.ExprColumn.ExprSpec;
 import org.apache.lens.cube.metadata.ReferencedDimAttribute.ChainRefCol;
@@ -42,11 +42,11 @@
 import org.apache.lens.server.api.error.LensException;
 import org.apache.lens.server.api.query.save.exception.PrivilegeException;
 import org.apache.lens.server.api.util.LensUtil;
-
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
 import org.apache.hadoop.hive.metastore.api.Database;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
 import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat;
 import org.apache.hadoop.hive.ql.metadata.Hive;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -56,7 +56,6 @@
 import org.apache.hadoop.hive.serde.serdeConstants;
 import org.apache.hadoop.mapred.SequenceFileInputFormat;
 import org.apache.hadoop.mapred.TextInputFormat;
-
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Test;
@@ -89,6 +88,7 @@ public class TestCubeMetastoreClient {
   private static final String CUBE_NAME_WITH_PROPS = "testMetastoreCubeWithProps";
   private static final String DERIVED_CUBE_NAME = "derivedTestMetastoreCube";
   private static final String DERIVED_CUBE_NAME_WITH_PROPS = "derivedTestMetastoreCubeWithProps";
+  private static final String X_FACT_NAME = "testMetastoreXFact";
   private static final Map<String, String> CUBE_PROPERTIES = new HashMap<>();
   private static HiveConf conf = new HiveConf(TestCubeMetastoreClient.class);
   private static FieldSchema dtPart = new FieldSchema(getDatePartitionKey(), serdeConstants.STRING_TYPE_NAME,
@@ -146,6 +146,13 @@ public static void setup() throws HiveException, AlreadyExistsException, LensExc
 
     Database database = new Database();
     database.setName(TestCubeMetastoreClient.class.getSimpleName());
+    try {
+        Hive.get().dropDatabase(TestCubeMetastoreClient.class.getSimpleName(), true, true, true);
+    } catch (NoSuchObjectException e) {
+        // TODO Auto-generated catch block
+        e.printStackTrace();
+    }
+
     Hive.get(conf).createDatabase(database);
     SessionState.get().setCurrentDatabase(TestCubeMetastoreClient.class.getSimpleName());
     client = CubeMetastoreClient.getInstance(conf);
@@ -161,6 +168,7 @@ public static void teardown() throws Exception {
     // Drop the cube
     client.dropCube(CUBE_NAME);
     client.dropCube(VIRTUAL_CUBE_NAME);
+    client.dropCube(X_FACT_NAME);
     client = CubeMetastoreClient.getInstance(conf);
     assertFalse(client.tableExists(CUBE_NAME));
     Hive.get().dropDatabase(TestCubeMetastoreClient.class.getSimpleName(), true, true, true);
@@ -1591,6 +1599,48 @@ public void testCubeFactWithTwoTimedParts() throws Exception {
     assertFalse(client.latestPartitionExists(cubeFact.getName(), c1, testDtPart.getName()));
   }
 
+  @Test(priority = 2)
+  public void testGetXFactTable() throws Exception {
+    List<FieldSchema> factColumns = new ArrayList<>(cubeMeasures.size());
+    for (CubeMeasure measure : cubeMeasures) {
+      factColumns.add(measure.getColumn());
+    }
+
+    // add one dimension of the cube
+    factColumns.add(new FieldSchema("zipcode", "int", "zip"));
+    FieldSchema itPart = new FieldSchema("it", "string", "date part");
+    FieldSchema etPart = new FieldSchema("et", "string", "date part");
+    StorageTableDesc s1 = new StorageTableDesc(TextInputFormat.class, HiveIgnoreKeyTextOutputFormat.class,
+        Lists.newArrayList(getDatePartition(), itPart, etPart), Lists.newArrayList(getDatePartitionKey(),
+        itPart.getName(), etPart.getName()));
+
+    Set<UpdatePeriod> c1Storage = new HashSet<UpdatePeriod>();
+    c1Storage.add(HOURLY);
+    Map<String, Set<UpdatePeriod>> updatePeriods = new HashMap<String, Set<UpdatePeriod>>();
+    updatePeriods.put(c1, c1Storage);
+
+    Map<String, StorageTableDesc> storageTableDescs = getHashMap("HOURLY_c1", s1);
+
+    Map<UpdatePeriod, String> updatePeriodMap = new HashMap<>();
+    updatePeriodMap.put(HOURLY, HOURLY.toString() + "_" + c1);
+
+    Map<String, Set<String>> storageTablePartitionColumns = new HashMap<String, Set<String>>();
+    storageTablePartitionColumns.put(c1, new HashSet<>());
+
+    Map<String, Map<UpdatePeriod, String>> storageUpdatePeriodMap = new HashMap<String, Map<UpdatePeriod, String>>();
+
+    storageUpdatePeriodMap.put(c1, updatePeriodMap);
+
+    client.createCubeFactTable(CUBE_NAME_WITH_PROPS, X_FACT_NAME, factColumns, updatePeriods, 0.0d, new HashMap<String,
+        String>(), storageTableDescs, storageUpdatePeriodMap, storageTablePartitionColumns);
+
+    CubeFactTable cubeFact = new CubeFactTable(CUBE_NAME_WITH_PROPS, X_FACT_NAME, factColumns, updatePeriods, 0.0d,
+        new HashMap<String, String>(), storageUpdatePeriodMap, new HashMap<String, Set<String>>());
+    XFact xfact = client.getXFactTable(cubeFact);
+    assertEquals(xfact.getCubeName(), CUBE_NAME_WITH_PROPS);
+    assertEquals(xfact.getName(), X_FACT_NAME.toLowerCase());
+  }
+
   @Test(priority = 2)
   public void testCubeFactWithThreeTimedParts() throws Exception {
     String factName = "testMetastoreFact3TimedParts";

From 8d3e015b63ff0753c89afe668e6cfe525e445c74 Mon Sep 17 00:00:00 2001
From: Rajashekhar <rajashekhar.c@inmobi.com>
Date: Mon, 22 Jul 2019 17:56:42 +0530
Subject: [PATCH 23/25] added the test case

---
 .../lens/cube/metadata/TestCubeMetastoreClient.java       | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
index 355f485fd..05edd6570 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
@@ -144,15 +144,14 @@ public static void setup() throws HiveException, AlreadyExistsException, LensExc
     conf.set(LensConfConstants.AUTHORIZER_CLASS, "org.apache.lens.cube.parse.MockAuthorizer");
     LensAuthorizer.get().init(conf);
 
-    Database database = new Database();
-    database.setName(TestCubeMetastoreClient.class.getSimpleName());
     try {
         Hive.get().dropDatabase(TestCubeMetastoreClient.class.getSimpleName(), true, true, true);
     } catch (NoSuchObjectException e) {
-        // TODO Auto-generated catch block
         e.printStackTrace();
+        fail();
     }
-
+    Database database = new Database();
+    database.setName(TestCubeMetastoreClient.class.getSimpleName());
     Hive.get(conf).createDatabase(database);
     SessionState.get().setCurrentDatabase(TestCubeMetastoreClient.class.getSimpleName());
     client = CubeMetastoreClient.getInstance(conf);
@@ -168,7 +167,6 @@ public static void teardown() throws Exception {
     // Drop the cube
     client.dropCube(CUBE_NAME);
     client.dropCube(VIRTUAL_CUBE_NAME);
-    client.dropCube(X_FACT_NAME);
     client = CubeMetastoreClient.getInstance(conf);
     assertFalse(client.tableExists(CUBE_NAME));
     Hive.get().dropDatabase(TestCubeMetastoreClient.class.getSimpleName(), true, true, true);

From af1f5d4085258861567e5fbce10c04aa877f7c7f Mon Sep 17 00:00:00 2001
From: Rajashekhar <rajashekhar.c@inmobi.com>
Date: Mon, 22 Jul 2019 18:00:32 +0530
Subject: [PATCH 24/25] fixed unit test case

---
 .../lens/cube/metadata/TestCubeMetastoreClient.java   | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
index 05edd6570..6e51c79ec 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
@@ -24,7 +24,8 @@
 import static org.apache.lens.cube.metadata.MetastoreUtil.*;
 import static org.apache.lens.cube.metadata.UpdatePeriod.*;
 import static org.apache.lens.server.api.util.LensUtil.getHashMap;
-import static org.testng.Assert.*;
+
+import static org.testng.Assert.fail;
 
 import java.text.SimpleDateFormat;
 import java.util.*;
@@ -42,6 +43,7 @@
 import org.apache.lens.server.api.error.LensException;
 import org.apache.lens.server.api.query.save.exception.PrivilegeException;
 import org.apache.lens.server.api.util.LensUtil;
+
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.api.AlreadyExistsException;
 import org.apache.hadoop.hive.metastore.api.Database;
@@ -56,6 +58,7 @@
 import org.apache.hadoop.hive.serde.serdeConstants;
 import org.apache.hadoop.mapred.SequenceFileInputFormat;
 import org.apache.hadoop.mapred.TextInputFormat;
+
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;
 import org.testng.annotations.Test;
@@ -145,10 +148,10 @@ public static void setup() throws HiveException, AlreadyExistsException, LensExc
     LensAuthorizer.get().init(conf);
 
     try {
-        Hive.get().dropDatabase(TestCubeMetastoreClient.class.getSimpleName(), true, true, true);
+      Hive.get().dropDatabase(TestCubeMetastoreClient.class.getSimpleName(), true, true, true);
     } catch (NoSuchObjectException e) {
-        e.printStackTrace();
-        fail();
+      e.printStackTrace();
+      fail();
     }
     Database database = new Database();
     database.setName(TestCubeMetastoreClient.class.getSimpleName());

From 24783d6a6b1dd9a2f0eb594a00cb1aea30394373 Mon Sep 17 00:00:00 2001
From: Rajashekhar <rajashekhar.c@inmobi.com>
Date: Mon, 22 Jul 2019 18:05:34 +0530
Subject: [PATCH 25/25] fixed unit test case

---
 .../org/apache/lens/cube/metadata/TestCubeMetastoreClient.java | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
index 6e51c79ec..ef5f1624b 100644
--- a/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
+++ b/lens-cube/src/test/java/org/apache/lens/cube/metadata/TestCubeMetastoreClient.java
@@ -25,7 +25,7 @@
 import static org.apache.lens.cube.metadata.UpdatePeriod.*;
 import static org.apache.lens.server.api.util.LensUtil.getHashMap;
 
-import static org.testng.Assert.fail;
+import static org.testng.Assert.*;
 
 import java.text.SimpleDateFormat;
 import java.util.*;
@@ -150,7 +150,6 @@ public static void setup() throws HiveException, AlreadyExistsException, LensExc
     try {
       Hive.get().dropDatabase(TestCubeMetastoreClient.class.getSimpleName(), true, true, true);
     } catch (NoSuchObjectException e) {
-      e.printStackTrace();
       fail();
     }
     Database database = new Database();