fix bugs when start multi FE at same time

fe/src/com/baidu/palo/journal/bdbje/BDBEnvironment.java

@@ -211,25 +211,51 @@ public Environment getReplicatedEnvironment() {
         return replicatedEnvironment;
     }
 
-    // return the database reference with the given name
+    // return the database reference with the given name
+    // also try to close previous opened database.
     public Database openDatabase(String dbName) {
         Database db = null;
         lock.writeLock().lock();
-        try {
+        try {
+            // find if the specified database is already opened. find and return it.
             for (java.util.Iterator<Database> iter = openedDatabases.iterator(); iter.hasNext();) {
-                Database openedDb = iter.next();
-                if (openedDb == null || openedDb.getDatabaseName() == null) {
-                    if (openedDb != null) {
-                        openedDb.close();
-                    }
-                    iter.remove();
-                    continue;
-                }
+                Database openedDb = iter.next();
+                try {
+                    if (openedDb.getDatabaseName() == null) {
+                        openedDb.close();
+                        iter.remove();
+                        continue;
+                    }
+                } catch (Exception e) {
+                    /*
+                     * In the case when 3 FE (1 master and 2 followers) start at same time,
+                     * We may catch com.sleepycat.je.rep.DatabasePreemptedException which said that
+                     * "Database xx has been forcibly closed in order to apply a replicated remove operation."
+                     * 
+                     * Because when Master FE finished to save image, it try to remove old journals,
+                     * and also remove the databases these old journals belongs to.
+                     * So after Master removed the database from replicatedEnvironment,
+                     * call db.getDatabaseName() will throw DatabasePreemptedException,
+                     * because it has already been destroyed.
+                     * 
+                     * The reason why Master can safely remove a database is because it knows that all
+                     * non-master FE have already load the journal ahead of this database. So remove the
+                     * database is safe.
+                     * 
+                     * Here we just try to close the useless database(which may be removed by Master),
+                     * so even we catch the exception, just ignore it is OK.
+                     */
+                    LOG.warn("get exception when try to close previously opened bdb database. ignore it", e);
+                    iter.remove();
+                    continue;
+                }
+
                 if (openedDb.getDatabaseName().equals(dbName)) {
                     return openedDb;
                 }
             }
-
+
+            // open the specified database.
             // the first parameter null means auto-commit
             try {
                 db = replicatedEnvironment.openDatabase(null, dbName, dbConfig);
@@ -252,7 +278,8 @@ public void removeDatabase(String dbName) {
             for (Database db : openedDatabases) {
                 String name = db.getDatabaseName();
                 if (dbName.equals(name)) {
-                    db.close();
+                    db.close();
+                    LOG.info("database {} has been closed", name);
                     targetDbName = name;
                     break;
                 }

fe/src/com/baidu/palo/qe/StmtExecutor.java

@@ -91,7 +91,7 @@ public class StmtExecutor {
     private RuntimeProfile summaryProfile;
     private volatile Coordinator coord = null;
     private MasterOpExecutor masterOpExecutor = null;
-    private RedirectStatus redicrtStatus = null;
+    private RedirectStatus redirectStatus = null;
     private Planner planner;
     private boolean isProxy;
     private ShowResultSet proxyResultSet = null;
@@ -139,10 +139,10 @@ public boolean isForwardToMaster() {
             return false;
         }
 
-        if (redicrtStatus == null) {
+        if (redirectStatus == null) {
             return false;
         } else {
-            return redicrtStatus.isForwardToMaster();
+            return redirectStatus.isForwardToMaster();
         }
     }
 
@@ -263,7 +263,7 @@ public void execute() throws Exception {
     }
 
     private void forwardToMaster() throws Exception {
-        masterOpExecutor = new MasterOpExecutor(originStmt, context, redicrtStatus);
+        masterOpExecutor = new MasterOpExecutor(originStmt, context, redirectStatus);
         LOG.debug("need to transfer to Master originStmt={}", originStmt);
         masterOpExecutor.execute();
     }
@@ -305,7 +305,7 @@ private void analyze() throws AnalysisException {
         SqlParser parser = new SqlParser(input);
         try {
             parsedStmt = (StatementBase) parser.parse().value;
-            redicrtStatus = parsedStmt.getRedirectStatus();
+            redirectStatus = parsedStmt.getRedirectStatus();
         } catch (Error e) {
             LOG.warn("error happens when parsing sql: {}", e);
             throw new AnalysisException("sql parsing error, please check your sql");