HDDS-13553. Recon Staging DB for OM full db reprocess

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ReconContainerMetadataManager.java

@@ -22,13 +22,15 @@
 import java.util.UUID;
 import org.apache.hadoop.hdds.annotation.InterfaceStability;
 import org.apache.hadoop.hdds.utils.db.BatchOperation;
+import org.apache.hadoop.hdds.utils.db.DBStore;
 import org.apache.hadoop.hdds.utils.db.RDBBatchOperation;
 import org.apache.hadoop.hdds.utils.db.Table;
 import org.apache.hadoop.hdds.utils.db.TableIterator;
 import org.apache.hadoop.ozone.recon.api.types.ContainerKeyPrefix;
 import org.apache.hadoop.ozone.recon.api.types.ContainerMetadata;
 import org.apache.hadoop.ozone.recon.api.types.KeyPrefixContainer;
 import org.apache.hadoop.ozone.recon.scm.ContainerReplicaHistory;
+import org.apache.hadoop.ozone.recon.spi.impl.ReconDBProvider;
 import org.apache.hadoop.ozone.util.SeekableIterator;
 
 /**
@@ -57,6 +59,21 @@ void reinitWithNewContainerDataFromOm(Map<ContainerKeyPrefix, Integer>
   void storeContainerKeyMapping(ContainerKeyPrefix containerKeyPrefix,
                                 Integer count) throws IOException;
 
+  /**
+   * Returns staged DB container metadata manager.
+   *
+   * @param stagedReconDbStore staged Recon DB store
+   * @return ReconContainerMetadataManager
+   */
+  ReconContainerMetadataManager getStagedReconContainerMetadataManager(DBStore stagedReconDbStore);
+
+  /**
+   * reinitialize the ReconContainerMetadataManage.
+   *
+   * @param reconDBProvider recon DB provider to reinitialize with.
+   */
+  void reinitialize(ReconDBProvider reconDBProvider);
+
   /**
    * Store the container to Key prefix mapping into a batch.
    *

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/ReconNamespaceSummaryManager.java

@@ -20,16 +20,22 @@
 import java.io.IOException;
 import org.apache.hadoop.hdds.annotation.InterfaceStability;
 import org.apache.hadoop.hdds.utils.db.BatchOperation;
+import org.apache.hadoop.hdds.utils.db.DBStore;
 import org.apache.hadoop.hdds.utils.db.RDBBatchOperation;
 import org.apache.hadoop.ozone.om.OMMetadataManager;
 import org.apache.hadoop.ozone.recon.api.types.NSSummary;
+import org.apache.hadoop.ozone.recon.spi.impl.ReconDBProvider;
 
 /**
  * Interface for DB operations on NSSummary.
  */
 @InterfaceStability.Unstable
 public interface ReconNamespaceSummaryManager {
 
+  ReconNamespaceSummaryManager getStagedNsSummaryManager(DBStore dbStore) throws IOException;
+
+  void reinitialize(ReconDBProvider reconDBProvider) throws IOException;
+
   void clearNSSummaryTable() throws IOException;
 
   @Deprecated

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconContainerMetadataManagerImpl.java

@@ -83,17 +83,34 @@ public class ReconContainerMetadataManagerImpl
 
   private DBStore containerDbStore;
 
-  @Inject
   private Configuration sqlConfiguration;
 
-  @Inject
   private ReconOMMetadataManager omMetadataManager;
 
   @Inject
-  public ReconContainerMetadataManagerImpl(ReconDBProvider reconDBProvider,
-                                           Configuration sqlConfiguration) {
-    containerDbStore = reconDBProvider.getDbStore();
+  public ReconContainerMetadataManagerImpl(
+      ReconDBProvider reconDBProvider, Configuration sqlConfiguration, ReconOMMetadataManager omMetadataManager) {
+    this(reconDBProvider.getDbStore(), sqlConfiguration, omMetadataManager);
+  }
+
+  private ReconContainerMetadataManagerImpl(
+      DBStore reconDBStore, Configuration sqlConfiguration, ReconOMMetadataManager omMetadataManager) {
+    containerDbStore = reconDBStore;
     globalStatsDao = new GlobalStatsDao(sqlConfiguration);
+    this.sqlConfiguration = sqlConfiguration;
+    this.omMetadataManager = omMetadataManager;
+    initializeTables();
+  }
+
+  @Override
+  public ReconContainerMetadataManager getStagedReconContainerMetadataManager(
+      DBStore stagedReconDbStore) {
+    return new ReconContainerMetadataManagerImpl(stagedReconDbStore, sqlConfiguration, omMetadataManager);
+  }
+
+  @Override
+  public void reinitialize(ReconDBProvider reconDBProvider) {
+    containerDbStore = reconDBProvider.getDbStore();
     initializeTables();
   }
 

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconDBProvider.java

@@ -24,7 +24,10 @@
 import com.google.inject.ProvisionException;
 import java.io.File;
 import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.StandardCopyOption;
 import javax.inject.Inject;
+import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.utils.db.DBStore;
 import org.apache.hadoop.hdds.utils.db.DBStoreBuilder;
@@ -39,6 +42,8 @@
  * Provider for Recon's RDB.
  */
 public class ReconDBProvider {
+  private static final String STAGED_EXT = ".staged";
+  private static final String BACKUP_EXT = ".backup";
   private OzoneConfiguration configuration;
   private ReconUtils reconUtils;
   private DBStore dbStore;
@@ -54,20 +59,47 @@ public class ReconDBProvider {
     this.dbStore = provideReconDB();
   }
 
+  private ReconDBProvider(OzoneConfiguration configuration, ReconUtils reconUtils, DBStore dbStore) {
+    this.configuration = configuration;
+    this.reconUtils = reconUtils;
+    this.dbStore = dbStore;
+  }
+
+  public ReconDBProvider getStagedReconDBProvider() throws IOException {
+    File reconDbDir = reconUtils.getReconDbDir(configuration, OZONE_RECON_DB_DIR);
+    String stagedDbName = RECON_CONTAINER_KEY_DB + STAGED_EXT;
+    FileUtils.deleteDirectory(new File(reconDbDir, stagedDbName));
+    DBStore db = initializeDBStore(configuration, stagedDbName);
+    if (db == null) {
+      throw new ProvisionException("Unable to initialize staged recon container DBStore");
+    }
+    return new ReconDBProvider(configuration, reconUtils, db);
+  }
+
   public DBStore provideReconDB() {
     DBStore db;
-    File reconDbDir =
-            reconUtils.getReconDbDir(configuration, OZONE_RECON_DB_DIR);
-    File lastKnownContainerKeyDb =
-            reconUtils.getLastKnownDB(reconDbDir, RECON_CONTAINER_KEY_DB);
-    if (lastKnownContainerKeyDb != null) {
-      LOG.info("Last known Recon DB : {}",
-              lastKnownContainerKeyDb.getAbsolutePath());
-      db = initializeDBStore(configuration,
-              lastKnownContainerKeyDb.getName());
-    } else {
-      db = getNewDBStore(configuration);
+    try {
+      // handle recover of last known container as old format removing timestamp
+      File reconDbDir = reconUtils.getReconDbDir(configuration, OZONE_RECON_DB_DIR);
+      File lastKnownContainerKeyDb = reconUtils.getLastKnownDB(reconDbDir, RECON_CONTAINER_KEY_DB);
+      if (lastKnownContainerKeyDb != null) {
+        LOG.info("Last known Recon DB : {}", lastKnownContainerKeyDb.getAbsolutePath());
+        Files.move(lastKnownContainerKeyDb.toPath(), new File(reconDbDir, RECON_CONTAINER_KEY_DB).toPath(),
+            StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
+      }
+      // if db does not exist, check if a backup exists and restore it. This can happen when replace with
+      // staged db fails and backup is not restored at that point of time.
+      if (!new File(reconDbDir, RECON_CONTAINER_KEY_DB).exists() &&
+          new File(reconDbDir, RECON_CONTAINER_KEY_DB + BACKUP_EXT).exists()) {
+        LOG.info("Recon DB backup found, restoring from backup.");
+        Files.move(new File(reconDbDir, RECON_CONTAINER_KEY_DB + BACKUP_EXT).toPath(),
+            new File(reconDbDir, RECON_CONTAINER_KEY_DB).toPath(),
+            StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
+      }
+    } catch (IOException e) {
+      throw new ProvisionException("Unable to recover container DB path.", e);
     }
+    db = initializeDBStore(configuration, RECON_CONTAINER_KEY_DB);
     if (db == null) {
       throw new ProvisionException("Unable to provide instance of DBStore " +
               "store.");
@@ -92,11 +124,6 @@ static void truncateTable(Table table) throws IOException {
     }
   }
 
-  static DBStore getNewDBStore(OzoneConfiguration configuration) {
-    String dbName = RECON_CONTAINER_KEY_DB + "_" + System.currentTimeMillis();
-    return initializeDBStore(configuration, dbName);
-  }
-
   private static DBStore initializeDBStore(OzoneConfiguration configuration,
                                            String dbName) {
     DBStore dbStore = null;
@@ -115,4 +142,29 @@ public void close() throws Exception {
       dbStore = null;
     }
   }
+
+  public void replaceStagedDb(ReconDBProvider stagedReconDBProvider) throws Exception {
+    File dbPath = dbStore.getDbLocation();
+    File stagedDbPath = stagedReconDBProvider.getDbStore().getDbLocation();
+    File backupPath = new File(dbPath.getAbsolutePath() + BACKUP_EXT);
+    stagedReconDBProvider.close();
+    try {
+      FileUtils.deleteDirectory(backupPath);
+      close();
+      Files.move(dbPath.toPath(), backupPath.toPath(), StandardCopyOption.ATOMIC_MOVE,
+          StandardCopyOption.REPLACE_EXISTING);
+      Files.move(stagedDbPath.toPath(), dbPath.toPath(), StandardCopyOption.ATOMIC_MOVE,
+          StandardCopyOption.REPLACE_EXISTING);
+      dbStore = initializeDBStore(configuration, dbPath.getName());
+    } catch (Exception e) {
+      if (dbStore == null) {
+        Files.move(dbPath.toPath(), stagedDbPath.toPath(), StandardCopyOption.ATOMIC_MOVE,
+            StandardCopyOption.REPLACE_EXISTING);
+        Files.move(backupPath.toPath(), dbPath.toPath(), StandardCopyOption.ATOMIC_MOVE,
+            StandardCopyOption.REPLACE_EXISTING);
+        dbStore = initializeDBStore(configuration, dbPath.getName());
+      }
+      throw e;
+    }
+  }
 }

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ReconNamespaceSummaryManagerImpl.java

@@ -44,11 +44,27 @@ public class ReconNamespaceSummaryManagerImpl
   @Inject
   public ReconNamespaceSummaryManagerImpl(ReconDBProvider reconDBProvider, NSSummaryTask nsSummaryTask)
           throws IOException {
-    namespaceDbStore = reconDBProvider.getDbStore();
+    this(reconDBProvider.getDbStore(), nsSummaryTask);
+  }
+
+  private ReconNamespaceSummaryManagerImpl(DBStore dbStore, NSSummaryTask nsSummaryTask)
+      throws IOException {
+    namespaceDbStore = dbStore;
     this.nsSummaryTable = NAMESPACE_SUMMARY.getTable(namespaceDbStore);
     this.nsSummaryTask = nsSummaryTask;
   }
 
+  @Override
+  public ReconNamespaceSummaryManager getStagedNsSummaryManager(DBStore dbStore) throws IOException {
+    return new ReconNamespaceSummaryManagerImpl(dbStore, nsSummaryTask);
+  }
+
+  @Override
+  public void reinitialize(ReconDBProvider reconDBProvider) throws IOException {
+    namespaceDbStore = reconDBProvider.getDbStore();
+    this.nsSummaryTable = NAMESPACE_SUMMARY.getTable(namespaceDbStore);
+  }
+
   @Override
   public void clearNSSummaryTable() throws IOException {
     truncateTable(nsSummaryTable);

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/ContainerKeyMapperTaskFSO.java

@@ -18,11 +18,14 @@
 package org.apache.hadoop.ozone.recon.tasks;
 
 import com.google.inject.Inject;
+import java.io.IOException;
 import java.util.Map;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.db.DBStore;
 import org.apache.hadoop.ozone.om.OMMetadataManager;
 import org.apache.hadoop.ozone.om.helpers.BucketLayout;
 import org.apache.hadoop.ozone.recon.ReconServerConfigKeys;
+import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager;
 import org.apache.hadoop.ozone.recon.spi.ReconContainerMetadataManager;
 
 /**
@@ -40,6 +43,13 @@ public ContainerKeyMapperTaskFSO(ReconContainerMetadataManager reconContainerMet
     this.ozoneConfiguration = configuration;
   }
 
+  @Override
+  public ReconOmTask getStagedTask(ReconOMMetadataManager stagedOmMetadataManager, DBStore stagedReconDbStore)
+      throws IOException {
+    return new ContainerKeyMapperTaskFSO(
+        reconContainerMetadataManager.getStagedReconContainerMetadataManager(stagedReconDbStore), ozoneConfiguration);
+  }
+
   @Override
   public TaskResult reprocess(OMMetadataManager omMetadataManager) {
     long containerKeyFlushToDBMaxThreshold = ozoneConfiguration.getLong(

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/ContainerKeyMapperTaskOBS.java

@@ -18,11 +18,14 @@
 package org.apache.hadoop.ozone.recon.tasks;
 
 import com.google.inject.Inject;
+import java.io.IOException;
 import java.util.Map;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.db.DBStore;
 import org.apache.hadoop.ozone.om.OMMetadataManager;
 import org.apache.hadoop.ozone.om.helpers.BucketLayout;
 import org.apache.hadoop.ozone.recon.ReconServerConfigKeys;
+import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager;
 import org.apache.hadoop.ozone.recon.spi.ReconContainerMetadataManager;
 
 /**
@@ -40,6 +43,13 @@ public ContainerKeyMapperTaskOBS(ReconContainerMetadataManager reconContainerMet
     this.ozoneConfiguration = configuration;
   }
 
+  @Override
+  public ReconOmTask getStagedTask(ReconOMMetadataManager stagedOmMetadataManager, DBStore stagedReconDbStore)
+      throws IOException {
+    return new ContainerKeyMapperTaskOBS(
+        reconContainerMetadataManager.getStagedReconContainerMetadataManager(stagedReconDbStore), ozoneConfiguration);
+  }
+
   @Override
   public TaskResult reprocess(OMMetadataManager omMetadataManager) {
     long containerKeyFlushToDBMaxThreshold = ozoneConfiguration.getLong(

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/NSSummaryTask.java

@@ -39,6 +39,7 @@
 import javax.inject.Inject;
 import org.apache.commons.lang3.tuple.Pair;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.utils.db.DBStore;
 import org.apache.hadoop.ozone.om.OMMetadataManager;
 import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager;
 import org.apache.hadoop.ozone.recon.spi.ReconNamespaceSummaryManager;
@@ -76,6 +77,7 @@ public class NSSummaryTask implements ReconOmTask {
 
   private final ReconNamespaceSummaryManager reconNamespaceSummaryManager;
   private final ReconOMMetadataManager reconOMMetadataManager;
+  private final OzoneConfiguration ozoneConfiguration;
   private final NSSummaryTaskWithFSO nsSummaryTaskWithFSO;
   private final NSSummaryTaskWithLegacy nsSummaryTaskWithLegacy;
   private final NSSummaryTaskWithOBS nsSummaryTaskWithOBS;
@@ -98,6 +100,7 @@ public NSSummaryTask(ReconNamespaceSummaryManager
                        ozoneConfiguration) {
     this.reconNamespaceSummaryManager = reconNamespaceSummaryManager;
     this.reconOMMetadataManager = reconOMMetadataManager;
+    this.ozoneConfiguration = ozoneConfiguration;
     long nsSummaryFlushToDBMaxThreshold = ozoneConfiguration.getLong(
         OZONE_RECON_NSSUMMARY_FLUSH_TO_DB_MAX_THRESHOLD,
         OZONE_RECON_NSSUMMARY_FLUSH_TO_DB_MAX_THRESHOLD_DEFAULT);
@@ -112,6 +115,14 @@ public NSSummaryTask(ReconNamespaceSummaryManager
         reconNamespaceSummaryManager, reconOMMetadataManager, nsSummaryFlushToDBMaxThreshold);
   }
 
+  @Override
+  public NSSummaryTask getStagedTask(ReconOMMetadataManager stagedOmMetadataManager, DBStore stagedReconDbStore)
+      throws IOException {
+    ReconNamespaceSummaryManager stagedNsSummaryManager =
+        reconNamespaceSummaryManager.getStagedNsSummaryManager(stagedReconDbStore);
+    return new NSSummaryTask(stagedNsSummaryManager, stagedOmMetadataManager, ozoneConfiguration);
+  }
+
   @Override
   public String getTaskName() {
     return "NSSummaryTask";

hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/ReconOmTask.java

@@ -17,9 +17,12 @@
 
 package org.apache.hadoop.ozone.recon.tasks;
 
+import java.io.IOException;
 import java.util.Collections;
 import java.util.Map;
+import org.apache.hadoop.hdds.utils.db.DBStore;
 import org.apache.hadoop.ozone.om.OMMetadataManager;
+import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager;
 
 /**
  * Interface used to denote a Recon task that needs to act on OM DB events.
@@ -62,6 +65,18 @@ TaskResult process(OMUpdateEventBatch events,
    */
   TaskResult reprocess(OMMetadataManager omMetadataManager);
 
+  /**
+   * Returns a staged task that can be used to reprocess events.
+   * @param stagedOmMetadataManager  om metadata manager for staged OM DB
+   * @param stagedReconDbStore recon DB store for staged
+   * @return task that can be used to reprocess events
+   * @throws IOException exception
+   */
+  default ReconOmTask getStagedTask(ReconOMMetadataManager stagedOmMetadataManager, DBStore stagedReconDbStore)
+      throws IOException {
+    return this;
+  }
+
   /**
    * Represents the result of a task execution, including the task name,
    * sub-task seek positions, and success status.