HDDS-13949. Move dbTxSequenceNumber from SnapshotInfo to LocalDataYaml file

hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java

@@ -222,6 +222,7 @@ public final class OzoneConsts {
   public static final String OM_SST_FILE_INFO_END_KEY = "endKey";
   public static final String OM_SST_FILE_INFO_COL_FAMILY = "columnFamily";
   public static final String OM_SLD_TXN_INFO = "transactionInfo";
+  public static final String OM_SLD_DB_TXN_SEQ_NUMBER = "dbTxSequenceNumber";
 
   // YAML fields for .container files
   public static final String CONTAINER_ID = "containerID";

hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java

@@ -771,9 +771,7 @@ private Path getSSTFullPath(SstFileInfo sstFileInfo, Path... dbPaths) throws IOE
    * @param dest destination snapshot
    * @param versionMap version map containing the connection between source snapshot version and dest snapshot version.
    * @param tablesToLookup tablesToLookup set of table (column family) names used to restrict which SST files to return.
-   * @return A list of SST files without extension.
-   *         e.g. ["/path/to/sstBackupDir/000050.sst",
-   *               "/path/to/sstBackupDir/000060.sst"]
+   * @return map of SST file absolute paths with extension to SstFileInfo.
    */
   public synchronized Optional<Map<Path, SstFileInfo>> getSSTDiffListWithFullPath(DifferSnapshotInfo src,
       DifferSnapshotInfo dest, Map<Integer, Integer> versionMap, TablePrefixInfo prefixInfo,

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java

@@ -71,10 +71,6 @@ public final class SnapshotInfo implements Auditable, CopyObject<SnapshotInfo> {
   private UUID pathPreviousSnapshotId;
   private UUID globalPreviousSnapshotId;
   private String snapshotPath; // snapshot mask
-  /**
-   * RocksDB's transaction sequence number at the time of checkpoint creation.
-   */
-  private long dbTxSequenceNumber;
   private boolean deepClean;
   private boolean sstFiltered;
   private long referencedSize;
@@ -98,7 +94,6 @@ private SnapshotInfo(Builder b) {
     this.pathPreviousSnapshotId = b.pathPreviousSnapshotId;
     this.globalPreviousSnapshotId = b.globalPreviousSnapshotId;
     this.snapshotPath = b.snapshotPath;
-    this.dbTxSequenceNumber = b.dbTxSequenceNumber;
     this.deepClean = b.deepClean;
     this.sstFiltered = b.sstFiltered;
     this.referencedSize = b.referencedSize;
@@ -221,7 +216,6 @@ public SnapshotInfo.Builder toBuilder() {
         .setPathPreviousSnapshotId(pathPreviousSnapshotId)
         .setGlobalPreviousSnapshotId(globalPreviousSnapshotId)
         .setSnapshotPath(snapshotPath)
-        .setDbTxSequenceNumber(dbTxSequenceNumber)
         .setDeepClean(deepClean)
         .setSstFiltered(sstFiltered)
         .setReferencedSize(referencedSize)
@@ -249,7 +243,6 @@ public static class Builder {
     private UUID pathPreviousSnapshotId;
     private UUID globalPreviousSnapshotId;
     private String snapshotPath;
-    private long dbTxSequenceNumber;
     private boolean deepClean;
     private boolean sstFiltered;
     private long referencedSize;
@@ -327,12 +320,6 @@ public Builder setSnapshotPath(String snapshotPath) {
       return this;
     }
 
-    /** @param dbTxSequenceNumber - RDB latest transaction sequence number. */
-    public Builder setDbTxSequenceNumber(long dbTxSequenceNumber) {
-      this.dbTxSequenceNumber = dbTxSequenceNumber;
-      return this;
-    }
-
     /** @param deepClean - To be deep cleaned status for snapshot. */
     public Builder setDeepClean(boolean deepClean) {
       this.deepClean = deepClean;
@@ -441,7 +428,6 @@ public OzoneManagerProtocolProtos.SnapshotInfo getProtobuf() {
     }
 
     sib.setSnapshotPath(snapshotPath)
-        .setDbTxSequenceNumber(dbTxSequenceNumber)
         .setDeepClean(deepClean);
     return sib.build();
   }
@@ -523,9 +509,7 @@ public static Builder builderFromProtobuf(
     if (snapshotInfoProto.hasCreateTransactionInfo()) {
       osib.setCreateTransactionInfo(snapshotInfoProto.getCreateTransactionInfo());
     }
-
-    osib.setSnapshotPath(snapshotInfoProto.getSnapshotPath())
-        .setDbTxSequenceNumber(snapshotInfoProto.getDbTxSequenceNumber());
+    osib.setSnapshotPath(snapshotInfoProto.getSnapshotPath());
 
     return osib;
   }
@@ -568,14 +552,6 @@ public String getCheckpointDirName(int version) {
     return getCheckpointDirName(getSnapshotId(), version);
   }
 
-  public long getDbTxSequenceNumber() {
-    return dbTxSequenceNumber;
-  }
-
-  public void setDbTxSequenceNumber(long dbTxSequenceNumber) {
-    this.dbTxSequenceNumber = dbTxSequenceNumber;
-  }
-
   /**
    * Get the table key for this snapshot.
    */
@@ -761,7 +737,6 @@ public String toString() {
         ", pathPreviousSnapshotId: '" + pathPreviousSnapshotId + '\'' +
         ", globalPreviousSnapshotId: '" + globalPreviousSnapshotId + '\'' +
         ", snapshotPath: '" + snapshotPath + '\'' +
-        ", dbTxSequenceNumber: '" + dbTxSequenceNumber + '\'' +
         ", deepClean: '" + deepClean + '\'' +
         ", sstFiltered: '" + sstFiltered + '\'' +
         ", referencedSize: '" + referencedSize + '\'' +

hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmSnapshotInfo.java

@@ -45,7 +45,6 @@ public class TestOmSnapshotInfo {
   private static final UUID GLOBAL_PREVIOUS_SNAPSHOT_ID =
       PATH_PREVIOUS_SNAPSHOT_ID;
   private static final String SNAPSHOT_PATH = "test/path";
-  private static final long DB_TX_SEQUENCE_NUMBER = 12345L;
 
   private SnapshotInfo createSnapshotInfo() {
     return new SnapshotInfo.Builder()
@@ -59,7 +58,6 @@ private SnapshotInfo createSnapshotInfo() {
         .setPathPreviousSnapshotId(PATH_PREVIOUS_SNAPSHOT_ID)
         .setGlobalPreviousSnapshotId(GLOBAL_PREVIOUS_SNAPSHOT_ID)
         .setSnapshotPath(SNAPSHOT_PATH)
-        .setDbTxSequenceNumber(DB_TX_SEQUENCE_NUMBER)
         .setDeepClean(false)
         .setSstFiltered(false)
         .setReferencedSize(2000L)
@@ -84,7 +82,6 @@ private OzoneManagerProtocolProtos.SnapshotInfo createSnapshotInfoProto() {
         .setPathPreviousSnapshotID(toProtobuf(PATH_PREVIOUS_SNAPSHOT_ID))
         .setGlobalPreviousSnapshotID(toProtobuf(GLOBAL_PREVIOUS_SNAPSHOT_ID))
         .setSnapshotPath(SNAPSHOT_PATH)
-        .setDbTxSequenceNumber(DB_TX_SEQUENCE_NUMBER)
         .setDeepClean(false)
         .setSstFiltered(false)
         .setReferencedSize(2000L)
@@ -164,8 +161,6 @@ public void testSnapshotInfoProtoToSnapshotInfo() {
         snapshotInfoActual.getBucketName());
     assertEquals(snapshotInfoExpected.getSnapshotStatus(),
         snapshotInfoActual.getSnapshotStatus());
-    assertEquals(snapshotInfoExpected.getDbTxSequenceNumber(),
-        snapshotInfoActual.getDbTxSequenceNumber());
     assertEquals(snapshotInfoExpected.isDeepCleaned(),
         snapshotInfoActual.isDeepCleaned());
     assertEquals(snapshotInfoExpected.isSstFiltered(),

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOMSnapshotDAG.java

@@ -161,7 +161,8 @@ private DifferSnapshotVersion getDifferSnapshotInfo(
               throw new IllegalStateException(String.format("Duplicate key %s", u));
             }, TreeMap::new));
       DifferSnapshotInfo dsi = new DifferSnapshotInfo((version) -> Paths.get(checkpointPath),
-          snapshotInfo.getSnapshotId(), snapshotInfo.getDbTxSequenceNumber(), versionSstFiles);
+          snapshotInfo.getSnapshotId(), snapshotLocalData.getSnapshotLocalData().getDbTxSequenceNumber(),
+          versionSstFiles);
       return new DifferSnapshotVersion(dsi, 0, COLUMN_FAMILIES_TO_TRACK_IN_DAG);
     }
   }

hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto

@@ -881,7 +881,7 @@ message SnapshotInfo {
   optional hadoop.hdds.UUID globalPreviousSnapshotID = 9;
   optional string snapshotPath = 10;
   optional string checkpointDir = 11 [deprecated = true];
-  optional int64 dbTxSequenceNumber = 12;
+  optional int64 dbTxSequenceNumber = 12 [deprecated = true];
   optional bool deepClean = 13;
   optional bool sstFiltered = 14;
   // snapshot reference size before any key replication or EC

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotLocalData.java

@@ -67,6 +67,9 @@ public class OmSnapshotLocalData implements WithChecksum<OmSnapshotLocalData> {
   // Stores the transactionInfo corresponding to OM when the snaphot is purged.
   private TransactionInfo transactionInfo;
 
+  // Stores the rocksDB's transaction sequence number at the time of snapshot creation.
+  private long dbTxSequenceNumber;
+
   // Map of version to VersionMeta, using linkedHashMap since the order of the map needs to be deterministic for
   // checksum computation.
   private final LinkedHashMap<Integer, VersionMeta> versionSstFileInfos;
@@ -78,7 +81,7 @@ public class OmSnapshotLocalData implements WithChecksum<OmSnapshotLocalData> {
    * Creates a OmSnapshotLocalData object with default values.
    */
   public OmSnapshotLocalData(UUID snapshotId, List<LiveFileMetaData> notDefraggedSSTFileList, UUID previousSnapshotId,
-      TransactionInfo transactionInfo) {
+      TransactionInfo transactionInfo, long dbTxSequenceNumber) {
     this.snapshotId = snapshotId;
     this.isSSTFiltered = false;
     this.lastDefragTime = 0L;
@@ -89,9 +92,14 @@ public OmSnapshotLocalData(UUID snapshotId, List<LiveFileMetaData> notDefraggedS
     this.version = 0;
     this.previousSnapshotId = previousSnapshotId;
     this.transactionInfo = transactionInfo;
+    this.dbTxSequenceNumber = dbTxSequenceNumber;
     setChecksumTo0ByteArray();
   }
 
+  public long getDbTxSequenceNumber() {
+    return dbTxSequenceNumber;
+  }
+
   /**
    * Copy constructor to create a deep copy of OmSnapshotLocalData object.
    * @param source The source OmSnapshotLocalData to copy from
@@ -108,6 +116,7 @@ public OmSnapshotLocalData(OmSnapshotLocalData source) {
     this.versionSstFileInfos = new LinkedHashMap<>();
     setVersionSstFileInfos(source.versionSstFileInfos);
     this.transactionInfo = source.transactionInfo;
+    this.dbTxSequenceNumber = source.dbTxSequenceNumber;
   }
 
   public TransactionInfo getTransactionInfo() {

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotLocalDataYaml.java

@@ -172,9 +172,10 @@ public Object construct(Node node) {
         final String prevSnapIdStr = (String) nodes.get(OzoneConsts.OM_SLD_PREV_SNAP_ID);
         UUID prevSnapId = prevSnapIdStr != null ? UUID.fromString(prevSnapIdStr) : null;
         final String purgeTxInfoStr = (String) nodes.get(OzoneConsts.OM_SLD_TXN_INFO);
+        final long dbTxnSeqNumber = ((Number)nodes.get(OzoneConsts.OM_SLD_DB_TXN_SEQ_NUMBER)).longValue();
         TransactionInfo transactionInfo = purgeTxInfoStr != null ? TransactionInfo.valueOf(purgeTxInfoStr) : null;
         OmSnapshotLocalData snapshotLocalData = new OmSnapshotLocalData(snapId, Collections.emptyList(), prevSnapId,
-            transactionInfo);
+            transactionInfo, dbTxnSeqNumber);
 
         // Set version from YAML
         Integer version = (Integer) nodes.get(OzoneConsts.OM_SLD_VERSION);

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java

@@ -32,7 +32,6 @@
 import org.apache.hadoop.hdds.client.DefaultReplicationConfig;
 import org.apache.hadoop.hdds.client.ReplicationConfig;
 import org.apache.hadoop.hdds.utils.TransactionInfo;
-import org.apache.hadoop.hdds.utils.db.RDBStore;
 import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
 import org.apache.hadoop.hdds.utils.db.cache.CacheValue;
 import org.apache.hadoop.ozone.OmUtils;
@@ -167,12 +166,6 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut
         throw new OMException("Snapshot already exists", FILE_ALREADY_EXISTS);
       }
 
-      // Note down RDB latest transaction sequence number, which is used
-      // as snapshot generation in the Differ.
-      final long dbLatestSequenceNumber =
-          ((RDBStore) omMetadataManager.getStore()).getDb()
-              .getLatestSequenceNumber();
-      snapshotInfo.setDbTxSequenceNumber(dbLatestSequenceNumber);
       ByteString txnBytes = TransactionInfo.valueOf(context.getTermIndex()).toByteString();
       snapshotInfo.setCreateTransactionInfo(txnBytes);
       snapshotInfo.setLastTransactionInfo(txnBytes);

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotLocalDataManager.java

@@ -181,11 +181,14 @@ public String getSnapshotLocalPropertyYamlPath(UUID snapshotId) {
   public void createNewOmSnapshotLocalDataFile(RDBStore snapshotStore, SnapshotInfo snapshotInfo) throws IOException {
     try (WritableOmSnapshotLocalDataProvider snapshotLocalData =
              new WritableOmSnapshotLocalDataProvider(snapshotInfo.getSnapshotId(),
-                 () -> Pair.of(new OmSnapshotLocalData(snapshotInfo.getSnapshotId(),
-                         getLiveSSTFilesForCFs(snapshotStore.getDb().getManagedRocksDb(),
-                             COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT),
-                         snapshotInfo.getPathPreviousSnapshotId(), null),
-                     null))) {
+                 () -> {
+                   List<LiveFileMetaData> lfms = getLiveSSTFilesForCFs(snapshotStore.getDb().getManagedRocksDb(),
+                       COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT);
+                   long dbTxnSeqNumber = lfms.stream().mapToLong(LiveFileMetaData::largestSeqno).max().orElse(0L);
+                   OmSnapshotLocalData localData = new OmSnapshotLocalData(snapshotInfo.getSnapshotId(),
+                       lfms, snapshotInfo.getPathPreviousSnapshotId(), null, dbTxnSeqNumber);
+                   return Pair.of(localData, null);
+                 })) {
       snapshotLocalData.commit();
     }
   }
@@ -263,16 +266,18 @@ private void addMissingSnapshotYamlFiles(
         // Create a yaml file for snapshots which are missing
         if (!snapshotLocalDataFile.exists()) {
           List<LiveFileMetaData> sstList = Collections.emptyList();
+          long dbTxnSeqNumber = 0L;
           if (snapshotInfo.getSnapshotStatus() == SNAPSHOT_ACTIVE) {
             try (OmMetadataManagerImpl snapshotMetadataManager = defaultSnapProvider.apply(snapshotInfo)) {
               ManagedRocksDB snapDB = ((RDBStore)snapshotMetadataManager.getStore()).getDb().getManagedRocksDb();
               sstList = getLiveSSTFilesForCFs(snapDB, COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT);
+              dbTxnSeqNumber = sstList.stream().mapToLong(LiveFileMetaData::largestSeqno).max().orElse(0L);
             } catch (Exception e) {
               throw new IOException(e);
             }
           }
           OmSnapshotLocalData snapshotLocalData = new OmSnapshotLocalData(snapshotId, sstList,
-              snapshotInfo.getPathPreviousSnapshotId(), null);
+              snapshotInfo.getPathPreviousSnapshotId(), null, dbTxnSeqNumber);
           // Set needsDefrag to true to indicate that the snapshot needs to be defragmented, since the snapshot has
           // never been defragmented before.
           snapshotLocalData.setNeedsDefrag(true);

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/diff/delta/RDBDifferComputer.java

@@ -73,9 +73,9 @@ public Optional<Map<Path, Pair<Path, SstFileInfo>>> computeDeltaFiles(SnapshotIn
     if (differ != null) {
       try (OmSnapshotLocalDataManager.ReadableOmSnapshotLocalDataProvider snapProvider =
                getLocalDataProvider(toSnapshot.getSnapshotId(), fromSnapshot.getSnapshotId())) {
-        final DifferSnapshotInfo fromDSI = getDSIFromSI(getActiveMetadataManager(), fromSnapshot,
+        final DifferSnapshotInfo fromDSI = toDifferSnapshotInfo(getActiveMetadataManager(), fromSnapshot,
             snapProvider.getPreviousSnapshotLocalData());
-        final DifferSnapshotInfo toDSI = getDSIFromSI(getActiveMetadataManager(), toSnapshot,
+        final DifferSnapshotInfo toDSI = toDifferSnapshotInfo(getActiveMetadataManager(), toSnapshot,
             snapProvider.getSnapshotLocalData());
         final Map<Integer, Integer> versionMap = snapProvider.getSnapshotLocalData().getVersionSstFileInfos().entrySet()
             .stream().collect(toMap(Map.Entry::getKey, entry -> entry.getValue().getPreviousSnapshotVersion()));
@@ -98,10 +98,10 @@ public Optional<Map<Path, Pair<Path, SstFileInfo>>> computeDeltaFiles(SnapshotIn
   /**
    * Convert from SnapshotInfo to DifferSnapshotInfo.
    */
-  private static DifferSnapshotInfo getDSIFromSI(OMMetadataManager activeOmMetadataManager,
+  private static DifferSnapshotInfo toDifferSnapshotInfo(OMMetadataManager activeOmMetadataManager,
       SnapshotInfo snapshotInfo, OmSnapshotLocalData snapshotLocalData) throws IOException {
     final UUID snapshotId = snapshotInfo.getSnapshotId();
-    final long dbTxSequenceNumber = snapshotInfo.getDbTxSequenceNumber();
+    final long dbTxSequenceNumber = snapshotLocalData.getDbTxSequenceNumber();
     NavigableMap<Integer, List<SstFileInfo>> versionSstFiles = snapshotLocalData.getVersionSstFileInfos().entrySet()
         .stream().collect(toMap(Map.Entry::getKey,
             entry -> entry.getValue().getSstFiles(), (u, v) -> {

hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotLocalDataYaml.java

@@ -118,7 +118,7 @@ private Pair<File, UUID> writeToYaml(UUID snapshotId, String snapshotName, Trans
         createLiveFileMetaData("sst2", "table1", "k3", "k4"),
         createLiveFileMetaData("sst3", "table2", "k4", "k5"));
     OmSnapshotLocalData dataYaml = new OmSnapshotLocalData(snapshotId, notDefraggedSSTFileList,
-        previousSnapshotId, transactionInfo);
+        previousSnapshotId, transactionInfo, 10);
 
     // Set version
     dataYaml.setVersion(42);
@@ -164,6 +164,7 @@ public void testWriteToYaml() throws IOException {
 
     // Verify fields
     assertEquals(44, snapshotData.getVersion());
+    assertEquals(10, snapshotData.getDbTxSequenceNumber());
     assertTrue(snapshotData.getSstFiltered());
     assertEquals(transactionInfo, snapshotData.getTransactionInfo());