diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java index 9f7a1421537f..42ca3f97b3b0 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java @@ -222,6 +222,7 @@ public final class OzoneConsts { public static final String OM_SST_FILE_INFO_END_KEY = "endKey"; public static final String OM_SST_FILE_INFO_COL_FAMILY = "columnFamily"; public static final String OM_SLD_TXN_INFO = "transactionInfo"; + public static final String OM_SLD_DB_TXN_SEQ_NUMBER = "dbTxSequenceNumber"; // YAML fields for .container files public static final String CONTAINER_ID = "containerID"; diff --git a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java index e11abc3fdb16..7c4b878d62be 100644 --- a/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java +++ b/hadoop-hdds/rocksdb-checkpoint-differ/src/main/java/org/apache/ozone/rocksdiff/RocksDBCheckpointDiffer.java @@ -771,9 +771,7 @@ private Path getSSTFullPath(SstFileInfo sstFileInfo, Path... dbPaths) throws IOE * @param dest destination snapshot * @param versionMap version map containing the connection between source snapshot version and dest snapshot version. * @param tablesToLookup tablesToLookup set of table (column family) names used to restrict which SST files to return. - * @return A list of SST files without extension. - * e.g. ["/path/to/sstBackupDir/000050.sst", - * "/path/to/sstBackupDir/000060.sst"] + * @return map of SST file absolute paths with extension to SstFileInfo. */ public synchronized Optional> getSSTDiffListWithFullPath(DifferSnapshotInfo src, DifferSnapshotInfo dest, Map versionMap, TablePrefixInfo prefixInfo, diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java index 2e899aeca6e8..0e2a4b6ee796 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotInfo.java @@ -71,10 +71,6 @@ public final class SnapshotInfo implements Auditable, CopyObject { private UUID pathPreviousSnapshotId; private UUID globalPreviousSnapshotId; private String snapshotPath; // snapshot mask - /** - * RocksDB's transaction sequence number at the time of checkpoint creation. - */ - private long dbTxSequenceNumber; private boolean deepClean; private boolean sstFiltered; private long referencedSize; @@ -98,7 +94,6 @@ private SnapshotInfo(Builder b) { this.pathPreviousSnapshotId = b.pathPreviousSnapshotId; this.globalPreviousSnapshotId = b.globalPreviousSnapshotId; this.snapshotPath = b.snapshotPath; - this.dbTxSequenceNumber = b.dbTxSequenceNumber; this.deepClean = b.deepClean; this.sstFiltered = b.sstFiltered; this.referencedSize = b.referencedSize; @@ -221,7 +216,6 @@ public SnapshotInfo.Builder toBuilder() { .setPathPreviousSnapshotId(pathPreviousSnapshotId) .setGlobalPreviousSnapshotId(globalPreviousSnapshotId) .setSnapshotPath(snapshotPath) - .setDbTxSequenceNumber(dbTxSequenceNumber) .setDeepClean(deepClean) .setSstFiltered(sstFiltered) .setReferencedSize(referencedSize) @@ -249,7 +243,6 @@ public static class Builder { private UUID pathPreviousSnapshotId; private UUID globalPreviousSnapshotId; private String snapshotPath; - private long dbTxSequenceNumber; private boolean deepClean; private boolean sstFiltered; private long referencedSize; @@ -327,12 +320,6 @@ public Builder setSnapshotPath(String snapshotPath) { return this; } - /** @param dbTxSequenceNumber - RDB latest transaction sequence number. */ - public Builder setDbTxSequenceNumber(long dbTxSequenceNumber) { - this.dbTxSequenceNumber = dbTxSequenceNumber; - return this; - } - /** @param deepClean - To be deep cleaned status for snapshot. */ public Builder setDeepClean(boolean deepClean) { this.deepClean = deepClean; @@ -441,7 +428,6 @@ public OzoneManagerProtocolProtos.SnapshotInfo getProtobuf() { } sib.setSnapshotPath(snapshotPath) - .setDbTxSequenceNumber(dbTxSequenceNumber) .setDeepClean(deepClean); return sib.build(); } @@ -523,9 +509,7 @@ public static Builder builderFromProtobuf( if (snapshotInfoProto.hasCreateTransactionInfo()) { osib.setCreateTransactionInfo(snapshotInfoProto.getCreateTransactionInfo()); } - - osib.setSnapshotPath(snapshotInfoProto.getSnapshotPath()) - .setDbTxSequenceNumber(snapshotInfoProto.getDbTxSequenceNumber()); + osib.setSnapshotPath(snapshotInfoProto.getSnapshotPath()); return osib; } @@ -568,14 +552,6 @@ public String getCheckpointDirName(int version) { return getCheckpointDirName(getSnapshotId(), version); } - public long getDbTxSequenceNumber() { - return dbTxSequenceNumber; - } - - public void setDbTxSequenceNumber(long dbTxSequenceNumber) { - this.dbTxSequenceNumber = dbTxSequenceNumber; - } - /** * Get the table key for this snapshot. */ @@ -761,7 +737,6 @@ public String toString() { ", pathPreviousSnapshotId: '" + pathPreviousSnapshotId + '\'' + ", globalPreviousSnapshotId: '" + globalPreviousSnapshotId + '\'' + ", snapshotPath: '" + snapshotPath + '\'' + - ", dbTxSequenceNumber: '" + dbTxSequenceNumber + '\'' + ", deepClean: '" + deepClean + '\'' + ", sstFiltered: '" + sstFiltered + '\'' + ", referencedSize: '" + referencedSize + '\'' + diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmSnapshotInfo.java b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmSnapshotInfo.java index e7695debd619..30f7e475c211 100644 --- a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmSnapshotInfo.java +++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmSnapshotInfo.java @@ -45,7 +45,6 @@ public class TestOmSnapshotInfo { private static final UUID GLOBAL_PREVIOUS_SNAPSHOT_ID = PATH_PREVIOUS_SNAPSHOT_ID; private static final String SNAPSHOT_PATH = "test/path"; - private static final long DB_TX_SEQUENCE_NUMBER = 12345L; private SnapshotInfo createSnapshotInfo() { return new SnapshotInfo.Builder() @@ -59,7 +58,6 @@ private SnapshotInfo createSnapshotInfo() { .setPathPreviousSnapshotId(PATH_PREVIOUS_SNAPSHOT_ID) .setGlobalPreviousSnapshotId(GLOBAL_PREVIOUS_SNAPSHOT_ID) .setSnapshotPath(SNAPSHOT_PATH) - .setDbTxSequenceNumber(DB_TX_SEQUENCE_NUMBER) .setDeepClean(false) .setSstFiltered(false) .setReferencedSize(2000L) @@ -84,7 +82,6 @@ private OzoneManagerProtocolProtos.SnapshotInfo createSnapshotInfoProto() { .setPathPreviousSnapshotID(toProtobuf(PATH_PREVIOUS_SNAPSHOT_ID)) .setGlobalPreviousSnapshotID(toProtobuf(GLOBAL_PREVIOUS_SNAPSHOT_ID)) .setSnapshotPath(SNAPSHOT_PATH) - .setDbTxSequenceNumber(DB_TX_SEQUENCE_NUMBER) .setDeepClean(false) .setSstFiltered(false) .setReferencedSize(2000L) @@ -164,8 +161,6 @@ public void testSnapshotInfoProtoToSnapshotInfo() { snapshotInfoActual.getBucketName()); assertEquals(snapshotInfoExpected.getSnapshotStatus(), snapshotInfoActual.getSnapshotStatus()); - assertEquals(snapshotInfoExpected.getDbTxSequenceNumber(), - snapshotInfoActual.getDbTxSequenceNumber()); assertEquals(snapshotInfoExpected.isDeepCleaned(), snapshotInfoActual.isDeepCleaned()); assertEquals(snapshotInfoExpected.isSstFiltered(), diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOMSnapshotDAG.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOMSnapshotDAG.java index 3901eeeb0e4d..c421e02705c8 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOMSnapshotDAG.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOMSnapshotDAG.java @@ -161,7 +161,8 @@ private DifferSnapshotVersion getDifferSnapshotInfo( throw new IllegalStateException(String.format("Duplicate key %s", u)); }, TreeMap::new)); DifferSnapshotInfo dsi = new DifferSnapshotInfo((version) -> Paths.get(checkpointPath), - snapshotInfo.getSnapshotId(), snapshotInfo.getDbTxSequenceNumber(), versionSstFiles); + snapshotInfo.getSnapshotId(), snapshotLocalData.getSnapshotLocalData().getDbTxSequenceNumber(), + versionSstFiles); return new DifferSnapshotVersion(dsi, 0, COLUMN_FAMILIES_TO_TRACK_IN_DAG); } } diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index 9d4177005371..bdb3cc3cee35 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -881,7 +881,7 @@ message SnapshotInfo { optional hadoop.hdds.UUID globalPreviousSnapshotID = 9; optional string snapshotPath = 10; optional string checkpointDir = 11 [deprecated = true]; - optional int64 dbTxSequenceNumber = 12; + optional int64 dbTxSequenceNumber = 12 [deprecated = true]; optional bool deepClean = 13; optional bool sstFiltered = 14; // snapshot reference size before any key replication or EC diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotLocalData.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotLocalData.java index 91ec8b673a89..f876a9606017 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotLocalData.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotLocalData.java @@ -67,6 +67,9 @@ public class OmSnapshotLocalData implements WithChecksum { // Stores the transactionInfo corresponding to OM when the snaphot is purged. private TransactionInfo transactionInfo; + // Stores the rocksDB's transaction sequence number at the time of snapshot creation. + private long dbTxSequenceNumber; + // Map of version to VersionMeta, using linkedHashMap since the order of the map needs to be deterministic for // checksum computation. private final LinkedHashMap versionSstFileInfos; @@ -78,7 +81,7 @@ public class OmSnapshotLocalData implements WithChecksum { * Creates a OmSnapshotLocalData object with default values. */ public OmSnapshotLocalData(UUID snapshotId, List notDefraggedSSTFileList, UUID previousSnapshotId, - TransactionInfo transactionInfo) { + TransactionInfo transactionInfo, long dbTxSequenceNumber) { this.snapshotId = snapshotId; this.isSSTFiltered = false; this.lastDefragTime = 0L; @@ -89,9 +92,14 @@ public OmSnapshotLocalData(UUID snapshotId, List notDefraggedS this.version = 0; this.previousSnapshotId = previousSnapshotId; this.transactionInfo = transactionInfo; + this.dbTxSequenceNumber = dbTxSequenceNumber; setChecksumTo0ByteArray(); } + public long getDbTxSequenceNumber() { + return dbTxSequenceNumber; + } + /** * Copy constructor to create a deep copy of OmSnapshotLocalData object. * @param source The source OmSnapshotLocalData to copy from @@ -108,6 +116,7 @@ public OmSnapshotLocalData(OmSnapshotLocalData source) { this.versionSstFileInfos = new LinkedHashMap<>(); setVersionSstFileInfos(source.versionSstFileInfos); this.transactionInfo = source.transactionInfo; + this.dbTxSequenceNumber = source.dbTxSequenceNumber; } public TransactionInfo getTransactionInfo() { diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotLocalDataYaml.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotLocalDataYaml.java index ad8046d719e0..b72e74cf4a6b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotLocalDataYaml.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotLocalDataYaml.java @@ -172,9 +172,10 @@ public Object construct(Node node) { final String prevSnapIdStr = (String) nodes.get(OzoneConsts.OM_SLD_PREV_SNAP_ID); UUID prevSnapId = prevSnapIdStr != null ? UUID.fromString(prevSnapIdStr) : null; final String purgeTxInfoStr = (String) nodes.get(OzoneConsts.OM_SLD_TXN_INFO); + final long dbTxnSeqNumber = ((Number)nodes.get(OzoneConsts.OM_SLD_DB_TXN_SEQ_NUMBER)).longValue(); TransactionInfo transactionInfo = purgeTxInfoStr != null ? TransactionInfo.valueOf(purgeTxInfoStr) : null; OmSnapshotLocalData snapshotLocalData = new OmSnapshotLocalData(snapId, Collections.emptyList(), prevSnapId, - transactionInfo); + transactionInfo, dbTxnSeqNumber); // Set version from YAML Integer version = (Integer) nodes.get(OzoneConsts.OM_SLD_VERSION); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java index 6211d4114005..07a8aeed3139 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/snapshot/OMSnapshotCreateRequest.java @@ -32,7 +32,6 @@ import org.apache.hadoop.hdds.client.DefaultReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.utils.TransactionInfo; -import org.apache.hadoop.hdds.utils.db.RDBStore; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; import org.apache.hadoop.ozone.OmUtils; @@ -167,12 +166,6 @@ public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, Execut throw new OMException("Snapshot already exists", FILE_ALREADY_EXISTS); } - // Note down RDB latest transaction sequence number, which is used - // as snapshot generation in the Differ. - final long dbLatestSequenceNumber = - ((RDBStore) omMetadataManager.getStore()).getDb() - .getLatestSequenceNumber(); - snapshotInfo.setDbTxSequenceNumber(dbLatestSequenceNumber); ByteString txnBytes = TransactionInfo.valueOf(context.getTermIndex()).toByteString(); snapshotInfo.setCreateTransactionInfo(txnBytes); snapshotInfo.setLastTransactionInfo(txnBytes); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotLocalDataManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotLocalDataManager.java index be2a6706049a..429c6776cab3 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotLocalDataManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/OmSnapshotLocalDataManager.java @@ -181,11 +181,14 @@ public String getSnapshotLocalPropertyYamlPath(UUID snapshotId) { public void createNewOmSnapshotLocalDataFile(RDBStore snapshotStore, SnapshotInfo snapshotInfo) throws IOException { try (WritableOmSnapshotLocalDataProvider snapshotLocalData = new WritableOmSnapshotLocalDataProvider(snapshotInfo.getSnapshotId(), - () -> Pair.of(new OmSnapshotLocalData(snapshotInfo.getSnapshotId(), - getLiveSSTFilesForCFs(snapshotStore.getDb().getManagedRocksDb(), - COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT), - snapshotInfo.getPathPreviousSnapshotId(), null), - null))) { + () -> { + List lfms = getLiveSSTFilesForCFs(snapshotStore.getDb().getManagedRocksDb(), + COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT); + long dbTxnSeqNumber = lfms.stream().mapToLong(LiveFileMetaData::largestSeqno).max().orElse(0L); + OmSnapshotLocalData localData = new OmSnapshotLocalData(snapshotInfo.getSnapshotId(), + lfms, snapshotInfo.getPathPreviousSnapshotId(), null, dbTxnSeqNumber); + return Pair.of(localData, null); + })) { snapshotLocalData.commit(); } } @@ -263,16 +266,18 @@ private void addMissingSnapshotYamlFiles( // Create a yaml file for snapshots which are missing if (!snapshotLocalDataFile.exists()) { List sstList = Collections.emptyList(); + long dbTxnSeqNumber = 0L; if (snapshotInfo.getSnapshotStatus() == SNAPSHOT_ACTIVE) { try (OmMetadataManagerImpl snapshotMetadataManager = defaultSnapProvider.apply(snapshotInfo)) { ManagedRocksDB snapDB = ((RDBStore)snapshotMetadataManager.getStore()).getDb().getManagedRocksDb(); sstList = getLiveSSTFilesForCFs(snapDB, COLUMN_FAMILIES_TO_TRACK_IN_SNAPSHOT); + dbTxnSeqNumber = sstList.stream().mapToLong(LiveFileMetaData::largestSeqno).max().orElse(0L); } catch (Exception e) { throw new IOException(e); } } OmSnapshotLocalData snapshotLocalData = new OmSnapshotLocalData(snapshotId, sstList, - snapshotInfo.getPathPreviousSnapshotId(), null); + snapshotInfo.getPathPreviousSnapshotId(), null, dbTxnSeqNumber); // Set needsDefrag to true to indicate that the snapshot needs to be defragmented, since the snapshot has // never been defragmented before. snapshotLocalData.setNeedsDefrag(true); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/diff/delta/RDBDifferComputer.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/diff/delta/RDBDifferComputer.java index 0a59029fb0f4..7ab7a5a68d75 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/diff/delta/RDBDifferComputer.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/diff/delta/RDBDifferComputer.java @@ -73,9 +73,9 @@ public Optional>> computeDeltaFiles(SnapshotIn if (differ != null) { try (OmSnapshotLocalDataManager.ReadableOmSnapshotLocalDataProvider snapProvider = getLocalDataProvider(toSnapshot.getSnapshotId(), fromSnapshot.getSnapshotId())) { - final DifferSnapshotInfo fromDSI = getDSIFromSI(getActiveMetadataManager(), fromSnapshot, + final DifferSnapshotInfo fromDSI = toDifferSnapshotInfo(getActiveMetadataManager(), fromSnapshot, snapProvider.getPreviousSnapshotLocalData()); - final DifferSnapshotInfo toDSI = getDSIFromSI(getActiveMetadataManager(), toSnapshot, + final DifferSnapshotInfo toDSI = toDifferSnapshotInfo(getActiveMetadataManager(), toSnapshot, snapProvider.getSnapshotLocalData()); final Map versionMap = snapProvider.getSnapshotLocalData().getVersionSstFileInfos().entrySet() .stream().collect(toMap(Map.Entry::getKey, entry -> entry.getValue().getPreviousSnapshotVersion())); @@ -98,10 +98,10 @@ public Optional>> computeDeltaFiles(SnapshotIn /** * Convert from SnapshotInfo to DifferSnapshotInfo. */ - private static DifferSnapshotInfo getDSIFromSI(OMMetadataManager activeOmMetadataManager, + private static DifferSnapshotInfo toDifferSnapshotInfo(OMMetadataManager activeOmMetadataManager, SnapshotInfo snapshotInfo, OmSnapshotLocalData snapshotLocalData) throws IOException { final UUID snapshotId = snapshotInfo.getSnapshotId(); - final long dbTxSequenceNumber = snapshotInfo.getDbTxSequenceNumber(); + final long dbTxSequenceNumber = snapshotLocalData.getDbTxSequenceNumber(); NavigableMap> versionSstFiles = snapshotLocalData.getVersionSstFileInfos().entrySet() .stream().collect(toMap(Map.Entry::getKey, entry -> entry.getValue().getSstFiles(), (u, v) -> { diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotLocalDataYaml.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotLocalDataYaml.java index 81f111e8464b..34b9fbe397ec 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotLocalDataYaml.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmSnapshotLocalDataYaml.java @@ -118,7 +118,7 @@ private Pair writeToYaml(UUID snapshotId, String snapshotName, Trans createLiveFileMetaData("sst2", "table1", "k3", "k4"), createLiveFileMetaData("sst3", "table2", "k4", "k5")); OmSnapshotLocalData dataYaml = new OmSnapshotLocalData(snapshotId, notDefraggedSSTFileList, - previousSnapshotId, transactionInfo); + previousSnapshotId, transactionInfo, 10); // Set version dataYaml.setVersion(42); @@ -164,6 +164,7 @@ public void testWriteToYaml() throws IOException { // Verify fields assertEquals(44, snapshotData.getVersion()); + assertEquals(10, snapshotData.getDbTxSequenceNumber()); assertTrue(snapshotData.getSstFiltered()); assertEquals(transactionInfo, snapshotData.getTransactionInfo()); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotLocalDataManager.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotLocalDataManager.java index 6e804f330ae8..76c013198e35 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotLocalDataManager.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/TestOmSnapshotLocalDataManager.java @@ -790,15 +790,17 @@ public void testCreateNewSnapshotLocalYaml() throws IOException { expNotDefraggedSSTFileList.put(DIRECTORY_TABLE, Stream.of("dt1", "dt2").collect(Collectors.toList())); List mockedLiveFiles = new ArrayList<>(); + int seqNumber = 0; for (Map.Entry> entry : expNotDefraggedSSTFileList.entrySet()) { String cfname = entry.getKey(); for (String fname : entry.getValue()) { - mockedLiveFiles.add(createMockLiveFileMetaData("/" + fname + ".sst", cfname, "k1", "k2")); + mockedLiveFiles.add(createMockLiveFileMetaData("/" + fname + ".sst", cfname, "k1", "k2", seqNumber++)); } } + int expectedDbTxSequenceNumber = seqNumber - 1; // Add some other column families and files that should be ignored - mockedLiveFiles.add(createMockLiveFileMetaData("ot1.sst", "otherTable", "k1", "k2")); - mockedLiveFiles.add(createMockLiveFileMetaData("ot2.sst", "otherTable", "k1", "k2")); + mockedLiveFiles.add(createMockLiveFileMetaData("ot1.sst", "otherTable", "k1", "k2", seqNumber++)); + mockedLiveFiles.add(createMockLiveFileMetaData("ot2.sst", "otherTable", "k1", "k2", seqNumber)); mockSnapshotStore(snapshotId, mockedLiveFiles); localDataManager = getNewOmSnapshotLocalDataManager(); @@ -820,6 +822,7 @@ public void testCreateNewSnapshotLocalYaml() throws IOException { assertEquals(0L, localData.getLastDefragTime()); assertTrue(localData.getNeedsDefrag()); assertEquals(1, localData.getVersionSstFileInfos().size()); + assertEquals(expectedDbTxSequenceNumber, localData.getDbTxSequenceNumber()); } @Test @@ -829,12 +832,12 @@ public void testCreateNewOmSnapshotLocalDataFile() throws IOException { // Setup snapshot store mock List sstFiles = new ArrayList<>(); - sstFiles.add(createMockLiveFileMetaData("file1.sst", KEY_TABLE, "key1", "key7")); - sstFiles.add(createMockLiveFileMetaData("file2.sst", KEY_TABLE, "key3", "key9")); - sstFiles.add(createMockLiveFileMetaData("file3.sst", FILE_TABLE, "key1", "key7")); - sstFiles.add(createMockLiveFileMetaData("file4.sst", FILE_TABLE, "key1", "key7")); - sstFiles.add(createMockLiveFileMetaData("file5.sst", DIRECTORY_TABLE, "key1", "key7")); - sstFiles.add(createMockLiveFileMetaData("file6.sst", "colFamily1", "key1", "key7")); + sstFiles.add(createMockLiveFileMetaData("file1.sst", KEY_TABLE, "key1", "key7", 10)); + sstFiles.add(createMockLiveFileMetaData("file2.sst", KEY_TABLE, "key3", "key9", 20)); + sstFiles.add(createMockLiveFileMetaData("file3.sst", FILE_TABLE, "key1", "key7", 30)); + sstFiles.add(createMockLiveFileMetaData("file4.sst", FILE_TABLE, "key1", "key7", 100)); + sstFiles.add(createMockLiveFileMetaData("file5.sst", DIRECTORY_TABLE, "key1", "key7", 5000)); + sstFiles.add(createMockLiveFileMetaData("file6.sst", "colFamily1", "key1", "key7", 6000)); List sstFileInfos = IntStream.range(0, sstFiles.size() - 1) .mapToObj(sstFiles::get).map(lfm -> new SstFileInfo(lfm.fileName().replace(".sst", ""), @@ -856,6 +859,7 @@ public void testCreateNewOmSnapshotLocalDataFile() throws IOException { assertEquals(expectedVersionMeta, versionMeta); // New Snapshot create needs to be defragged always. assertTrue(snapshotLocalData.needsDefrag()); + assertEquals(5000, snapshotLocalData.getSnapshotLocalData().getDbTxSequenceNumber()); } } @@ -1062,11 +1066,17 @@ private SnapshotInfo createMockSnapshotInfo(UUID snapshotId, UUID previousSnapsh private LiveFileMetaData createMockLiveFileMetaData(String fileName, String columnFamilyName, String smallestKey, String largestKey) { + return createMockLiveFileMetaData(fileName, columnFamilyName, smallestKey, largestKey, 0); + } + + private LiveFileMetaData createMockLiveFileMetaData(String fileName, String columnFamilyName, String smallestKey, + String largestKey, long largestSeqNumber) { LiveFileMetaData liveFileMetaData = mock(LiveFileMetaData.class); when(liveFileMetaData.columnFamilyName()).thenReturn(StringUtils.string2Bytes(columnFamilyName)); when(liveFileMetaData.fileName()).thenReturn(fileName); when(liveFileMetaData.smallestKey()).thenReturn(StringUtils.string2Bytes(smallestKey)); when(liveFileMetaData.largestKey()).thenReturn(StringUtils.string2Bytes(largestKey)); + when(liveFileMetaData.largestSeqno()).thenReturn(largestSeqNumber); return liveFileMetaData; } @@ -1076,7 +1086,7 @@ private OmSnapshotLocalData createMockLocalData(UUID snapshotId, UUID previousSn sstFiles.add(createMockLiveFileMetaData("file2.sst", "columnFamily1", "key3", "key10")); sstFiles.add(createMockLiveFileMetaData("file3.sst", "columnFamily2", "key1", "key8")); sstFiles.add(createMockLiveFileMetaData("file4.sst", "columnFamily2", "key0", "key10")); - return new OmSnapshotLocalData(snapshotId, sstFiles, previousSnapshotId, null); + return new OmSnapshotLocalData(snapshotId, sstFiles, previousSnapshotId, null, 10); } private void createSnapshotLocalDataFile(UUID snapshotId, UUID previousSnapshotId) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/diff/delta/TestCompositeDeltaDiffComputer.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/diff/delta/TestCompositeDeltaDiffComputer.java index e8af3f84dd72..b64520a05c14 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/diff/delta/TestCompositeDeltaDiffComputer.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/diff/delta/TestCompositeDeltaDiffComputer.java @@ -718,8 +718,7 @@ private SnapshotInfo createMockSnapshotInfo(String volumeName, String bucketName .setVolumeName(volumeName) .setBucketName(bucketName) .setName(snapshotName) - .setSnapshotId(snapshotId) - .setDbTxSequenceNumber(100L); + .setSnapshotId(snapshotId); return builder.build(); } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/diff/delta/TestRDBDifferComputer.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/diff/delta/TestRDBDifferComputer.java index b4ba058a43c2..19579a59e16e 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/diff/delta/TestRDBDifferComputer.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/diff/delta/TestRDBDifferComputer.java @@ -360,10 +360,10 @@ public void testComputeDeltaFilesWithVersionMapping() throws IOException { } /** - * Tests that getDSIFromSI throws exception when no versions found. + * Tests that toDifferSnapshotInfo throws exception when no versions found. */ @Test - public void testGetDSIFromSIWithNoVersions() throws IOException { + public void testToDifferSnapshotInfoWithNoVersions() throws IOException { rdbDifferComputer = new RDBDifferComputer(omSnapshotManager, activeMetadataManager, deltaDirPath, activityReporter); @@ -486,8 +486,7 @@ private SnapshotInfo createMockSnapshotInfo(String volumeName, String bucketName .setVolumeName(volumeName) .setBucketName(bucketName) .setName(snapshotName) - .setSnapshotId(snapshotId) - .setDbTxSequenceNumber(100L); + .setSnapshotId(snapshotId); return builder.build(); } @@ -528,8 +527,3 @@ private OmSnapshotLocalData createMockSnapshotLocalDataWithVersions(UUID snapsho return localData; } } - - - - -